From ba02209141c08004c13904a4bc847397802014d7 Mon Sep 17 00:00:00 2001 From: Renuka Manavalan <47282725+renukamanavalan@users.noreply.github.com> Date: Tue, 22 Dec 2020 08:01:33 -0800 Subject: [PATCH] First cut image update for kubernetes support. (#5421) * First cut image update for kubernetes support. With this, 1) dockers dhcp_relay, lldp, pmon, radv, snmp, telemetry are enabled for kube management init_cfg.json configure set_owner as kube for these 2) Each docker's start.sh updated to call container_startup.py to register going up As part of this call, it registers the current owner as local/kube and its version The images are built with its version ingrained into image during build 3) Update all docker's bash script to call 'container start/stop/wait' instead of 'docker start/stop/wait'. For all locally managed containers, it calls docker commands, hence no change for locally managed. 4) Introduced a new ctrmgrd service, that helps with transition between owners as kube & local and carry over any labels update from STATE-DB to API server 5) hostcfgd updated to handle owner change 6) Reboot scripts are updatd to tag kube running images as local, so upon reboot they run the same image. 7) Added kube_commands.py to handle all updates with Kubernetes API serrver -- dedicated for k8s interaction only. --- dockers/docker-dhcp-relay/Dockerfile.j2 | 4 + dockers/docker-dhcp-relay/start.sh | 10 + dockers/docker-lldp/Dockerfile.j2 | 4 + dockers/docker-lldp/start.sh | 10 + dockers/docker-platform-monitor/Dockerfile.j2 | 4 + dockers/docker-platform-monitor/start.sh | 10 + .../docker-router-advertiser/Dockerfile.j2 | 5 + ...cker-router-advertiser.supervisord.conf.j2 | 13 +- dockers/docker-router-advertiser/start.sh | 12 + dockers/docker-snmp/Dockerfile.j2 | 4 + dockers/docker-snmp/start.sh | 11 + dockers/docker-sonic-telemetry/Dockerfile.j2 | 4 + dockers/docker-sonic-telemetry/start.sh | 10 + files/build_templates/docker_image_ctl.j2 | 20 +- files/build_templates/init_cfg.json.j2 | 4 + .../kube_cni.10-flannel.conflist | 21 + .../build_templates/sonic_debian_extension.j2 | 49 +- rules/docker-config-engine-buster.mk | 1 + rules/docker-dhcp-relay.mk | 2 + rules/docker-lldp.mk | 1 + rules/docker-platform-monitor.mk | 1 + rules/docker-router-advertiser.mk | 1 + rules/docker-snmp.mk | 1 + rules/docker-telemetry.mk | 1 + rules/sonic-ctrmgrd.dep | 8 + rules/sonic-ctrmgrd.mk | 31 + slave.mk | 5 + src/sonic-ctrmgrd/.gitignore | 12 + src/sonic-ctrmgrd/README.rst | 3 + src/sonic-ctrmgrd/ctrmgr/__init__.py | 0 src/sonic-ctrmgrd/ctrmgr/container | 409 +++++++++++ src/sonic-ctrmgrd/ctrmgr/container_startup.py | 291 ++++++++ src/sonic-ctrmgrd/ctrmgr/ctrmgr_tools.py | 147 ++++ src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py | 598 ++++++++++++++++ src/sonic-ctrmgrd/ctrmgr/ctrmgrd.service | 14 + src/sonic-ctrmgrd/ctrmgr/kube_commands.py | 391 ++++++++++ .../ctrmgr/remote_ctr.config.json | 7 + src/sonic-ctrmgrd/pytest.ini | 2 + src/sonic-ctrmgrd/setup.cfg | 5 + src/sonic-ctrmgrd/setup.py | 39 + src/sonic-ctrmgrd/tests/__init__.py | 0 src/sonic-ctrmgrd/tests/common_test.py | 666 ++++++++++++++++++ .../tests/container_startup_test.py | 389 ++++++++++ src/sonic-ctrmgrd/tests/container_test.py | 532 ++++++++++++++ src/sonic-ctrmgrd/tests/ctrmgr_tools_test.py | 292 ++++++++ src/sonic-ctrmgrd/tests/ctrmgrd_test.py | 488 +++++++++++++ src/sonic-ctrmgrd/tests/kube_commands_test.py | 391 ++++++++++ src/sonic-ctrmgrd/tests/mock_docker.py | 2 + 48 files changed, 4917 insertions(+), 8 deletions(-) create mode 100755 dockers/docker-router-advertiser/start.sh create mode 100644 files/build_templates/kube_cni.10-flannel.conflist create mode 100644 rules/sonic-ctrmgrd.dep create mode 100644 rules/sonic-ctrmgrd.mk create mode 100644 src/sonic-ctrmgrd/.gitignore create mode 100644 src/sonic-ctrmgrd/README.rst create mode 100644 src/sonic-ctrmgrd/ctrmgr/__init__.py create mode 100755 src/sonic-ctrmgrd/ctrmgr/container create mode 100755 src/sonic-ctrmgrd/ctrmgr/container_startup.py create mode 100755 src/sonic-ctrmgrd/ctrmgr/ctrmgr_tools.py create mode 100755 src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py create mode 100644 src/sonic-ctrmgrd/ctrmgr/ctrmgrd.service create mode 100755 src/sonic-ctrmgrd/ctrmgr/kube_commands.py create mode 100644 src/sonic-ctrmgrd/ctrmgr/remote_ctr.config.json create mode 100644 src/sonic-ctrmgrd/pytest.ini create mode 100644 src/sonic-ctrmgrd/setup.cfg create mode 100644 src/sonic-ctrmgrd/setup.py create mode 100644 src/sonic-ctrmgrd/tests/__init__.py create mode 100755 src/sonic-ctrmgrd/tests/common_test.py create mode 100755 src/sonic-ctrmgrd/tests/container_startup_test.py create mode 100755 src/sonic-ctrmgrd/tests/container_test.py create mode 100755 src/sonic-ctrmgrd/tests/ctrmgr_tools_test.py create mode 100755 src/sonic-ctrmgrd/tests/ctrmgrd_test.py create mode 100755 src/sonic-ctrmgrd/tests/kube_commands_test.py create mode 100644 src/sonic-ctrmgrd/tests/mock_docker.py diff --git a/dockers/docker-dhcp-relay/Dockerfile.j2 b/dockers/docker-dhcp-relay/Dockerfile.j2 index 58796ca79e..1100aa5107 100644 --- a/dockers/docker-dhcp-relay/Dockerfile.j2 +++ b/dockers/docker-dhcp-relay/Dockerfile.j2 @@ -2,11 +2,15 @@ FROM docker-config-engine-buster ARG docker_container_name +ARG image_version RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf # Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Pass the image_version to container +ENV IMAGE_VERSION=$image_version + # Update apt's cache of available packages RUN apt-get update diff --git a/dockers/docker-dhcp-relay/start.sh b/dockers/docker-dhcp-relay/start.sh index 86ba91147e..cb563eb003 100755 --- a/dockers/docker-dhcp-relay/start.sh +++ b/dockers/docker-dhcp-relay/start.sh @@ -1,5 +1,15 @@ #!/usr/bin/env bash +if [ "${RUNTIME_OWNER}" == "" ]; then + RUNTIME_OWNER="kube" +fi + +CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py" +if test -f ${CTR_SCRIPT} +then + ${CTR_SCRIPT} -f dhcp_relay -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION} +fi + # If our supervisor config has entries in the "isc-dhcp-relay" group... if [ $(supervisorctl status | grep -c "^isc-dhcp-relay:") -gt 0 ]; then # Wait for all interfaces to come up and be assigned IPv4 addresses before diff --git a/dockers/docker-lldp/Dockerfile.j2 b/dockers/docker-lldp/Dockerfile.j2 index 1306582b0a..297f776651 100644 --- a/dockers/docker-lldp/Dockerfile.j2 +++ b/dockers/docker-lldp/Dockerfile.j2 @@ -2,11 +2,15 @@ FROM docker-config-engine-buster ARG docker_container_name +ARG image_version RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf # Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Pass the image_version to container +ENV IMAGE_VERSION=$image_version + # Update apt's cache of available packages RUN apt-get update diff --git a/dockers/docker-lldp/start.sh b/dockers/docker-lldp/start.sh index 5cb6042cee..b2c82d6d04 100755 --- a/dockers/docker-lldp/start.sh +++ b/dockers/docker-lldp/start.sh @@ -1,5 +1,15 @@ #!/usr/bin/env bash +if [ "${RUNTIME_OWNER}" == "" ]; then + RUNTIME_OWNER="kube" +fi + +CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py" +if test -f ${CTR_SCRIPT} +then + ${CTR_SCRIPT} -f lldp -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION} +fi + sonic-cfggen -d -t /usr/share/sonic/templates/lldpd.conf.j2 > /etc/lldpd.conf mkdir -p /var/sonic diff --git a/dockers/docker-platform-monitor/Dockerfile.j2 b/dockers/docker-platform-monitor/Dockerfile.j2 index b45fbc84d9..1aed981db1 100755 --- a/dockers/docker-platform-monitor/Dockerfile.j2 +++ b/dockers/docker-platform-monitor/Dockerfile.j2 @@ -2,11 +2,15 @@ FROM docker-config-engine-buster ARG docker_container_name +ARG image_version RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf # Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Pass the image_version to container +ENV IMAGE_VERSION=$image_version + # Install required packages RUN apt-get update && \ apt-get install -y \ diff --git a/dockers/docker-platform-monitor/start.sh b/dockers/docker-platform-monitor/start.sh index 2f6dc61a84..98aa974e13 100755 --- a/dockers/docker-platform-monitor/start.sh +++ b/dockers/docker-platform-monitor/start.sh @@ -2,6 +2,16 @@ declare -r EXIT_SUCCESS="0" +if [ "${RUNTIME_OWNER}" == "" ]; then + RUNTIME_OWNER="kube" +fi + +CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py" +if test -f ${CTR_SCRIPT} +then + ${CTR_SCRIPT} -f pmon -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION} +fi + mkdir -p /var/sonic echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status diff --git a/dockers/docker-router-advertiser/Dockerfile.j2 b/dockers/docker-router-advertiser/Dockerfile.j2 index 3896286556..7d225cbd4e 100644 --- a/dockers/docker-router-advertiser/Dockerfile.j2 +++ b/dockers/docker-router-advertiser/Dockerfile.j2 @@ -2,11 +2,15 @@ FROM docker-config-engine-buster ARG docker_container_name +ARG image_version RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf # Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Pass the image_version to container +ENV IMAGE_VERSION=$image_version + # Update apt's cache of available packages RUN apt-get update @@ -27,6 +31,7 @@ RUN apt-get clean -y && \ apt-get autoremove -y && \ rm -rf /debs +COPY ["start.sh", "/usr/bin/"] COPY ["docker-init.sh", "/usr/bin/"] COPY ["radvd.conf.j2", "wait_for_link.sh.j2", "docker-router-advertiser.supervisord.conf.j2", "/usr/share/sonic/templates/"] COPY ["files/supervisor-proc-exit-listener", "/usr/bin"] diff --git a/dockers/docker-router-advertiser/docker-router-advertiser.supervisord.conf.j2 b/dockers/docker-router-advertiser/docker-router-advertiser.supervisord.conf.j2 index ae73cd6e98..ae48792285 100644 --- a/dockers/docker-router-advertiser/docker-router-advertiser.supervisord.conf.j2 +++ b/dockers/docker-router-advertiser/docker-router-advertiser.supervisord.conf.j2 @@ -27,6 +27,17 @@ stdout_logfile=syslog stderr_logfile=syslog dependent_startup=true +[program:start] +command=/usr/bin/start.sh +priority=1 +autostart=true +autorestart=false +startsecs=0 +stdout_logfile=syslog +stderr_logfile=syslog +dependent_startup=true +dependent_startup_wait_for=rsyslogd:running + {# Router advertiser should only run on ToR (T0) devices which have #} {# at least one VLAN interface which has an IPv6 address asigned #} {%- set vlan_v6 = namespace(count=0) -%} @@ -51,7 +62,7 @@ startsecs=0 stdout_logfile=syslog stderr_logfile=syslog dependent_startup=true -dependent_startup_wait_for=rsyslogd:running +dependent_startup_wait_for=start:exited [program:radvd] command=/usr/sbin/radvd -n diff --git a/dockers/docker-router-advertiser/start.sh b/dockers/docker-router-advertiser/start.sh new file mode 100755 index 0000000000..561cd5d7ce --- /dev/null +++ b/dockers/docker-router-advertiser/start.sh @@ -0,0 +1,12 @@ +#! /bin/bash + +if [ "${RUNTIME_OWNER}" == "" ]; then + RUNTIME_OWNER="kube" +fi + +CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py" +if test -f ${CTR_SCRIPT} +then + ${CTR_SCRIPT} -f radv -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION} +fi + diff --git a/dockers/docker-snmp/Dockerfile.j2 b/dockers/docker-snmp/Dockerfile.j2 index d479392bb2..2fdde61252 100644 --- a/dockers/docker-snmp/Dockerfile.j2 +++ b/dockers/docker-snmp/Dockerfile.j2 @@ -2,6 +2,7 @@ FROM docker-config-engine-buster ARG docker_container_name +ARG image_version RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf # Enable -O for all Python calls @@ -10,6 +11,9 @@ ENV PYTHONOPTIMIZE 1 # Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Pass the image_version to container +ENV IMAGE_VERSION=$image_version + # Update apt's cache of available packages # Install make/gcc which is required for installing hiredis RUN apt-get update && \ diff --git a/dockers/docker-snmp/start.sh b/dockers/docker-snmp/start.sh index 37ba3577e5..7b02c5c086 100755 --- a/dockers/docker-snmp/start.sh +++ b/dockers/docker-snmp/start.sh @@ -1,5 +1,16 @@ #!/usr/bin/env bash + +if [ "${RUNTIME_OWNER}" == "" ]; then + RUNTIME_OWNER="kube" +fi + +CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py" +if test -f ${CTR_SCRIPT} +then + ${CTR_SCRIPT} -f snmp -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION} +fi + mkdir -p /etc/ssw /etc/snmp SONIC_CFGGEN_ARGS=" \ diff --git a/dockers/docker-sonic-telemetry/Dockerfile.j2 b/dockers/docker-sonic-telemetry/Dockerfile.j2 index 65e436d293..88ff943182 100644 --- a/dockers/docker-sonic-telemetry/Dockerfile.j2 +++ b/dockers/docker-sonic-telemetry/Dockerfile.j2 @@ -2,11 +2,15 @@ FROM docker-config-engine-buster ARG docker_container_name +ARG image_version RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf ## Make apt-get non-interactive ENV DEBIAN_FRONTEND=noninteractive +# Pass the image_version to container +ENV IMAGE_VERSION=$image_version + RUN apt-get update {% if docker_sonic_telemetry_debs.strip() -%} diff --git a/dockers/docker-sonic-telemetry/start.sh b/dockers/docker-sonic-telemetry/start.sh index d6722a27fc..08f7292f55 100755 --- a/dockers/docker-sonic-telemetry/start.sh +++ b/dockers/docker-sonic-telemetry/start.sh @@ -1,4 +1,14 @@ #!/usr/bin/env bash +if [ "${RUNTIME_OWNER}" == "" ]; then + RUNTIME_OWNER="kube" +fi + +CTR_SCRIPT="/usr/share/sonic/scripts/container_startup.py" +if test -f ${CTR_SCRIPT} +then + ${CTR_SCRIPT} -f telemetry -o ${RUNTIME_OWNER} -v ${IMAGE_VERSION} +fi + mkdir -p /var/sonic echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status diff --git a/files/build_templates/docker_image_ctl.j2 b/files/build_templates/docker_image_ctl.j2 index 9c7afa84b2..dd37506f37 100644 --- a/files/build_templates/docker_image_ctl.j2 +++ b/files/build_templates/docker_image_ctl.j2 @@ -210,13 +210,14 @@ start() { DOCKERMOUNT=`getMountPoint "$DOCKERCHECK"` {%- endif %} if [ x"$DOCKERMOUNT" == x"$MOUNTPATH" ]; then + preStartAction {%- if docker_container_name == "database" %} echo "Starting existing ${DOCKERNAME} container" + docker start ${DOCKERNAME} {%- else %} echo "Starting existing ${DOCKERNAME} container with HWSKU $HWSKU" + /usr/local/bin/container start ${DOCKERNAME} {%- endif %} - preStartAction - docker start ${DOCKERNAME} postStartAction exit $? fi @@ -343,6 +344,7 @@ start() { {%- endif %} docker create {{docker_image_run_opt}} \ --net=$NET \ + -e RUNTIME_OWNER=local \ --uts=host \{# W/A: this should be set per-docker, for those dockers which really need host's UTS namespace #} {%- if install_debug_image == "y" %} -v /src:/src:ro -v /debug:/debug:rw \ @@ -400,21 +402,31 @@ start() { } preStartAction + {%- if docker_container_name == "database" %} docker start $DOCKERNAME + {%- else %} + /usr/local/bin/container start ${DOCKERNAME} + {%- endif %} postStartAction } wait() { + {%- if docker_container_name == "database" %} docker wait $DOCKERNAME + {%- else %} + /usr/local/bin/container wait $DOCKERNAME + {%- endif %} } stop() { + {%- if docker_container_name == "database" %} docker stop $DOCKERNAME -{%- if docker_container_name == "database" %} if [ "$DEV" ]; then ip netns delete "$NET_NS" fi -{%- endif %} + {%- else %} + /usr/local/bin/container stop $DOCKERNAME + {%- endif %} } DOCKERNAME={{docker_container_name}} diff --git a/files/build_templates/init_cfg.json.j2 b/files/build_templates/init_cfg.json.j2 index 82f300e007..cb68dd82f3 100644 --- a/files/build_templates/init_cfg.json.j2 +++ b/files/build_templates/init_cfg.json.j2 @@ -44,6 +44,10 @@ "has_global_scope": {% if feature + '.service' in installer_services.split(' ') %}true{% else %}false{% endif %}, "has_per_asic_scope": {% if feature + '@.service' in installer_services.split(' ') %}true{% else %}false{% endif %}, "auto_restart": "{{autorestart}}", +{%- if include_kubernetes == "y" %} +{%- if feature in ["dhcp_relay", "lldp", "pmon", "radv", "snmp", "telemetry"] %} + "set_owner": "kube", {% else %} + "set_owner": "local", {% endif %} {% endif %} "high_mem_alert": "disabled" }{% if not loop.last %},{% endif -%} {% endfor %} diff --git a/files/build_templates/kube_cni.10-flannel.conflist b/files/build_templates/kube_cni.10-flannel.conflist new file mode 100644 index 0000000000..17b50fda12 --- /dev/null +++ b/files/build_templates/kube_cni.10-flannel.conflist @@ -0,0 +1,21 @@ +{ + "_comment": "This is a dummy file to simulate flannel so that node status will show as Ready", + "name": "cbr0", + "cniVersion": "0.3.1", + "plugins": [ + { + "type": "flannel", + "delegate": { + "hairpinMode": true, + "isDefaultGateway": true + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + } + ] +} + diff --git a/files/build_templates/sonic_debian_extension.j2 b/files/build_templates/sonic_debian_extension.j2 index c346b8455f..1aa3e42fe7 100644 --- a/files/build_templates/sonic_debian_extension.j2 +++ b/files/build_templates/sonic_debian_extension.j2 @@ -37,6 +37,7 @@ FILESYSTEM_ROOT_USR="$FILESYSTEM_ROOT/usr" FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM="$FILESYSTEM_ROOT/usr/lib/systemd/system" FILESYSTEM_ROOT_USR_SHARE="$FILESYSTEM_ROOT_USR/share" FILESYSTEM_ROOT_USR_SHARE_SONIC="$FILESYSTEM_ROOT_USR_SHARE/sonic" +FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS="$FILESYSTEM_ROOT_USR_SHARE_SONIC/scripts" FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES="$FILESYSTEM_ROOT_USR_SHARE_SONIC/templates" FILESYSTEM_ROOT_ETC="$FILESYSTEM_ROOT/etc" FILESYSTEM_ROOT_ETC_SONIC="$FILESYSTEM_ROOT_ETC/sonic" @@ -395,10 +396,36 @@ sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install watchd sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip2 install futures==3.3.0 {% if include_kubernetes == "y" %} -# Copy kubelet service files -# Keep it disabled until join, else it continuously restart and as well spew too many -# non-required log lines wasting syslog resources. +# Install remote Container mgmt package +# Required even if include_kubernetes != y, as it contains the +# the container wrapper for docker start/stop/wait commands. +# +SONIC_CTRMGMT_WHEEL_NAME=$(basename {{sonic_ctrmgmt_py3_wheel_path}}) +sudo cp {{sonic_ctrmgmt_py3_wheel_path}} $FILESYSTEM_ROOT/$SONIC_CTRMGMT_WHEEL_NAME +sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install $SONIC_CTRMGMT_WHEEL_NAME +sudo rm -rf $FILESYSTEM_ROOT/$SONIC_CTRMGMT_WHEEL_NAME + +# Copy remote container mangement files +# File called from each container upon start/stop to record the state +sudo mkdir -p ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS} +sudo cp ${files_path}/container_startup.py ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/ +sudo chmod a+x ${FILESYSTEM_ROOT_USR_SHARE_SONIC_SCRIPTS}/container_startup.py + +# Config file used by container mgmt scripts/service +sudo cp ${files_path}/remote_ctr.config.json ${FILESYSTEM_ROOT_ETC_SONIC}/ + +# Remote container management service files +sudo cp ${files_path}/ctrmgrd.service ${FILESYSTEM_ROOT_USR_LIB_SYSTEMD_SYSTEM}/ +sudo LANG=C chroot $FILESYSTEM_ROOT systemctl enable ctrmgrd.service + +# kubelet service is controlled by ctrmgrd daemon. sudo LANG=C chroot $FILESYSTEM_ROOT systemctl disable kubelet.service +{% else %} +# container script for docker commands, which is required as +# all docker commands are replaced with container commands. +# So just copy that file only. +# +sudo cp ${files_path}/container $FILESYSTEM_ROOT/usr/local/bin/ {% endif %} # Copy the buffer configuration template @@ -585,6 +612,22 @@ else fi sudo umount /proc || true sudo rm $FILESYSTEM_ROOT/etc/init.d/docker + +sudo bash -c "echo { > $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json" +{% for entry in feature_vs_image_names.split(' ') -%} +{% if entry|length %} +{% set lst = entry.split(':') %} +{% set lst1 = lst[1].split('.') %} + sudo bash -c "echo -n -e \"\x22{{ lst[0] }}\x22 : \x22{{ lst1[0] }}\x22\" >> $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json" +{% if not loop.last %} + sudo bash -c "echo \",\" >> $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json" +{% else %} + sudo bash -c "echo \"\" >> $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json" +{% endif %} +{% endif %} +{% endfor %} +sudo bash -c "echo } >> $FILESYSTEM_ROOT_USR_SHARE_SONIC_TEMPLATES/ctr_image_names.json" + {% for script in installer_start_scripts.split(' ') -%} sudo cp {{script}} $FILESYSTEM_ROOT/usr/bin/ {% endfor %} diff --git a/rules/docker-config-engine-buster.mk b/rules/docker-config-engine-buster.mk index 66ec301d1f..a3d037b9ca 100644 --- a/rules/docker-config-engine-buster.mk +++ b/rules/docker-config-engine-buster.mk @@ -12,6 +12,7 @@ $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY2) $(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3) $(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER) $(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $(SWSS_VARS_TEMPLATE) +$(DOCKER_CONFIG_ENGINE_BUSTER)_FILES += $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT) $(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS = $($(DOCKER_BASE_BUSTER)_DBG_DEPENDS) $(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_PACKAGES = $($(DOCKER_BASE_BUSTER)_DBG_IMAGE_PACKAGES) diff --git a/rules/docker-dhcp-relay.mk b/rules/docker-dhcp-relay.mk index 860928bf81..6be9bb79fd 100644 --- a/rules/docker-dhcp-relay.mk +++ b/rules/docker-dhcp-relay.mk @@ -7,6 +7,7 @@ DOCKER_DHCP_RELAY_DBG = $(DOCKER_DHCP_RELAY_STEM)-$(DBG_IMAGE_MARK).gz $(DOCKER_DHCP_RELAY)_PATH = $(DOCKERS_PATH)/$(DOCKER_DHCP_RELAY_STEM) $(DOCKER_DHCP_RELAY)_DEPENDS += $(ISC_DHCP_RELAY) $(SONIC_DHCPMON) + $(DOCKER_DHCP_RELAY)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS) $(DOCKER_DHCP_RELAY)_DBG_DEPENDS += $(ISC_DHCP_RELAY_DBG) @@ -23,4 +24,5 @@ SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_DHCP_RELAY_DBG) $(DOCKER_DHCP_RELAY)_CONTAINER_NAME = dhcp_relay $(DOCKER_DHCP_RELAY)_RUN_OPT += --privileged -t $(DOCKER_DHCP_RELAY)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_DHCP_RELAY)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro $(DOCKER_DHCP_RELAY)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT) diff --git a/rules/docker-lldp.mk b/rules/docker-lldp.mk index b84ff6d7e4..5cd4fd02a0 100644 --- a/rules/docker-lldp.mk +++ b/rules/docker-lldp.mk @@ -25,6 +25,7 @@ SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_LLDP_DBG) $(DOCKER_LLDP)_CONTAINER_NAME = lldp $(DOCKER_LLDP)_RUN_OPT += --privileged -t $(DOCKER_LLDP)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_LLDP)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro $(DOCKER_LLDP)_BASE_IMAGE_FILES += lldpctl:/usr/bin/lldpctl $(DOCKER_LLDP)_BASE_IMAGE_FILES += lldpcli:/usr/bin/lldpcli diff --git a/rules/docker-platform-monitor.mk b/rules/docker-platform-monitor.mk index e9e414cadd..1bd6a68660 100644 --- a/rules/docker-platform-monitor.mk +++ b/rules/docker-platform-monitor.mk @@ -45,6 +45,7 @@ SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_PLATFORM_MONITOR_DBG) $(DOCKER_PLATFORM_MONITOR)_CONTAINER_NAME = pmon $(DOCKER_PLATFORM_MONITOR)_RUN_OPT += --privileged -t $(DOCKER_PLATFORM_MONITOR)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_PLATFORM_MONITOR)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro $(DOCKER_PLATFORM_MONITOR)_RUN_OPT += -v /var/run/platform_cache:/var/run/platform_cache:ro $(DOCKER_PLATFORM_MONITOR)_RUN_OPT += -v /usr/share/sonic/device/pddf:/usr/share/sonic/device/pddf:ro diff --git a/rules/docker-router-advertiser.mk b/rules/docker-router-advertiser.mk index 3b9f7ae8ba..38bd959056 100644 --- a/rules/docker-router-advertiser.mk +++ b/rules/docker-router-advertiser.mk @@ -22,4 +22,5 @@ SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_ROUTER_ADVERTISER_DBG) $(DOCKER_ROUTER_ADVERTISER)_CONTAINER_NAME = radv $(DOCKER_ROUTER_ADVERTISER)_RUN_OPT += --privileged -t $(DOCKER_ROUTER_ADVERTISER)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_ROUTER_ADVERTISER)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro $(DOCKER_ROUTER_ADVERTISER)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT) diff --git a/rules/docker-snmp.mk b/rules/docker-snmp.mk index 3493a2dfa3..cbc69dc92b 100644 --- a/rules/docker-snmp.mk +++ b/rules/docker-snmp.mk @@ -26,5 +26,6 @@ SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_SNMP_DBG) $(DOCKER_SNMP)_CONTAINER_NAME = snmp $(DOCKER_SNMP)_RUN_OPT += --privileged -t $(DOCKER_SNMP)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_SNMP)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro $(DOCKER_SNMP)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT) $(DOCKER_SNMP)_BASE_IMAGE_FILES += monit_snmp:/etc/monit/conf.d diff --git a/rules/docker-telemetry.mk b/rules/docker-telemetry.mk index 15f90c74c8..5152a38696 100644 --- a/rules/docker-telemetry.mk +++ b/rules/docker-telemetry.mk @@ -26,6 +26,7 @@ endif $(DOCKER_TELEMETRY)_CONTAINER_NAME = telemetry $(DOCKER_TELEMETRY)_RUN_OPT += --privileged -t $(DOCKER_TELEMETRY)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_TELEMETRY)_RUN_OPT += -v /usr/share/sonic/scripts:/usr/share/sonic/scripts:ro $(DOCKER_TELEMETRY)_RUN_OPT += -v /var/run/dbus:/var/run/dbus:rw $(DOCKER_TELEMETRY)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT) diff --git a/rules/sonic-ctrmgrd.dep b/rules/sonic-ctrmgrd.dep new file mode 100644 index 0000000000..b656421276 --- /dev/null +++ b/rules/sonic-ctrmgrd.dep @@ -0,0 +1,8 @@ +SPATH := $($(SONIC_CTRMGRD)_SRC_PATH) +DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-ctrmgrd.mk rules/sonic-ctrmgrd.dep +DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST) +DEP_FILES += $(shell git ls-files $(SPATH)) + +$(SONIC_CTRMGRD)_CACHE_MODE := GIT_CONTENT_SHA +$(SONIC_CTRMGRD)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST) +$(SONIC_CTRMGRD)_DEP_FILES := $(DEP_FILES) diff --git a/rules/sonic-ctrmgrd.mk b/rules/sonic-ctrmgrd.mk new file mode 100644 index 0000000000..659a2cf4ac --- /dev/null +++ b/rules/sonic-ctrmgrd.mk @@ -0,0 +1,31 @@ +# sonic-ctrmgrd package + +SONIC_CTRMGRD = sonic_ctrmgrd-1.0.0-py3-none-any.whl +$(SONIC_CTRMGRD)_SRC_PATH = $(SRC_PATH)/sonic-ctrmgrd +$(SONIC_CTRMGRD)_FILES_PATH = $($(SONIC_CTRMGRD)_SRC_PATH)/ctrmgr + +$(SONIC_CTRMGRD)_PYTHON_VERSION = 3 +$(SONIC_CTRMGRD)_DEBS_DEPENDS += $(PYTHON3_SWSSCOMMON) +$(SONIC_CTRMGRD)_DEPENDS += $(SONIC_PY_COMMON_PY3) + +$(SONIC_CTRMGRD)_CONTAINER_SCRIPT = container +$($(SONIC_CTRMGRD)_CONTAINER_SCRIPT)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH) + +$(SONIC_CTRMGRD)_STARTUP_SCRIPT = container_startup.py +$($(SONIC_CTRMGRD)_STARTUP_SCRIPT)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH) + +$(SONIC_CTRMGRD)_CFG_JSON = remote_ctr.config.json +$($(SONIC_CTRMGRD)_CFG_JSON)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH) + +$(SONIC_CTRMGRD)_SERVICE = ctrmgrd.service +$($(SONIC_CTRMGRD)_SERVICE)_PATH = $($(SONIC_CTRMGRD)_FILES_PATH) + +SONIC_PYTHON_WHEELS += $(SONIC_CTRMGRD) + +$(SONIC_CTRMGRD)_FILES = $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT) +$(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT) +$(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_CFG_JSON) +$(SONIC_CTRMGRD)_FILES += $($(SONIC_CTRMGRD)_SERVICE) + +SONIC_COPY_FILES += $($(SONIC_CTRMGRD)_FILES) + diff --git a/slave.mk b/slave.mk index 6bd66ff589..a7b3ae033d 100644 --- a/slave.mk +++ b/slave.mk @@ -750,6 +750,7 @@ $(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform --build-arg docker_container_name=$($*.gz_CONTAINER_NAME) \ --build-arg frr_user_uid=$(FRR_USER_UID) \ --build-arg frr_user_gid=$(FRR_USER_GID) \ + --build-arg image_version=$(SONIC_IMAGE_VERSION) \ --label Tag=$(SONIC_IMAGE_VERSION) \ -t $* $($*.gz_PATH) $(LOG) scripts/collect_docker_version_files.sh $* $(TARGET_PATH) @@ -865,6 +866,8 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PLATFORM_API_PY2)) \ $(if $(findstring y,$(PDDF_SUPPORT)),$(addprefix $(PYTHON_WHEELS_PATH)/,$(PDDF_PLATFORM_API_BASE_PY2))) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3)) \ + $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CTRMGRD)) \ + $(addprefix $(FILES_PATH)/,$($(SONIC_CTRMGRD)_FILES)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY2)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \ @@ -911,6 +914,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ export redis_dump_load_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(REDIS_DUMP_LOAD_PY3))" export install_debug_image="$(INSTALL_DEBUG_TOOLS)" export sonic_yang_models_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MODELS_PY3))" + export sonic_ctrmgmt_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CTRMGRD))" export sonic_yang_mgmt_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY2))" export sonic_yang_mgmt_py3_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3))" export multi_instance="false" @@ -959,6 +963,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ # Exported variables are used by sonic_debian_extension.sh export installer_start_scripts="$(foreach docker, $($*_DOCKERS),$(addsuffix .sh, $($(docker:-dbg.gz=.gz)_CONTAINER_NAME)))" + export feature_vs_image_names="$(foreach docker, $($*_DOCKERS), $(addsuffix :, $($(docker:-dbg.gz=.gz)_CONTAINER_NAME):$(docker:-dbg.gz=.gz)))" # Marks template services with an "@" according to systemd convention # If the $($docker)_TEMPLATE) variable is set, the service will be treated as a template diff --git a/src/sonic-ctrmgrd/.gitignore b/src/sonic-ctrmgrd/.gitignore new file mode 100644 index 0000000000..bdebd5e838 --- /dev/null +++ b/src/sonic-ctrmgrd/.gitignore @@ -0,0 +1,12 @@ +.eggs/ +build/ +dist/ +*.egg-info/ +ctrmgr/*.pyc +tests/*.pyc +tests/__pycache__/ +.idea +.coverage +ctrmgr/__pycache__/ +venv +tests/.coverage* diff --git a/src/sonic-ctrmgrd/README.rst b/src/sonic-ctrmgrd/README.rst new file mode 100644 index 0000000000..30207f44b3 --- /dev/null +++ b/src/sonic-ctrmgrd/README.rst @@ -0,0 +1,3 @@ +" +This Package contains modules required for remote container management. +" diff --git a/src/sonic-ctrmgrd/ctrmgr/__init__.py b/src/sonic-ctrmgrd/ctrmgr/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/sonic-ctrmgrd/ctrmgr/container b/src/sonic-ctrmgrd/ctrmgr/container new file mode 100755 index 0000000000..ca0963946f --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/container @@ -0,0 +1,409 @@ +#!/usr/bin/env python3 + +import argparse +import os +import inspect +import json +import syslog +import time +import datetime + +import docker +from swsscommon import swsscommon + +CTR_STATE_SCR_PATH = '/usr/share/sonic/scripts/container_startup.py' + +state_db = None + +# DB field names +FEATURE_TABLE = "FEATURE" +SET_OWNER = "set_owner" +NO_FALLBACK = "no_fallback_to_local" + +CURRENT_OWNER = "current_owner" +UPD_TIMESTAMP = "update_time" +CONTAINER_ID = "container_id" +REMOTE_STATE = "remote_state" +VERSION = "container_version" +SYSTEM_STATE = "system_state" + +KUBE_LABEL_TABLE = "KUBE_LABELS" +KUBE_LABEL_SET_KEY = "SET" + +# Get seconds to wait for remote docker to start. +# If not, revert to local +# +SONIC_CTR_CONFIG = "/etc/sonic/remote_ctr.config.json" +SONIC_CTR_CONFIG_PEND_SECS = "revert_to_local_on_wait_seconds" +DEFAULT_PEND_SECS = ( 5 * 60 ) +WAIT_POLL_SECS = 2 + +remote_ctr_enabled = False + +def debug_msg(m): + msg = "{}: {}".format(inspect.stack()[1][3], m) + # print(msg) + syslog.syslog(syslog.LOG_DEBUG, msg) + + +def init(): + """ Get DB connections """ + global state_db, cfg_db, remote_ctr_enabled + + cfg_db = swsscommon.DBConnector("CONFIG_DB", 0) + state_db = swsscommon.DBConnector("STATE_DB", 0) + + remote_ctr_enabled = os.path.exists(CTR_STATE_SCR_PATH) + + +def get_config_data(fld, dflt): + """ Read entry from kube config file """ + if os.path.exists(SONIC_CTR_CONFIG): + with open(SONIC_CTR_CONFIG, "r") as s: + d = json.load(s) + if fld in d: + return d[fld] + return dflt + + +def read_data(is_config, feature, fields): + """ Read data from DB for desired fields using given defaults""" + ret = [] + + db = cfg_db if is_config else state_db + + tbl = swsscommon.Table(db, FEATURE_TABLE) + + data = dict(tbl.get(feature)[1]) + for (field, default) in fields: + val = data.get(field, default) + ret += [val] + + debug_msg("config:{} feature:{} fields:{} val:{}".format( + is_config, feature, str(fields), str(ret))) + + return tuple(ret) + + +def read_config(feature): + """ Read requried feature config """ + set_owner, no_fallback = read_data(True, feature, + [(SET_OWNER, "local"), (NO_FALLBACK, False)]) + + return (set_owner, not no_fallback) + + +def read_state(feature): + """ Read requried feature state """ + + return read_data(False, feature, + [(CURRENT_OWNER, "none"), (REMOTE_STATE, "none"), (CONTAINER_ID, "")]) + + +def docker_action(action, feature): + """ Execute docker action """ + try: + client = docker.from_env() + container = client.containers.get(feature) + getattr(container, action)() + syslog.syslog(syslog.LOG_INFO, "docker cmd: {} for {}".format(action, feature)) + return 0 + + except (docker.errors.NotFound, docker.errors.APIError) as err: + syslog.syslog(syslog.LOG_ERR, "docker cmd: {} for {} failed with {}". + format(action, feature, str(err))) + return -1 + + +def set_label(feature, create): + """ Set/drop label as required + Update is done in state-db. + ctrmgrd sets it with kube API server as required + """ + if remote_ctr_enabled: + tbl = swsscommon.Table(state_db, KUBE_LABEL_TABLE) + fld = "{}_enabled".format(feature) + + # redundant set (data already exist) can still raise subscriber + # notification. So check & set. + # Redundant delete (data doesn't exist) does not raise any + # subscriber notification. So no need to pre-check for delete. + # + tbl.set(KUBE_LABEL_SET_KEY, [(fld, "true" if create else "false")]) + + +def update_data(feature, data): + if remote_ctr_enabled: + debug_msg("feature:{} data:{}".format(feature, str(data))) + tbl = swsscommon.Table(state_db, FEATURE_TABLE) + tbl.set(feature, list(data.items())) + + +def container_id(feature): + """ + Return the container ID for the feature. + + if current_owner is local, use feature name as the start/stop + of local image is synchronous. + Else get it from FEATURE table in STATE-DB + + :param feature: Name of the feature to start. + + """ + init() + + tbl = swsscommon.Table(state_db, "FEATURE") + data = dict(tbl.get(feature)[1]) + + if (data.get(CURRENT_OWNER, "").lower() == "local"): + return feature + else: + return data.get(CONTAINER_ID, feature) + + +def container_start(feature): + """ + Starts a container for given feature. + + Starts from local image and/or trigger kubernetes to deploy the image + for this feature. Marks the feature state up in STATE-DB FEATURE table. + + If feature's set_owner is local, invoke docker start. + If feature's set_owner is kube, it creates a node label that + would trigger kubernetes to start the container. With kube as + owner, if fallback is enabled and remote_state==none, it starts + the local image using docker, which will run until kube + deployment occurs. + + :param feature: Name of the feature to start. + + """ + START_LOCAL = 1 + START_KUBE = 2 + + ret = 0 + debug_msg("BEGIN") + + init() + + set_owner, fallback = read_config(feature) + _, remote_state, _ = read_state(feature) + + debug_msg("{}: set_owner:{} fallback:{} remote_state:{}".format( + feature, set_owner, fallback, remote_state)) + + data = { + SYSTEM_STATE: "up", + UPD_TIMESTAMP: str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + } + + + start_val = 0 + if set_owner == "local": + start_val = START_LOCAL + else: + start_val = START_KUBE + if fallback and (remote_state == "none"): + start_val |= START_LOCAL + + if start_val == START_LOCAL: + # Implies *only* local. + # Ensure label is not there, to block kube deployment. + set_label(feature, False) + data[REMOTE_STATE] = "none" + + if (start_val & START_LOCAL): + data[CURRENT_OWNER] = "local" + data[CONTAINER_ID] = feature + + update_data(feature, data) + + if (start_val & START_LOCAL): + ret = docker_action("start", feature) + + if (start_val & START_KUBE): + set_label(feature, True) + debug_msg("END") + return ret + + +def container_stop(feature): + """ + Stops the running container for this feature. + + Instruct/ensure kube terminates, by removing label, unless + an kube upgrade is happening. + + Gets the container ID for this feature and call docker stop. + + Marks the feature state down in STATE-DB FEATURE table. + + :param feature: Name of the feature to stop. + + """ + debug_msg("BEGIN") + + init() + + set_owner, _ = read_config(feature) + current_owner, remote_state, _ = read_state(feature) + docker_id = container_id(feature) + remove_label = (remote_state != "pending") or (set_owner == "local") + + debug_msg("{}: set_owner:{} current_owner:{} remote_state:{} docker_id:{}".format( + feature, set_owner, current_owner, remote_state, docker_id)) + + if remove_label: + set_label(feature, False) + + if docker_id: + docker_action("stop", docker_id) + else: + syslog.syslog( + syslog.LOG_ERR if current_owner != "none" else syslog.LOG_INFO, + "docker stop skipped as no docker-id for {}".format(feature)) + + # Container could get killed or crashed. In either case + # it does not have opportunity to mark itself down. + # Even during normal termination, with SIGTERM received + # container process may not have enough window of time to + # mark itself down and has the potential to get aborted. + # + # systemctl ensures that it handles only one instance for + # a feature at anytime and however the feature container + # exits, upon stop/kill/crash, systemctl-stop process + # is assured to get called. So mark the feature down here. + # + data = { + CURRENT_OWNER: "none", + UPD_TIMESTAMP: str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")), + CONTAINER_ID: "", + VERSION: "", + SYSTEM_STATE: "down" + } + if remote_state == "running": + data[REMOTE_STATE] = "stopped" + + update_data(feature, data) + + debug_msg("END") + + +def container_kill(feature): + """ + Kills the running container for this feature. + + Instruct/ensure kube terminates, by removing label. + + :param feature: Name of the feature to kill. + + """ + debug_msg("BEGIN") + + init() + + set_owner, _ = read_config(feature) + current_owner, remote_state, _ = read_state(feature) + docker_id = container_id(feature) + remove_label = (set_owner != "local") or (current_owner != "local") + + debug_msg("{}: set_owner:{} current_owner:{} remote_state:{} docker_id:{}".format( + feature, set_owner, current_owner, remote_state, docker_id)) + + if remove_label: + set_label(feature, False) + + if docker_id: + docker_action("kill", docker_id) + + else: + syslog.syslog( + syslog.LOG_ERR if current_owner != "none" else syslog.LOG_INFO, + "docker stop skipped as no docker-id for {}".format(feature)) + + + debug_msg("END") + + +def container_wait(feature): + """ + Waits on the running container for this feature. + + Get the container-id and call docker wait. + + If docker-id can't be obtained for a configurable fail-duration + the wait clears the feature's remote-state in STATE-DB FEATURE + table and exit. + + :param feature: Name of the feature to wait. + + """ + debug_msg("BEGIN") + + init() + + set_owner, fallback = read_config(feature) + current_owner, remote_state, _ = read_state(feature) + docker_id = container_id(feature) + pend_wait_secs = 0 + + if not docker_id and fallback: + pend_wait_secs = get_config_data( + SONIC_CTR_CONFIG_PEND_SECS, DEFAULT_PEND_SECS) + + debug_msg("{}: set_owner:{} ct_owner:{} state:{} id:{} pend={}".format( + feature, set_owner, current_owner, remote_state, docker_id, + pend_wait_secs)) + + while not docker_id: + if fallback: + pend_wait_secs = pend_wait_secs - WAIT_POLL_SECS + if pend_wait_secs < 0: + break + + time.sleep(WAIT_POLL_SECS) + + current_owner, remote_state, docker_id = read_state(feature) + + debug_msg("wait_loop: {} = {} {} {}".format(feature, current_owner, remote_state, docker_id)) + + if (remote_state == "pending"): + update_data(feature, {REMOTE_STATE: "ready"}) + + if not docker_id: + # Clear remote state and exit. + # systemd would restart and fallback to local + update_data(feature, { REMOTE_STATE: "none" }) + debug_msg("{}: Exiting to fallback as remote is *not* starting". + format(feature)) + else: + debug_msg("END -- transitioning to docker wait") + docker_action("wait", docker_id) + + +def main(): + parser=argparse.ArgumentParser(description="container commands for start/stop/wait/kill/id") + parser.add_argument("action", choices=["start", "stop", "wait", "kill", "id"]) + parser.add_argument("name") + + args = parser.parse_args() + + if args.action == "start": + container_start(args.name) + + elif args.action == "stop": + container_stop(args.name) + + elif args.action == "kill": + container_kill(args.name) + + elif args.action == "wait": + container_wait(args.name) + + elif args.action == "id": + id = container_id(args.name) + print(id) + + +if __name__ == "__main__": + main() diff --git a/src/sonic-ctrmgrd/ctrmgr/container_startup.py b/src/sonic-ctrmgrd/ctrmgr/container_startup.py new file mode 100755 index 0000000000..c56160aa48 --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/container_startup.py @@ -0,0 +1,291 @@ +#!/usr/bin/env python3 + +import argparse +import datetime +import inspect +import json +import subprocess +import syslog +import time + +from swsscommon import swsscommon + +# DB field names +SET_OWNER = "set_owner" + +CURRENT_OWNER = "current_owner" +UPD_TIMESTAMP = "update_time" +DOCKER_ID = "container_id" +REMOTE_STATE = "remote_state" +VERSION = "container_version" +SYSTEM_STATE = "system_state" + +KUBE_LABEL_TABLE = "KUBE_LABELS" +KUBE_LABEL_SET_KEY = "SET" + +UNIT_TESTING = 0 + + +def debug_msg(m): + msg = "{}: {}".format(inspect.stack()[1][3], m) + print(msg) + syslog.syslog(syslog.LOG_DEBUG, msg) + + +def _get_version_key(feature, version): + # Coin label for version control + return "{}_{}_enabled".format(feature, version) + + +def read_data(feature): + state_data = { + CURRENT_OWNER: "none", + UPD_TIMESTAMP: "", + DOCKER_ID: "", + REMOTE_STATE: "none", + VERSION: "0.0.0", + SYSTEM_STATE: "" + } + + set_owner = "local" + + # read owner from config-db and current state data from state-db. + db = swsscommon.DBConnector("CONFIG_DB", 0) + tbl = swsscommon.Table(db, 'FEATURE') + data = dict(tbl.get(feature)[1]) + + if (SET_OWNER in data): + set_owner = data[SET_OWNER] + + state_db = swsscommon.DBConnector("STATE_DB", 0) + tbl = swsscommon.Table(state_db, 'FEATURE') + state_data.update(dict(tbl.get(feature)[1])) + + return (state_db, set_owner, state_data) + + +def read_fields(state_db, feature, fields): + # Read directly from STATE-DB, given fields + # for given feature. + # Fields is a list of tuples (, ) + # + tbl = swsscommon.Table(state_db, 'FEATURE') + ret = [] + + # tbl.get for non-existing feature would return + # [False, {} ] + # + data = dict(tbl.get(feature)[1]) + for (field, default) in fields: + val = data[field] if field in data else default + ret += [val] + + return tuple(ret) + + +def check_version_blocked(state_db, feature, version): + # Ensure this version is *not* blocked explicitly. + # + tbl = swsscommon.Table(state_db, KUBE_LABEL_TABLE) + labels = dict(tbl.get(KUBE_LABEL_SET_KEY)[1]) + key = _get_version_key(feature, version) + return (key in labels) and (labels[key].lower() == "false") + + +def drop_label(state_db, feature, version): + # Mark given feature version as dropped in labels. + # Update is done in state-db. + # ctrmgrd sets it with kube API server per reaschability + + tbl = swsscommon.Table(state_db, KUBE_LABEL_TABLE) + name = _get_version_key(feature, version) + tbl.set(KUBE_LABEL_SET_KEY, [ (name, "false")]) + + +def update_data(state_db, feature, data): + # Update STATE-DB entry for this feature with given data + # + debug_msg("{}: {}".format(feature, str(data))) + tbl = swsscommon.Table(state_db, "FEATURE") + tbl.set(feature, list(data.items())) + + +def get_docker_id(): + # Read the container-id + # Note: This script runs inside the context of container + # + cmd = 'cat /proc/self/cgroup | grep -e ":memory:" | rev | cut -f1 -d\'/\' | rev' + proc = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + output = proc.communicate()[0].decode("utf-8") + return output.strip()[:12] + + +def instance_higher(feature, ct_version, version): + # Compares given version against current version in STATE-DB. + # Return True if this version is higher than current. + # + ret = False + if ct_version: + ct = ct_version.split('.') + nxt = version.split('.') + for cs, ns in zip(ct, nxt): + c = int(cs) + n = int(ns) + if n < c: + break + elif n > c: + ret = True + break + else: + # Empty version implies no one is running. + ret = True + + debug_msg("compare version: new:{} current:{} res={}".format( + version, ct_version, ret)) + return ret + + +def is_active(feature, system_state): + # Check current system state of the feature + if system_state == "up": + return True + else: + syslog.syslog(syslog.LOG_ERR, "Found inactive for {}".format(feature)) + return False + + +def update_state(state_db, feature, owner=None, version=None): + """ + sets owner, version & container-id for this feature in state-db. + + If owner is local, update label to block remote deploying same version or + if kube, sets state to "running". + + """ + data = { + CURRENT_OWNER: owner, + DOCKER_ID: get_docker_id() if owner != "local" else feature, + UPD_TIMESTAMP: str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")), + "hello": "world", + VERSION: version + } + + if (owner == "local"): + # Disable deployment of this version as available locally + drop_label(state_db, feature, version) + else: + data[REMOTE_STATE] = "running" + + debug_msg("{} up data:{}".format(feature, str(data))) + update_data(state_db, feature, data) + + +def do_freeze(feat, m): + # Exiting will kick off the container to run. + # So sleep forever with periodic logs. + # + while True: + syslog.syslog(syslog.LOG_ERR, "Blocking .... feat:{} docker_id:{} msg:{}".format( + feat, get_docker_id(), m)) + if UNIT_TESTING: + break + time.sleep(60) + + +def container_up(feature, owner, version): + """ + This is called by container upon post start. + + The container will run its application, only upon this call + complete. + + This call does the basic check for if this starting-container can be allowed + to run based on current state, and owner & version of this starting + container. + + If allowed to proceed, this info is recorded in state-db and return + to enable container start the main application. Else it proceeds to + sleep forever, blocking the container from starting the main application. + + """ + debug_msg("BEGIN") + (state_db, set_owner, state_data) = read_data(feature) + + debug_msg("args: feature={}, owner={}, version={} DB: set_owner={} state_data={}".format( + feature, owner, version, set_owner, json.dumps(state_data, indent=4))) + + if owner == "local": + update_state(state_db, feature, owner, version) + else: + if (set_owner == "local"): + do_freeze(feature, "bail out as set_owner is local") + return + + if not is_active(feature, state_data[SYSTEM_STATE]): + do_freeze(feature, "bail out as system state not active") + return + + if check_version_blocked(state_db, feature, version): + do_freeze(feature, "This version is marked disabled. Exiting ...") + return + + if not instance_higher(feature, state_data[VERSION], version): + # TODO: May Remove label __enabled + # Else kubelet will continue to re-deploy every 5 mins, until + # master removes the lable to un-deploy. + # + do_freeze(feature, "bail out as current deploy version {} is not higher". + format(version)) + return + + update_data(state_db, feature, { VERSION: version }) + + mode = state_data[REMOTE_STATE] + if mode in ("none", "running", "stopped"): + update_data(state_db, feature, { REMOTE_STATE: "pending" }) + mode = "pending" + else: + debug_msg("{}: Skip remote_state({}) update".format(feature, mode)) + + + i = 0 + while (mode != "ready"): + if (i % 10) == 0: + debug_msg("{}: remote_state={}. Waiting to go ready".format(feature, mode)) + i += 1 + + time.sleep(2) + mode, db_version = read_fields(state_db, + feature, [(REMOTE_STATE, "none"), (VERSION, "")]) + if version != db_version: + # looks like another instance has overwritten. Exit for now. + # If this happens to be higher version, next deploy by kube will fix + # This is a very rare window of opportunity, for this version to be higher. + # + do_freeze(feature, + "bail out as current deploy version={} is different than {}. re-deploy higher one". + format(version, db_version)) + return + if UNIT_TESTING: + return + + + update_state(state_db, feature, owner, version) + + debug_msg("END") + + +def main(): + parser = argparse.ArgumentParser(description="container_startup kube/local []") + + parser.add_argument("-f", "--feature", required=True) + parser.add_argument("-o", "--owner", choices=["local", "kube"], required=True) + parser.add_argument("-v", "--version", required=True) + + args = parser.parse_args() + container_up(args.feature, args.owner, args.version) + + + +if __name__ == "__main__": + main() diff --git a/src/sonic-ctrmgrd/ctrmgr/ctrmgr_tools.py b/src/sonic-ctrmgrd/ctrmgr/ctrmgr_tools.py new file mode 100755 index 0000000000..d85f76c4f9 --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/ctrmgr_tools.py @@ -0,0 +1,147 @@ +#! /usr/bin/env python3 + +import argparse +import json +import os +import syslog + +import docker +from swsscommon import swsscommon + +CTR_NAMES_FILE = "/usr/share/sonic/templates/ctr_image_names.json" + +LATEST_TAG = "latest" + +# DB fields +CT_OWNER = 'current_owner' +CT_ID = 'container_id' +SYSTEM_STATE = 'system_state' + +def _get_local_image_name(feature): + d = {} + if os.path.exists(CTR_NAMES_FILE): + with open(CTR_NAMES_FILE, "r") as s: + d = json.load(s) + return d[feature] if feature in d else None + + +def _remove_container(client, feature): + try: + # Remove stopped container instance, if any for this feature + container = client.containers.get(feature) + container.remove() + syslog.syslog(syslog.LOG_INFO, "Removed container for {}". + format(feature)) + return 0 + except Exception as err: + syslog.syslog(syslog.LOG_INFO, "No container to remove for {} err={}". + format(feature, str(err))) + return -1 + + +def _tag_container_image(feature, container_id, image_name, image_tag): + client = docker.from_env() + + try: + container = client.containers.get(container_id) + + # Tag this image for given name & tag + container.image.tag(image_name, image_tag, force=True) + + syslog.syslog(syslog.LOG_INFO, + "Tagged image for {} with container-id={} to {}:{}". + format(feature, container_id, image_name, image_tag)) + ret = _remove_container(client, feature) + return ret + + except Exception as err: + syslog.syslog(syslog.LOG_ERR, "Image tag: container:{} {}:{} failed with {}". + format(container_id, image_name, image_tag, str(err))) + return -1 + + +def tag_feature(feature=None, image_name=None, image_tag=None): + ret = 0 + state_db = swsscommon.DBConnector("STATE_DB", 0) + tbl = swsscommon.Table(state_db, 'FEATURE') + keys = tbl.getKeys() + for k in keys: + if (not feature) or (k == feature): + d = dict(tbl.get(k)[1]) + owner = d.get(CT_OWNER, "") + id = d.get(CT_ID, "") + if not image_name: + image_name = _get_local_image_name(k) + if not image_tag: + image_tag = LATEST_TAG + if id and (owner == "kube") and image_name: + ret = _tag_container_image(k, id, image_name, image_tag) + else: + syslog.syslog(syslog.LOG_ERR, + "Skip to tag feature={} image={} tag={}".format( + k, str(image_name), str(image_tag))) + ret = -1 + + image_name=None + return ret + + +def func_tag_feature(args): + return tag_feature(args.feature, args.image_name, args.image_tag) + + +def func_tag_all(args): + return tag_feature() + + +def func_kill_all(args): + client = docker.from_env() + state_db = swsscommon.DBConnector("STATE_DB", 0) + tbl = swsscommon.Table(state_db, 'FEATURE') + + keys = tbl.getKeys() + for k in keys: + data = dict(tbl.get(k)[1]) + is_up = data.get(SYSTEM_STATE, "").lower() == "up" + id = data.get(CT_ID, "") if is_up else "" + if id: + try: + container = client.containers.get(id) + container.kill() + syslog.syslog(syslog.LOG_INFO, + "Killed container for {} with id={}".format(k, id)) + except Exception as err: + syslog.syslog(syslog.LOG_ERR, + "kill: feature={} id={} unable to get container". + format(k, id)) + return -1 + return 0 + + +def main(): + parser = argparse.ArgumentParser(description="Remote container mgmt tools") + subparsers = parser.add_subparsers(title="ctrmgr_tools") + + parser_tag = subparsers.add_parser('tag') + parser_tag.add_argument("-f", "--feature", required=True) + parser_tag.add_argument("-n", "--image-name") + parser_tag.add_argument("-t", "--image-tag") + parser_tag.set_defaults(func=func_tag_feature) + + parser_tag_all = subparsers.add_parser('tag-all') + parser_tag_all.set_defaults(func=func_tag_all) + + parser_kill_all = subparsers.add_parser('kill-all') + parser_kill_all.set_defaults(func=func_kill_all) + + args = parser.parse_args() + if len(args.__dict__) < 1: + parser.print_help() + return -1 + + ret = args.func(args) + return ret + + +if __name__ == "__main__": + main() diff --git a/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py new file mode 100755 index 0000000000..ba4f0057bd --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.py @@ -0,0 +1,598 @@ +#!/usr/bin/env python3 + +import datetime +import inspect +import json +import os +import sys +import syslog + +from collections import defaultdict + +from swsscommon import swsscommon +from sonic_py_common import device_info + +sys.path.append(os.path.dirname(os.path.realpath(__file__))) +import kube_commands + +UNIT_TESTING = 0 +UNIT_TESTING_ACTIVE = 0 + +# Kube config file +SONIC_CTR_CONFIG = "/etc/sonic/remote_ctr.config.json" + +CONFIG_DB_NAME = "CONFIG_DB" +STATE_DB_NAME = "STATE_DB" + +# DB SERVER +SERVER_TABLE = "KUBERNETES_MASTER" +SERVER_KEY = "SERVER" +CFG_SER_IP = "ip" +CFG_SER_PORT = "port" +CFG_SER_DISABLE = "disable" +CFG_SER_INSECURE = "insecure" + +ST_SER_IP = "ip" +ST_SER_PORT = "port" +ST_SER_CONNECTED = "connected" +ST_SER_UPDATE_TS = "update_time" + +# DB FEATURE +FEATURE_TABLE = "FEATURE" +CFG_FEAT_OWNER = "set_owner" +CFG_FEAT_NO_FALLBACK = "no_fallback_to_local" +CFG_FEAT_FAIL_DETECTION = "remote_fail_detection" + +ST_FEAT_OWNER = "current_owner" +ST_FEAT_UPDATE_TS = "update_time" +ST_FEAT_CTR_ID = "container_id" +ST_FEAT_CTR_VER = "container_version" +ST_FEAT_REMOTE_STATE = "remote_state" +ST_FEAT_SYS_STATE = "system_state" + +KUBE_LABEL_TABLE = "KUBE_LABELS" +KUBE_LABEL_SET_KEY = "SET" + +remote_connected = False + +dflt_cfg_ser = { + CFG_SER_IP: "", + CFG_SER_PORT: "6443", + CFG_SER_DISABLE: "false", + CFG_SER_INSECURE: "false" + } + +dflt_st_ser = { + ST_SER_IP: "", + ST_SER_PORT: "", + ST_SER_CONNECTED: "", + ST_SER_UPDATE_TS: "" + } + +dflt_cfg_feat= { + CFG_FEAT_OWNER: "local", + CFG_FEAT_NO_FALLBACK: "false", + CFG_FEAT_FAIL_DETECTION: "300" + } + +dflt_st_feat= { + ST_FEAT_OWNER: "none", + ST_FEAT_UPDATE_TS: "", + ST_FEAT_CTR_ID: "", + ST_FEAT_CTR_VER: "", + ST_FEAT_REMOTE_STATE: "none", + ST_FEAT_SYS_STATE: "" + } + +JOIN_LATENCY = "join_latency_on_boot_seconds" +JOIN_RETRY = "retry_join_interval_seconds" +LABEL_RETRY = "retry_labels_update_seconds" + +remote_ctr_config = { + JOIN_LATENCY: 10, + JOIN_RETRY: 10, + LABEL_RETRY: 2 + } + +def log_debug(m): + msg = "{}: {}".format(inspect.stack()[1][3], m) + print(msg) + syslog.syslog(syslog.LOG_DEBUG, msg) + + +def log_error(m): + syslog.syslog(syslog.LOG_ERR, msg) + + +def log_info(m): + syslog.syslog(syslog.LOG_INFO, msg) + + +def ts_now(): + return str(datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")) + + +def is_systemd_active(feat): + if not UNIT_TESTING: + status = os.system('systemctl is-active --quiet {}'.format(feat)) + else: + status = UNIT_TESTING_ACTIVE + log_debug("system status for {}: {}".format(feat, str(status))) + return status == 0 + + +def restart_systemd_service(server, feat, owner): + log_debug("Restart service {} to owner:{}".format(feat, owner)) + if not UNIT_TESTING: + status = os.system("systemctl restart {}".format(feat)) + else: + server.mod_db_entry(STATE_DB_NAME, + FEATURE_TABLE, feat, {"restart": "true"}) + status = 0 + if status != 0: + syslog.syslog(syslog.LOG_ERR, + "Failed to restart {} to switch to {}".format(feat, owner)) + else: + syslog.syslog(syslog.LOG_INFO, + "Restarted {} to switch to {}".format(feat, owner)) + return status + + +def init(): + if os.path.exists(SONIC_CTR_CONFIG): + with open(SONIC_CTR_CONFIG, "r") as s: + d = json.load(s) + remote_ctr_config.update(d) + + +class MainServer: + """ Implements main io-loop of the application + Accept handler registration per db/table. + Call handler on update + """ + + SELECT_TIMEOUT = 1000 + + def __init__(self): + """ Constructor """ + self.db_connectors = {} + self.selector = swsscommon.Select() + self.callbacks = defaultdict(lambda: defaultdict(list)) # db -> table -> handlers[] + self.timer_handlers = defaultdict(list) + self.subscribers = set() + + def register_db(self, db_name): + """ Get DB connector, if not there """ + if db_name not in self.db_connectors: + self.db_connectors[db_name] = swsscommon.DBConnector(db_name, 0) + + + def register_timer(self, ts, handler): + """ Register timer based handler. + The handler will be called on/after give timestamp, ts + """ + self.timer_handlers[ts].append(handler) + + + def register_handler(self, db_name, table_name, handler): + """ + Handler registration for any update in given table + in given db. The handler will be called for any update + to the table in this db. + """ + self.register_db(db_name) + + if table_name not in self.callbacks[db_name]: + conn = self.db_connectors[db_name] + subscriber = swsscommon.SubscriberStateTable(conn, table_name) + self.subscribers.add(subscriber) + self.selector.addSelectable(subscriber) + self.callbacks[db_name][table_name].append(handler) + + + def get_db_entry(self, db_name, table_name, key): + """ Return empty dict if key not present """ + conn = self.db_connectors[db_name] + tbl = swsscommon.Table(conn, table_name) + return dict(tbl.get(key)[1]) + + + def mod_db_entry(self, db_name, table_name, key, data): + """ Modify entry for given table|key with given dict type data """ + conn = self.db_connectors[db_name] + tbl = swsscommon.Table(conn, table_name) + print("mod_db_entry: db={} tbl={} key={} data={}".format(db_name, table_name, key, str(data))) + tbl.set(key, list(data.items())) + + + def set_db_entry(self, db_name, table_name, key, data): + """ Set given data as complete data, which includes + removing any fields that are in DB but not in data + """ + conn = self.db_connectors[db_name] + tbl = swsscommon.Table(conn, table_name) + ct_data = dict(tbl.get(key)[1]) + for k in ct_data: + if k not in data: + # Drop fields that are not in data + tbl.hdel(key, k) + tbl.set(key, list(data.items())) + + + def run(self): + """ Main loop """ + while True: + timeout = MainServer.SELECT_TIMEOUT + ct_ts = datetime.datetime.now() + while self.timer_handlers: + k = sorted(self.timer_handlers.keys())[0] + if k <= ct_ts: + lst = self.timer_handlers[k] + del self.timer_handlers[k] + for fn in lst: + fn() + else: + timeout = (k - ct_ts).seconds + break + + state, _ = self.selector.select(timeout) + if state == self.selector.TIMEOUT: + continue + elif state == self.selector.ERROR: + if not UNIT_TESTING: + raise Exception("Received error from select") + else: + print("Skipped Exception; Received error from select") + return + + for subscriber in self.subscribers: + key, op, fvs = subscriber.pop() + if not key: + continue + log_debug("Received message : '%s'" % str((key, op, fvs))) + for callback in (self.callbacks + [subscriber.getDbConnector().getDbName()] + [subscriber.getTableName()]): + callback(key, op, dict(fvs)) + + + +def set_node_labels(server): + labels = {} + + version_info = (device_info.get_sonic_version_info() if not UNIT_TESTING + else { "build_version": "20201230.111"}) + dev_data = server.get_db_entry(CONFIG_DB_NAME, 'DEVICE_METADATA', + 'localhost') + dep_type = dev_data['type'] if 'type' in dev_data else "unknown" + + labels["sonic_version"] = version_info['build_version'] + labels["hwsku"] = device_info.get_hwsku() if not UNIT_TESTING else "mock" + labels["deployment_type"] = dep_type + server.mod_db_entry(STATE_DB_NAME, + KUBE_LABEL_TABLE, KUBE_LABEL_SET_KEY, labels) + + +def _update_entry(ct, upd): + # Helper function, to update with case lowered. + ret = dict(ct) + for (k, v) in upd.items(): + ret[k.lower()] = v.lower() + + return ret + + +# +# SERVER-config changes: +# Act if IP or disable changes. +# If disabled or IP removed: +# reset connection +# else: +# join +# Update state-DB appropriately +# +class RemoteServerHandler: + def __init__(self, server): + """ Register self for updates """ + self.server = server + + server.register_handler( + CONFIG_DB_NAME, SERVER_TABLE, self.on_config_update) + self.cfg_server = _update_entry(dflt_cfg_ser, server.get_db_entry( + CONFIG_DB_NAME, SERVER_TABLE, SERVER_KEY)) + + log_debug("startup config: {}".format(str(self.cfg_server))) + + server.register_db(STATE_DB_NAME) + self.st_server = _update_entry(dflt_st_ser, server.get_db_entry( + STATE_DB_NAME, SERVER_TABLE, SERVER_KEY)) + + self.start_time = datetime.datetime.now() + + if not self.st_server[ST_FEAT_UPDATE_TS]: + # This is upon system start. Sleep 10m before join + self.start_time += datetime.timedelta( + seconds=remote_ctr_config[JOIN_LATENCY]) + server.register_timer(self.start_time, self.handle_update) + self.pending = True + log_debug("Pause to join {} seconds @ {}".format( + remote_ctr_config[JOIN_LATENCY], self.start_time)) + else: + self.pending = False + self.handle_update() + + + + def on_config_update(self, key, op, data): + """ On config update """ + if key != SERVER_KEY: + return + + cfg_data = _update_entry(dflt_cfg_ser, data) + if self.cfg_server == cfg_data: + log_debug("No change in server config") + return + + log_debug("Received config update: {}".format(str(data))) + self.cfg_server = cfg_data + + if self.pending: + tnow = datetime.datetime.now() + if tnow < self.start_time: + # Pausing for initial latency since reboot or last retry + due_secs = (self.start_time - tnow).seconds + else: + due_secs = 0 + log_debug("Pending to start in {} seconds at {}".format( + due_secs, self.start_time)) + return + + self.handle_update() + + + def handle_update(self): + # Called upon start and upon any config-update only. + # Called by timer / main thread. + # Main thread calls only if timer is not running. + # + self.pending = False + + ip = self.cfg_server[CFG_SER_IP] + disable = self.cfg_server[CFG_SER_DISABLE] != "false" + + pre_state = dict(self.st_server) + log_debug("server: handle_update: disable={} ip={}".format(disable, ip)) + if disable or not ip: + self.do_reset() + else: + self.do_join(ip, self.cfg_server[ST_SER_PORT], + self.cfg_server[CFG_SER_INSECURE]) + + if pre_state != self.st_server: + self.st_server[ST_SER_UPDATE_TS] = ts_now() + self.server.set_db_entry(STATE_DB_NAME, + SERVER_TABLE, SERVER_KEY, self.st_server) + log_debug("Update server state={}".format(str(self.st_server))) + + + def do_reset(self): + global remote_connected + + kube_commands.kube_reset_master(True) + + self.st_server[ST_SER_CONNECTED] = "false" + log_debug("kube_reset_master called") + + remote_connected = False + + + def do_join(self, ip, port, insecure): + global remote_connected + (ret, out, err) = kube_commands.kube_join_master( + ip, port, insecure) + + if ret == 0: + self.st_server[ST_SER_CONNECTED] = "true" + self.st_server[ST_SER_IP] = ip + self.st_server[ST_SER_PORT] = port + remote_connected = True + + set_node_labels(self.server) + log_debug("kube_join_master succeeded") + else: + # Ensure state reflects the current status + self.st_server[ST_SER_CONNECTED] = "false" + remote_connected = False + + # Join failed. Retry after an interval + self.start_time = datetime.datetime.now() + self.start_time += datetime.timedelta( + seconds=remote_ctr_config[JOIN_RETRY]) + self.server.register_timer(self.start_time, self.handle_update) + self.pending = True + + log_debug("kube_join_master failed retry after {} seconds @{}". + format(remote_ctr_config[JOIN_RETRY], self.start_time)) + + +# +# Feature changes +# +# Handle Set_owner change: +# restart service and/or label add/drop +# +# Handle remote_state change: +# When pending, trigger restart +# +class FeatureTransitionHandler: + def __init__(self, server): + self.server = server + self.cfg_data = defaultdict(lambda: defaultdict(str)) + self.st_data = defaultdict(lambda: defaultdict(str)) + server.register_handler( + CONFIG_DB_NAME, FEATURE_TABLE, self.on_config_update) + server.register_handler( + STATE_DB_NAME, FEATURE_TABLE, self.on_state_update) + return + + + def handle_update(self, feat, set_owner, ct_owner, remote_state): + # Called upon startup once for every feature in config & state DBs. + # There after only called upon changes in either that requires action + # + if not is_systemd_active(feat): + # Nothing todo, if system state is down + return + + label_add = set_owner == "kube" + service_restart = False + + if set_owner == "local": + if ct_owner != "local": + service_restart = True + else: + if (ct_owner != "none") and (remote_state == "pending"): + service_restart = True + + log_debug( + "feat={} set={} ct={} state={} restart={} label_add={}".format( + feat, set_owner, ct_owner, remote_state, service_restart, + label_add)) + # read labels and add/drop if different + self.server.mod_db_entry(STATE_DB_NAME, KUBE_LABEL_TABLE, KUBE_LABEL_SET_KEY, + { "{}_enabled".format(feat): ("true" if label_add else "false") }) + + + # service_restart + if service_restart: + restart_systemd_service(self.server, feat, set_owner) + + + + def on_config_update(self, key, op, data): + # Hint/Note: + # If the key don't pre-exist: + # This attempt to read will create the key for given + # field with empty string as value and return empty string + + init = key not in self.cfg_data + old_set_owner = self.cfg_data[key][CFG_FEAT_OWNER] + + self.cfg_data[key] = _update_entry(dflt_cfg_feat, data) + set_owner = self.cfg_data[key][CFG_FEAT_OWNER] + + if (not init) and (old_set_owner == set_owner): + # No change, bail out + log_debug("No change in feat={} set_owner={}. Bail out.".format( + key, set_owner)) + return + + if key in self.st_data: + log_debug("{} init={} old_set_owner={} owner={}".format(key, init, old_set_owner, set_owner)) + self.handle_update(key, set_owner, + self.st_data[key][ST_FEAT_OWNER], + self.st_data[key][ST_FEAT_REMOTE_STATE]) + + else: + # Handle it upon data from state-db, which must come + # if systemd is up + log_debug("Yet to get state info key={}. Bail out.".format(key)) + return + + + def on_state_update(self, key, op, data): + # Hint/Note: + # If the key don't pre-exist: + # This attempt to read will create the key for given + # field with empty string as value and return empty string + + init = key not in self.st_data + old_remote_state = self.st_data[key][ST_FEAT_REMOTE_STATE] + + self.st_data[key] = _update_entry(dflt_st_feat, data) + remote_state = self.st_data[key][ST_FEAT_REMOTE_STATE] + + if (not init) and ( + (old_remote_state == remote_state) or (remote_state != "pending")): + # no change or nothing to do. + return + + if key in self.cfg_data: + log_debug("{} init={} old_remote_state={} remote_state={}".format(key, init, old_remote_state, remote_state)) + self.handle_update(key, self.cfg_data[key][CFG_FEAT_OWNER], + self.st_data[key][ST_FEAT_OWNER], + remote_state) + return + + +# +# Label re-sync +# When labels applied at remote are pending, start monitoring +# API server reachability and trigger the label handler to +# apply the labels +# +# Non-empty KUBE_LABELS|PENDING, monitor API Server reachability +# +class LabelsPendingHandler: + def __init__(self, server): + server.register_handler(STATE_DB_NAME, KUBE_LABEL_TABLE, self.on_update) + self.server = server + self.pending = False + self.set_labels = {} + return + + + def on_update(self, key, op, data): + # For any update sync with kube API server. + # Don't optimize, as API server could differ with DB's contents + # in many ways. + # + if (key == KUBE_LABEL_SET_KEY): + self.set_labels = dict(data) + else: + return + + if remote_connected and not self.pending: + self.update_node_labels() + else: + log_debug("Skip label update: connected:{} pending:{}". + format(remote_connected, self.pending)) + + + + def update_node_labels(self): + # Called on config update by main thread upon init or config-change or + # it was not connected during last config update + # NOTE: remote-server-handler forces a config update notification upon + # join. + # Or it could be called by timer thread, if last upate to API server + # failed. + # + self.pending = False + ret = kube_commands.kube_write_labels(self.set_labels) + if (ret != 0): + self.pending = True + pause = remote_ctr_config[LABEL_RETRY] + ts = (datetime.datetime.now() + datetime.timedelta(seconds=pause)) + self.server.register_timer(ts, self.update_node_labels) + + log_debug("ret={} set={} pending={}".format(ret, + str(self.set_labels), self.pending)) + return + + +def main(): + init() + server = MainServer() + RemoteServerHandler(server) + FeatureTransitionHandler(server) + LabelsPendingHandler(server) + server.run() + print("ctrmgrd.py main called") + return 0 + + +if __name__ == '__main__': + if os.geteuid() != 0: + exit("Please run as root. Exiting ...") + main() diff --git a/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.service b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.service new file mode 100644 index 0000000000..6052c10785 --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/ctrmgrd.service @@ -0,0 +1,14 @@ +[Unit] +Description=Container Manager watcher daemon +Requires=updategraph.service +After=updategraph.service + + +[Service] +Type=simple +ExecStart=/usr/local/bin/ctrmgrd.py +Restart=always +RestartSec=30 + +[Install] +WantedBy=multi-user.target diff --git a/src/sonic-ctrmgrd/ctrmgr/kube_commands.py b/src/sonic-ctrmgrd/ctrmgr/kube_commands.py new file mode 100755 index 0000000000..1ebfa606f0 --- /dev/null +++ b/src/sonic-ctrmgrd/ctrmgr/kube_commands.py @@ -0,0 +1,391 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +import argparse +import fcntl +import inspect +import json +import os +import shutil +import ssl +import subprocess +import sys +import syslog +import tempfile +import urllib.request +from urllib.parse import urlparse + +import yaml +from sonic_py_common import device_info + +KUBE_ADMIN_CONF = "/etc/sonic/kube_admin.conf" +KUBELET_YAML = "/var/lib/kubelet/config.yaml" +SERVER_ADMIN_URL = "https://{}/admin.conf" +LOCK_FILE = "/var/lock/kube_join.lock" + +# kubectl --kubeconfig label nodes +#