Add Bookworm base and config-engine layers (#17742)

The layers compile for amd64; however, functionality has not been
tested.

Signed-off-by: Saikrishna Arcot <sarcot@microsoft.com>

dockers/docker-base-bookworm/Dockerfile.j2

@@ -0,0 +1,126 @@
+{% set prefix = DEFAULT_CONTAINER_REGISTRY %}
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+{% if CONFIGURED_ARCH == "armhf" and (MULTIARCH_QEMU_ENVIRON == "y" or CROSS_BUILD_ENVIRON == "y") %}
+FROM {{ prefix }}multiarch/debian-debootstrap:armhf-bookworm
+{% elif CONFIGURED_ARCH == "arm64" and (MULTIARCH_QEMU_ENVIRON == "y" or CROSS_BUILD_ENVIRON == "y") %}
+FROM {{ prefix }}multiarch/debian-debootstrap:arm64-bookworm
+{% else %}
+FROM {{ prefix }}{{DOCKER_BASE_ARCH}}/debian:bookworm
+{% endif %}
+
+# Clean documentation in FROM image
+RUN find /usr/share/doc -depth \( -type f -o -type l \) ! -name copyright | xargs rm || true
+
+# Clean doc directories that are empty or only contain empty directories
+RUN while [ -n "$(find /usr/share/doc -depth -type d -empty -print -exec rmdir {} +)" ]; do :; done && \
+    rm -rf               \
+    /usr/share/man/*     \
+    /usr/share/groff/*   \
+    /usr/share/info/*    \
+    /usr/share/lintian/* \
+    /usr/share/linda/*   \
+    /var/cache/man/*     \
+    /usr/share/locale/*
+
+# Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+# Configure data sources for apt/dpkg
+COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
+COPY ["sources.list.{{ CONFIGURED_ARCH }}", "/etc/apt/sources.list"]
+COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
+COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
+COPY ["apt-multiple-retries", "/etc/apt/apt.conf.d"]
+
+# Update apt cache and
+# pre-install fundamental packages
+RUN apt update &&            \
+    apt -y install           \
+        curl                 \
+        less                 \
+        perl                 \
+        procps               \
+        python3              \
+        python3-distutils    \
+        python3-pip          \
+        python3-setuptools   \
+        python3-wheel        \
+        python-is-python3    \
+        vim-tiny             \
+        rsyslog              \
+# Install redis-tools
+        redis-tools          \
+# common dependencies
+        libdaemon0           \
+        libdbus-1-3          \
+        libjansson4          \
+# ip and ifconfig utility missing in docker for arm arch
+        iproute2             \
+        net-tools            \
+# for processing/handling json files in bash environment
+        jq                   \
+# for sairedis zmq rpc channel
+        libzmq5              \
+        libwrap0
+
+# Add a config file to allow pip to install packages outside of apt/the Debian repos
+COPY ["pip.conf", "/etc/pip.conf"]
+
+# Upgrade pip via PyPI and uninstall the Debian version
+RUN pip3 install --upgrade pip
+RUN apt purge -y python3-pip
+
+# For templating
+RUN pip3 install j2cli
+
+# Install supervisor
+RUN pip3 install supervisor==4.2.5
+
+# Add support for supervisord to handle startup dependencies
+RUN pip3 install supervisord-dependent-startup==1.4.0
+
+RUN mkdir -p /var/log/supervisor /etc/supervisor/conf.d
+
+# Install gcc, libc6-dev and python3-dev for compiling python-lzf
+RUN apt-get -y install build-essential libc6-dev python3-dev
+
+# Install python-lzf
+RUN pip3 install 'python-lzf==0.2.4'
+
+# Install rdbtools
+RUN pip3 install 'rdbtools==0.1.15'
+
+# Uninstall gcc, libc6-dev and python3-dev for compiling python-lzf
+RUN apt-get -y purge build-essential libc6-dev python3-dev
+
+# Uninstall unused dependencies
+RUN apt autoremove -y --purge
+
+RUN apt-get -y purge   \
+    exim4              \
+    exim4-base         \
+    exim4-config       \
+    exim4-daemon-light
+
+{% if docker_base_bookworm_debs.strip() -%}
+# Copy locally-built Debian package dependencies
+{{ copy_files("debs/", docker_base_bookworm_debs.split(' '), "/debs/") }}
+
+# Install built Debian packages and implicitly install their dependencies
+{{ install_debian_packages(docker_base_bookworm_debs.split(' ')) }}
+{%- endif %}
+
+# Clean up apt
+# Remove /var/lib/apt/lists/*, could be obsoleted for derived images
+RUN apt-get clean -y                     && \
+    apt-get autoclean -y                 && \
+    apt-get autoremove -y                && \
+    rm -rf /var/lib/apt/lists/* /tmp/* ~/.cache
+
+COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
+COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
+COPY ["root/.vimrc", "/root/.vimrc"]
+
+RUN ln /usr/bin/vim.tiny /usr/bin/vim
+
+COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]

dockers/docker-base-bookworm/LICENSE

@@ -0,0 +1,13 @@
+Copyright 2016 Microsoft, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

dockers/docker-base-bookworm/apt-multiple-retries

@@ -0,0 +1,4 @@
+# Instruct apt to retry downloads on failures
+# This is required only for bullseye.
+
+Acquire::Retries "3";

dockers/docker-base-bookworm/dpkg_01_drop

@@ -0,0 +1,29 @@
+## Drop unnecessary files
+## ref: https://wiki.ubuntu.com/ReducingDiskFootprint
+
+## Documentation
+path-exclude /usr/share/doc/*
+# we need to keep copyright files for legal reasons
+path-include /usr/share/doc/*/copyright
+path-exclude /usr/share/man/*
+path-exclude /usr/share/groff/*
+path-exclude /usr/share/info/*
+# lintian stuff is small, but really unnecessary
+path-exclude /usr/share/lintian/*
+path-exclude /usr/share/linda/*
+
+## Translations
+path-exclude /usr/share/locale/*
+
+## Landscape
+path-exclude /usr/share/pyshared/twisted/test*
+path-exclude /usr/lib/python*/dist-packages/twisted/test*
+path-exclude /usr/share/pyshared/twisted/*/test*
+path-exclude /usr/lib/python*/dist-packages/twisted/*/test*
+
+## install the configuration file if it’s currently missing
+force-confmiss
+## combined with confold: overwrite configuration files that you have not modified
+force-confdef
+## do not modify the current configuration file, the new version is installed with a .dpkg-dist suffix
+force-confold

dockers/docker-base-bookworm/etc/rsyslog.conf

@@ -0,0 +1,78 @@
+#
+#  /etc/rsyslog.conf    Configuration file for rsyslog.
+#
+#                       For more information see
+#                       /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
+
+
+#################
+#### MODULES ####
+#################
+
+$ModLoad imuxsock # provides support for local system logging
+
+#
+# Set a rate limit on messages from the container
+#
+$SystemLogRateLimitInterval 300
+$SystemLogRateLimitBurst 20000
+
+#$ModLoad imklog  # provides kernel logging support
+#$ModLoad immark  # provides --MARK-- message capability
+
+# provides UDP syslog reception
+#$ModLoad imudp
+#$UDPServerRun 514
+
+# provides TCP syslog reception
+#$ModLoad imtcp
+#$InputTCPServerRun 514
+
+
+###########################
+#### GLOBAL DIRECTIVES ####
+###########################
+
+set $.CONTAINER_NAME=getenv("CONTAINER_NAME");
+
+# Set remote syslog server
+template (name="ForwardFormatInContainer" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg%")
+*.* action(type="omfwd" target=`echo $SYSLOG_TARGET_IP` port="514" protocol="udp" Template="ForwardFormatInContainer")
+
+#
+# Use traditional timestamp format.
+# To enable high precision timestamps, comment out the following line.
+#
+#$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
+
+# Define a custom template
+$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %$.CONTAINER_NAME%#%syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
+$ActionFileDefaultTemplate SONiCFileFormat
+
+#
+# Set the default permissions for all log files.
+#
+$FileOwner root
+$FileGroup adm
+$FileCreateMode 0640
+$DirCreateMode 0755
+$Umask 0022
+
+#
+# Where to place spool and state files
+#
+$WorkDirectory /var/spool/rsyslog
+
+#
+# Include all config files in /etc/rsyslog.d/
+#
+$IncludeConfig /etc/rsyslog.d/*.conf
+
+#
+# Suppress duplicate messages and report "message repeated n times"
+#
+$RepeatedMsgReduction on
+
+###############
+#### RULES ####
+###############

dockers/docker-base-bookworm/etc/rsyslog.d/supervisor.conf

@@ -0,0 +1,9 @@
+module(load="imfile" mode="inotify") # Ensure "inotify" mode is used
+$WorkDirectory /var/log/supervisor
+# Start Monitoring the file
+input(type="imfile"
+      File="/var/log/supervisor/supervisord.log"
+      Tag="supervisord"
+      Severity="info"
+      Facility="local0"
+      PersistStateInterval="1")

dockers/docker-base-bookworm/etc/supervisor/supervisord.conf

@@ -0,0 +1,29 @@
+; supervisor config file
+
+[unix_http_server]
+file=/var/run/supervisor.sock  ; (the path to the socket file)
+chmod=0700                     ; socket file mode (default 0700)
+
+[supervisord]
+logfile=/var/log/supervisor/supervisord.log  ; (main log file;default $CWD/supervisord.log)
+pidfile=/var/run/supervisord.pid             ; (supervisord pidfile;default supervisord.pid)
+childlogdir=/var/log/supervisor              ; ('AUTO' child log dir, default $TEMP)
+user=root
+
+; the below section must remain in the config file for RPC
+; (supervisorctl/web interface) to work, additional interfaces may be
+; added by defining them in separate rpcinterface: sections
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl=unix:///var/run/supervisor.sock  ; use a unix:// URL  for a unix socket
+
+; The [include] section can just contain the "files" setting.  This
+; setting can list multiple files (separated by whitespace or
+; newlines).  It can also contain wildcards.  The filenames are
+; interpreted as relative to this file.  Included files *cannot*
+; include files themselves.
+
+[include]
+files = /etc/supervisor/conf.d/*.conf

dockers/docker-base-bookworm/no-check-valid-until

@@ -0,0 +1,4 @@
+# Instruct apt-get to NOT check the "Valid Until" date in Release files
+# Once the Debian team archives a repo, they stop updating this date
+
+Acquire::Check-Valid-Until "false";

dockers/docker-base-bookworm/no_install_recommend_suggest

@@ -0,0 +1,5 @@
+# Instruct apt-get to NOT install "recommended" or "suggested" packages by
+# default when installing a package.
+
+APT::Install-Recommends "false";
+APT::Install-Suggests "false";

dockers/docker-base-bookworm/root/.vimrc

@@ -0,0 +1,2 @@
+" enable vim features
+set nocompatible

dockers/docker-config-engine-bookworm/00-load-omprog.conf

@@ -0,0 +1 @@
+$ModLoad omprog

dockers/docker-config-engine-bookworm/Dockerfile.j2

@@ -0,0 +1,59 @@
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+FROM docker-base-bookworm-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}}
+
+## Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update         && \
+    apt-get install -y        \
+        apt-utils             \
+        build-essential       \
+        python3-dev           \
+        python3-yaml
+
+{%- if CONFIGURED_ARCH == "armhf" or CONFIGURED_ARCH == "arm64" %}
+RUN apt-get install -y        \
+        libxslt-dev           \
+        libz-dev
+{%- endif %}
+
+# For sonic-config-engine Python 3 package
+# Explicitly install pyangbind here, as pyangbind causes enum34 to be installed.
+# enum34 causes Python 're' package to not work properly as it redefines an incompatible enum.py module
+# https://github.com/robshakir/pyangbind/issues/232
+RUN pip3 install pyangbind==0.8.1
+RUN pip3 uninstall -y enum34
+
+# Install python-redis
+RUN pip3 install redis==5.0.1
+
+{% if docker_config_engine_bookworm_debs.strip() %}
+# Copy locally-built Debian package dependencies
+{{ copy_files("debs/", docker_config_engine_bookworm_debs.split(' '), "/debs/") }}
+
+# Install locally-built Debian packages and implicitly install their dependencies
+{{ install_debian_packages(docker_config_engine_bookworm_debs.split(' ')) }}
+{% endif %}
+
+{% if docker_config_engine_bookworm_whls.strip() %}
+# Copy locally-built Python wheel dependencies
+{{ copy_files("python-wheels/", docker_config_engine_bookworm_whls.split(' '), "/python-wheels/") }}
+
+# Install locally-built Python wheel dependencies
+{{ install_python_wheels(docker_config_engine_bookworm_whls.split(' ')) }}
+{% endif %}
+
+# Copy files
+COPY ["files/swss_vars.j2", "/usr/share/sonic/templates/"]
+COPY ["files/readiness_probe.sh", "/usr/bin/"]
+COPY ["files/container_startup.py", "/usr/share/sonic/scripts/"]
+COPY ["00-load-omprog.conf", "/etc/rsyslog.d/"]
+
+## Clean up
+RUN apt-get purge -y               \
+        python3-dev                \
+        build-essential         && \
+    apt-get clean -y            && \
+    apt-get autoclean -y        && \
+    apt-get autoremove -y       && \
+    rm -rf /debs /python-wheels ~/.cache

rules/docker-base-bookworm.dep

@@ -0,0 +1,9 @@
+
+DPATH       := $($(DOCKER_BASE_BOOKWORM)_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/docker-base-bookworm.mk rules/docker-base-bookworm.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(DPATH))
+
+$(DOCKER_BASE_BOOKWORM)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(DOCKER_BASE_BOOKWORM)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(DOCKER_BASE_BOOKWORM)_DEP_FILES   := $(DEP_FILES)

rules/docker-base-bookworm.mk

@@ -0,0 +1,22 @@
+# Docker base image (based on Debian Bookworm)
+
+DOCKER_BASE_BOOKWORM = docker-base-bookworm.gz
+$(DOCKER_BASE_BOOKWORM)_PATH = $(DOCKERS_PATH)/docker-base-bookworm
+
+$(DOCKER_BASE_BOOKWORM)_DEPENDS += $(SOCAT)
+
+GDB = gdb
+GDBSERVER = gdbserver
+VIM = vim
+OPENSSH = openssh-client
+SSHPASS = sshpass
+STRACE = strace
+
+ifeq ($(INCLUDE_FIPS), y)
+$(DOCKER_BASE_BOOKWORM)_DEPENDS += $(FIPS_OPENSSL_LIBSSL) $(FIPS_OPENSSL_LIBSSL_DEV) $(FIPS_OPENSSL) $(SYMCRYPT_OPENSSL) $(FIPS_KRB5)
+endif
+
+$(DOCKER_BASE_BOOKWORM)_DBG_IMAGE_PACKAGES += $(GDB) $(GDBSERVER) $(VIM) $(OPENSSH) $(SSHPASS) $(STRACE)
+
+SONIC_DOCKER_IMAGES += $(DOCKER_BASE_BOOKWORM)
+SONIC_BOOKWORM_DOCKERS += $(DOCKER_BASE_BOOKWORM)

rules/docker-config-engine-bookworm.dep

@@ -0,0 +1,9 @@
+
+DPATH       := $($(DOCKER_CONFIG_ENGINE_BOOKWORM)_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/docker-config-engine-bookworm.mk rules/docker-config-engine-bookworm.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES   += $(shell git ls-files $(DPATH))
+
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_DEP_FILES   := $(DEP_FILES)

rules/docker-config-engine-bookworm.mk

@@ -0,0 +1,30 @@
+# docker image for sonic config engine
+
+DOCKER_CONFIG_ENGINE_BOOKWORM = docker-config-engine-bookworm.gz
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_PATH = $(DOCKERS_PATH)/docker-config-engine-bookworm
+
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_DEPENDS += $(LIBSWSSCOMMON) \
+                                          $(LIBYANG) \
+                                          $(LIBYANG_CPP) \
+                                          $(LIBYANG_PY3) \
+                                          $(PYTHON3_SWSSCOMMON) \
+                                          $(SONIC_DB_CLI) \
+                                          $(SONIC_EVENTD)
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_PYTHON_WHEELS += $(SONIC_PY_COMMON_PY3) \
+                                                  $(SONIC_YANG_MGMT_PY3) \
+                                                  $(SONIC_YANG_MODELS_PY3) \
+                                                  $(SONIC_CONTAINERCFGD)
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE_PY3)
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_LOAD_DOCKERS += $(DOCKER_BASE_BOOKWORM)
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_FILES += $(SWSS_VARS_TEMPLATE)
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_FILES += $(RSYSLOG_PLUGIN_CONF_J2)
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_FILES += $($(SONIC_CTRMGRD)_CONTAINER_SCRIPT)
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_FILES += $($(SONIC_CTRMGRD)_HEALTH_PROBE)
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_FILES += $($(SONIC_CTRMGRD)_STARTUP_SCRIPT)
+
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_DBG_DEPENDS = $($(DOCKER_BASE_BOOKWORM)_DBG_DEPENDS) \
+                                             $(LIBSWSSCOMMON_DBG)
+$(DOCKER_CONFIG_ENGINE_BOOKWORM)_DBG_IMAGE_PACKAGES = $($(DOCKER_BASE_BOOKWORM)_DBG_IMAGE_PACKAGES)
+
+SONIC_DOCKER_IMAGES += $(DOCKER_CONFIG_ENGINE_BOOKWORM)
+SONIC_BOOKWORM_DOCKERS += $(DOCKER_CONFIG_ENGINE_BOOKWORM)