[mgmt-framework]: convert mgmt-framework to use buster docker (#4480)

This change adds support to build dockers using buster as base.
- Define docker-base-buster using docker-base-stretch as starting point
- Define docker-config-engine-buster using docker-config-engine-stretch as starting point.
- sonic-mgmt-framework docker is updated to build using buster as base

Signed-off-by: Joyas Joseph <joyas_joseph@dell.com>
This commit is contained in:
joyas-joseph 2020-04-28 15:57:13 -07:00 committed by GitHub
parent a40fe3ba15
commit fc55329b22
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
20 changed files with 402 additions and 18 deletions

View File

@ -0,0 +1,98 @@
{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
{% if CONFIGURED_ARCH == "armhf" %}
FROM multiarch/debian-debootstrap:armhf-buster
{% elif CONFIGURED_ARCH == "arm64" %}
FROM multiarch/debian-debootstrap:arm64-buster
{% else %}
FROM debian:buster
{% endif %}
# Clean documentation in FROM image
RUN find /usr/share/doc -depth \( -type f -o -type l \) ! -name copyright | xargs rm || true
# Clean doc directories that are empty or only contain empty directories
RUN while [ -n "$(find /usr/share/doc -depth -type d -empty -print -exec rmdir {} +)" ]; do :; done && \
rm -rf \
/usr/share/man/* \
/usr/share/groff/* \
/usr/share/info/* \
/usr/share/lintian/* \
/usr/share/linda/* \
/var/cache/man/* \
/usr/share/locale/*
# Make apt-get non-interactive
ENV DEBIAN_FRONTEND=noninteractive
# Configure data sources for apt/dpkg
COPY ["dpkg_01_drop", "/etc/dpkg/dpkg.cfg.d/01_drop"]
{% if CONFIGURED_ARCH == "armhf" %}
COPY ["sources.list.armhf", "/etc/apt/sources.list"]
{% elif CONFIGURED_ARCH == "arm64" %}
COPY ["sources.list.arm64", "/etc/apt/sources.list"]
{% else %}
COPY ["sources.list", "/etc/apt/sources.list"]
{% endif %}
COPY ["no_install_recommend_suggest", "/etc/apt/apt.conf.d"]
COPY ["no-check-valid-until", "/etc/apt/apt.conf.d"]
# Update apt cache and
# pre-install fundamental packages
RUN apt-get update && \
apt-get -y install \
less \
perl \
procps \
python \
rsyslog \
vim-tiny \
# Install dependencies of supervisor
python-pkg-resources \
python-meld3 \
# dependencies of redis-tools
libatomic1 \
libjemalloc2 \
liblua5.1-0 \
lua-bitop \
lua-cjson \
# common dependencies
libpython2.7 \
libdaemon0 \
libdbus-1-3 \
libjansson4
# ip and ifconfig utility missing in docker for arm arch
RUN apt-get -y install \
iproute2 \
net-tools
RUN mkdir -p /etc/supervisor /var/log/supervisor
RUN apt-get -y purge \
exim4 \
exim4-base \
exim4-config \
exim4-daemon-light
{% if docker_base_buster_debs.strip() -%}
# Copy locally-built Debian package dependencies
{{ copy_files("debs/", docker_base_buster_debs.split(' '), "/debs/") }}
# Install built Debian packages and implicitly install their dependencies
{{ install_debian_packages(docker_base_buster_debs.split(' ')) }}
{%- endif %}
# Clean up apt
# Remove /var/lib/apt/lists/*, could be obsoleted for derived images
RUN apt-get clean -y && \
apt-get autoclean -y && \
apt-get autoremove -y && \
rm -rf /var/lib/apt/lists/* /tmp/*
COPY ["etc/rsyslog.conf", "/etc/rsyslog.conf"]
COPY ["etc/rsyslog.d/*", "/etc/rsyslog.d/"]
COPY ["root/.vimrc", "/root/.vimrc"]
RUN ln /usr/bin/vim.tiny /usr/bin/vim
COPY ["etc/supervisor/supervisord.conf", "/etc/supervisor/"]

View File

@ -0,0 +1,13 @@
Copyright 2016 Microsoft, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

View File

@ -0,0 +1,30 @@
## Drop unnecessary files
## ref: https://wiki.ubuntu.com/ReducingDiskFootprint
## Documentation
path-exclude /usr/share/doc/*
# we need to keep copyright files for legal reasons
path-include /usr/share/doc/*/copyright
path-exclude /usr/share/man/*
path-exclude /usr/share/groff/*
path-exclude /usr/share/info/*
# lintian stuff is small, but really unnecessary
path-exclude /usr/share/lintian/*
path-exclude /usr/share/linda/*
## Translations
path-exclude /usr/share/locale/*
## Landscape
path-exclude /usr/share/pyshared/twisted/test*
path-exclude /usr/lib/python*/dist-packages/twisted/test*
path-exclude /usr/share/pyshared/twisted/*/test*
path-exclude /usr/lib/python*/dist-packages/twisted/*/test*
## install the configuration file if its currently missing
force-confmiss
## combined with confold: overwrite configuration files that you have not modified
force-confdef
## do not modify the current configuration file, the new version is installed with a .dpkg-dist suffix
force-confold

View File

@ -0,0 +1,76 @@
#
# /etc/rsyslog.conf Configuration file for rsyslog.
#
# For more information see
# /usr/share/doc/rsyslog-doc/html/rsyslog_conf.html
#################
#### MODULES ####
#################
$ModLoad imuxsock # provides support for local system logging
#
# Set a rate limit on messages from the container
#
$SystemLogRateLimitInterval 300
$SystemLogRateLimitBurst 20000
#$ModLoad imklog # provides kernel logging support
#$ModLoad immark # provides --MARK-- message capability
# provides UDP syslog reception
#$ModLoad imudp
#$UDPServerRun 514
# provides TCP syslog reception
#$ModLoad imtcp
#$InputTCPServerRun 514
###########################
#### GLOBAL DIRECTIVES ####
###########################
# Set remote syslog server
template (name="ForwardFormatInContainer" type="string" string="<%PRI%>%TIMESTAMP:::date-rfc3339% %HOSTNAME% %syslogtag%%msg:::sp-if-no-1st-sp%%msg%")
*.* action(type="omfwd" target="127.0.0.1" port="514" protocol="udp" Template="ForwardFormatInContainer")
#
# Use traditional timestamp format.
# To enable high precision timestamps, comment out the following line.
#
#$ActionFileDefaultTemplate RSYSLOG_TraditionalFileFormat
# Define a custom template
$template SONiCFileFormat,"%TIMESTAMP%.%timestamp:::date-subseconds% %HOSTNAME% %syslogseverity-text:::uppercase% %syslogtag%%msg:::sp-if-no-1st-sp%%msg:::drop-last-lf%\n"
$ActionFileDefaultTemplate SONiCFileFormat
#
# Set the default permissions for all log files.
#
$FileOwner root
$FileGroup adm
$FileCreateMode 0640
$DirCreateMode 0755
$Umask 0022
#
# Where to place spool and state files
#
$WorkDirectory /var/spool/rsyslog
#
# Include all config files in /etc/rsyslog.d/
#
$IncludeConfig /etc/rsyslog.d/*.conf
#
# Suppress duplicate messages and report "message repeated n times"
#
$RepeatedMsgReduction on
###############
#### RULES ####
###############

View File

@ -0,0 +1,9 @@
$ModLoad imfile
$InputFileName /var/log/supervisor/supervisord.log
$InputFileTag supervisord
$InputFileStateFile state-supervisor
$InputFileSeverity info
$InputFileFacility local0
$InputFilePersistStateInterval 1
$InputRunFileMonitor

View File

@ -0,0 +1,33 @@
; supervisor config file
[unix_http_server]
file=/var/run/supervisor.sock ; (the path to the socket file)
chmod=0700 ; socket file mode (default 0700)
username=dummy
password=dummy
[supervisord]
logfile=/var/log/supervisor/supervisord.log ; (main log file;default $CWD/supervisord.log)
pidfile=/var/run/supervisord.pid ; (supervisord pidfile;default supervisord.pid)
childlogdir=/var/log/supervisor ; ('AUTO' child log dir, default $TEMP)
user=root
; the below section must remain in the config file for RPC
; (supervisorctl/web interface) to work, additional interfaces may be
; added by defining them in separate rpcinterface: sections
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///var/run/supervisor.sock ; use a unix:// URL for a unix socket
username=dummy
password=dummy
; The [include] section can just contain the "files" setting. This
; setting can list multiple files (separated by whitespace or
; newlines). It can also contain wildcards. The filenames are
; interpreted as relative to this file. Included files *cannot*
; include files themselves.
[include]
files = /etc/supervisor/conf.d/*.conf

View File

@ -0,0 +1,4 @@
# Instruct apt-get to NOT check the "Valid Until" date in Release files
# Once the Debian team archives a repo, they stop updating this date
Acquire::Check-Valid-Until "false";

View File

@ -0,0 +1,5 @@
# Instruct apt-get to NOT install "recommended" or "suggested" packages by
# default when installing a package.
APT::Install-Recommends "false";
APT::Install-Suggests "false";

View File

@ -0,0 +1,2 @@
" enable vim features
set nocompatible

View File

@ -0,0 +1,8 @@
## Debian mirror on Microsoft Azure
## Ref: http://debian-archive.trafficmanager.net/
deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free
deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster main contrib non-free
deb [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free
deb-src [arch=amd64] http://debian-archive.trafficmanager.net/debian-security/ buster/updates main contrib non-free
deb [arch=amd64] http://debian-archive.trafficmanager.net/debian/ buster-backports main contrib non-free

View File

@ -0,0 +1,7 @@
## Debian mirror for ARM repo
# ARM repo
deb [arch=arm64] http://deb.debian.org/debian buster main contrib non-free
deb-src [arch=arm64] http://deb.debian.org/debian buster main contrib non-free
deb [arch=arm64] http://security.debian.org buster/updates main contrib non-free
deb-src [arch=arm64] http://security.debian.org buster/updates main contrib non-free

View File

@ -0,0 +1,7 @@
## Debian mirror for ARM repo
# ARM repo
deb [arch=armhf] http://deb.debian.org/debian buster main contrib non-free
deb-src [arch=armhf] http://deb.debian.org/debian buster main contrib non-free
deb [arch=armhf] http://security.debian.org buster/updates main contrib non-free
deb-src [arch=armhf] http://security.debian.org buster/updates main contrib non-free

View File

@ -0,0 +1,50 @@
{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
FROM docker-base-buster
## Make apt-get non-interactive
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y \
# Dependencies for sonic-cfggen
python-lxml \
python-yaml \
python-bitarray \
python-pip \
python-dev \
python-natsort \
apt-utils \
python-setuptools
RUN pip install --upgrade pip
RUN pip install \
netaddr \
ipaddr \
jinja2 \
pyangbind==0.6.0
{% if docker_config_engine_buster_debs.strip() %}
# Copy locally-built Debian package dependencies
{{ copy_files("debs/", docker_config_engine_buster_debs.split(' '), "/debs/") }}
# Install locally-built Debian packages and implicitly install their dependencies
{{ install_debian_packages(docker_config_engine_buster_debs.split(' ')) }}
{% endif %}
{% if docker_config_engine_buster_whls.strip() %}
# Copy locally-built Python wheel dependencies
{{ copy_files("python-wheels/", docker_config_engine_buster_whls.split(' '), "/python-wheels/") }}
# Install locally-built Python wheel dependencies
{{ install_python_wheels(docker_config_engine_buster_whls.split(' ')) }}
{% endif %}
## Clean up
RUN apt-get purge -y \
python-pip \
python-dev && \
apt-get clean -y && \
apt-get autoclean -y && \
apt-get autoremove -y && \
rm -rf /debs /python-wheels

View File

@ -1,4 +1,4 @@
FROM docker-config-engine-stretch
FROM docker-config-engine-buster
ARG docker_container_name
RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
@ -10,18 +10,12 @@ RUN apt-get update
RUN pip install connexion==1.1.15 \
setuptools==21.0.0 \
grpcio-tools==1.20.0 \
pyangbind==0.6.0 \
certifi==2017.4.17 \
python-dateutil==2.6.0 \
six==1.11.0 \
urllib3==1.21.1
## Install redis-tools dependencies
## TODO: implicitly install dependencies
RUN apt-get -y install libjemalloc1 libatomic1 liblua5.1-0 lua-bitop lua-cjson
COPY \
{% for deb in docker_sonic_mgmt_framework_debs.split(' ') -%}
debs/{{ deb }}{{' '}}

View File

@ -0,0 +1,10 @@
DPATH := $($(DOCKER_BASE_BUSTER)_PATH)
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/docker-base-buster.mk rules/docker-base-buster.dep
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
DEP_FILES += $(shell git ls-files $(DPATH))
$(DOCKER_BASE_BUSTER)_CACHE_MODE := GIT_CONTENT_SHA
$(DOCKER_BASE_BUSTER)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
$(DOCKER_BASE_BUSTER)_DEP_FILES := $(DEP_FILES)

View File

@ -0,0 +1,17 @@
# Docker base image (based on Debian Buster)
DOCKER_BASE_BUSTER = docker-base-buster.gz
$(DOCKER_BASE_BUSTER)_PATH = $(DOCKERS_PATH)/docker-base-buster
$(DOCKER_BASE_BUSTER)_DEPENDS += $(SUPERVISOR) $(REDIS_TOOLS)
$(DOCKER_BASE_BUSTER)_DEPENDS += $(SOCAT)
GDB = gdb
GDBSERVER = gdbserver
VIM = vim
OPENSSH = openssh-client
SSHPASS = sshpass
STRACE = strace
$(DOCKER_BASE_BUSTER)_DBG_IMAGE_PACKAGES += $(GDB) $(GDBSERVER) $(VIM) $(OPENSSH) $(SSHPASS) $(STRACE)
SONIC_DOCKER_IMAGES += $(DOCKER_BASE_BUSTER)

View File

@ -0,0 +1,10 @@
DPATH := $($(DOCKER_CONFIG_ENGINE_BUSTER)_PATH)
DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/docker-config-engine-buster.mk rules/docker-config-engine-buster.dep
DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
DEP_FILES += $(shell git ls-files $(DPATH))
$(DOCKER_CONFIG_ENGINE_BUSTER)_CACHE_MODE := GIT_CONTENT_SHA
$(DOCKER_CONFIG_ENGINE_BUSTER)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
$(DOCKER_CONFIG_ENGINE_BUSTER)_DEP_FILES := $(DEP_FILES)

View File

@ -0,0 +1,13 @@
# docker image for sonic config engine
DOCKER_CONFIG_ENGINE_BUSTER = docker-config-engine-buster.gz
$(DOCKER_CONFIG_ENGINE_BUSTER)_PATH = $(DOCKERS_PATH)/docker-config-engine-buster
$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SWSSSDK_PY2)
$(DOCKER_CONFIG_ENGINE_BUSTER)_PYTHON_WHEELS += $(SONIC_CONFIG_ENGINE)
$(DOCKER_CONFIG_ENGINE_BUSTER)_LOAD_DOCKERS += $(DOCKER_BASE_BUSTER)
$(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS = $($(DOCKER_BASE_BUSTER)_DBG_DEPENDS)
$(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_PACKAGES = $($(DOCKER_BASE_BUSTER)_DBG_IMAGE_PACKAGES)
SONIC_DOCKER_IMAGES += $(DOCKER_CONFIG_ENGINE_BUSTER)

View File

@ -7,22 +7,20 @@ DOCKER_MGMT_FRAMEWORK_DBG = $(DOCKER_MGMT_FRAMEWORK_STEM)-$(DBG_IMAGE_MARK).gz
$(DOCKER_MGMT_FRAMEWORK)_PATH = $(DOCKERS_PATH)/$(DOCKER_MGMT_FRAMEWORK_STEM)
$(DOCKER_MGMT_FRAMEWORK)_DEPENDS += $(REDIS_TOOLS) $(SONIC_MGMT_FRAMEWORK)
$(DOCKER_MGMT_FRAMEWORK)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_DEPENDS)
$(DOCKER_MGMT_FRAMEWORK)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS)
$(DOCKER_MGMT_FRAMEWORK)_DBG_DEPENDS += $(REDIS_TOOLS) $(SONIC_MGMT_FRAMEWORK_DBG)
SONIC_DOCKER_IMAGES += $(DOCKER_MGMT_FRAMEWORK)
$(DOCKER_MGMT_FRAMEWORK)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_STRETCH)
$(DOCKER_MGMT_FRAMEWORK)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES)
$(DOCKER_MGMT_FRAMEWORK)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_BUSTER)
$(DOCKER_MGMT_FRAMEWORK)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_PACKAGES)
ifeq ($(ENABLE_MGMT_FRAMEWORK), y)
SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_MGMT_FRAMEWORK)
SONIC_STRETCH_DOCKERS += $(DOCKER_MGMT_FRAMEWORK)
endif
SONIC_DOCKER_DBG_IMAGES += $(DOCKER_MGMT_FRAMEWORK_DBG)
ifeq ($(ENABLE_MGMT_FRAMEWORK), y)
SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_MGMT_FRAMEWORK_DBG)
SONIC_STRETCH_DBG_DOCKERS += $(DOCKER_MGMT_FRAMEWORK_DBG)
endif
$(DOCKER_MGMT_FRAMEWORK)_CONTAINER_NAME = mgmt-framework

View File

@ -638,15 +638,15 @@ ifeq ($(BLDENV),stretch)
STRETCH_DOCKER_IMAGES = $(filter $(SONIC_STRETCH_DOCKERS),$(DOCKER_IMAGES_FOR_INSTALLERS) $(EXTRA_DOCKER_TARGETS))
STRETCH_DBG_DOCKER_IMAGES = $(filter $(SONIC_STRETCH_DBG_DOCKERS), $(patsubst %.gz,%-$(DBG_IMAGE_MARK).gz, $(STRETCH_DOCKER_IMAGES)))
else
DOCKER_IMAGES := $(SONIC_DOCKER_IMAGES)
DOCKER_DBG_IMAGES := $(SONIC_DOCKER_DBG_IMAGES)
DOCKER_IMAGES = $(filter-out $(SONIC_JESSIE_DOCKERS) $(SONIC_STRETCH_DOCKERS),$(SONIC_DOCKER_IMAGES))
DOCKER_DBG_IMAGES = $(filter-out $(SONIC_JESSIE_DBG_DOCKERS) $(SONIC_STRETCH_DBG_DOCKERS), $(SONIC_DOCKER_DBG_IMAGES))
endif
endif
$(foreach IMAGE,$(SONIC_STRETCH_DOCKERS), $(eval $(IMAGE)_DEBS_PATH := $(STRETCH_DEBS_PATH)))
$(foreach IMAGE,$(SONIC_STRETCH_DOCKERS), $(eval $(IMAGE)_FILES_PATH := $(STRETCH_FILES_PATH)))
$(foreach IMAGE,$(SONIC_STRETCH_DBG_DOCKERS), $(eval $(IMAGE)_DEBS_PATH := $(STRETCH_DEBS_PATH)))
$(foreach IMAGE,$(SONIC_STRETCH_DBG_DOCKERS), $(eval $(IMAGE)_FILES_PATH := $(STRETCH_FILES_PATH)))
$(foreach IMAGE,$(DOCKER_IMAGES), $(eval $(IMAGE)_DEBS_PATH := $(DEBS_PATH)))
$(foreach IMAGE,$(DOCKER_IMAGES), $(eval $(IMAGE)_FILES_PATH := $(FILES_PATH)))
$(foreach IMAGE,$(DOCKER_DBG_IMAGES), $(eval $(IMAGE)_DEBS_PATH := $(DEBS_PATH)))
$(foreach IMAGE,$(DOCKER_DBG_IMAGES), $(eval $(IMAGE)_FILES_PATH := $(FILES_PATH)))
# Targets for building docker images
$(addprefix $(TARGET_PATH)/, $(DOCKER_IMAGES)) : $(TARGET_PATH)/%.gz : .platform docker-start \