diff --git a/.gitmodules b/.gitmodules index cfc6878c72..b9831ba332 100644 --- a/.gitmodules +++ b/.gitmodules @@ -66,6 +66,12 @@ [submodule "platform/mellanox/mlnx-sai/SAI-Implementation"] path = platform/mellanox/mlnx-sai/SAI-Implementation url = https://github.com/Mellanox/SAI-Implementation +[submodule "src/sonic-mgmt-framework"] + path = src/sonic-mgmt-framework + url = https://github.com/Azure/sonic-mgmt-framework +[submodule "src/sonic-telemetry"] + path = src/sonic-telemetry + url = https://github.com/Azure/sonic-telemetry [submodule "Switch-SDK-drivers"] path = platform/mellanox/sdk-src/sx-kernel/Switch-SDK-drivers url = https://github.com/Mellanox/Switch-SDK-drivers diff --git a/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 new file mode 100644 index 0000000000..08c819c1cc --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/Dockerfile.j2 @@ -0,0 +1,42 @@ +FROM docker-config-engine-stretch + +ARG docker_container_name +RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf + +## Make apt-get non-interactive +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update +RUN pip install connexion==1.1.15 \ + setuptools==21.0.0 \ + grpcio-tools==1.20.0 \ + pyangbind==0.6.0 \ + certifi==2017.4.17 \ + python-dateutil==2.6.0 \ + six==1.11.0 \ + urllib3==1.21.1 + + + +## Install redis-tools dependencies +## TODO: implicitly install dependencies +RUN apt-get -y install libjemalloc1 libatomic1 liblua5.1-0 lua-bitop lua-cjson + +COPY \ +{% for deb in docker_sonic_mgmt_framework_debs.split(' ') -%} +debs/{{ deb }}{{' '}} +{%- endfor -%} +debs/ + +RUN dpkg -i \ +{% for deb in docker_sonic_mgmt_framework_debs.split(' ') -%} +debs/{{ deb }}{{' '}} +{%- endfor %} + +COPY ["start.sh", "rest-server.sh", "/usr/bin/"] +COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] + +RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y +RUN rm -rf /debs + +ENTRYPOINT ["/usr/bin/supervisord"] diff --git a/dockers/docker-sonic-mgmt-framework/base_image_files/sonic-cli b/dockers/docker-sonic-mgmt-framework/base_image_files/sonic-cli new file mode 100755 index 0000000000..6675e3badb --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/base_image_files/sonic-cli @@ -0,0 +1,4 @@ +#!/bin/bash + +docker exec -it mgmt-framework /usr/sbin/cli/clish_start "$@" + diff --git a/dockers/docker-sonic-mgmt-framework/rest-server.sh b/dockers/docker-sonic-mgmt-framework/rest-server.sh new file mode 100755 index 0000000000..f2a29c9b1e --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/rest-server.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +# Startup script for SONiC Management REST Server + +SERVER_PORT= +LOG_LEVEL= +CLIENT_AUTH= +SERVER_CRT= +SERVER_KEY= +CA_CERT= + +# Read basic server settings from REST_SERVER|default entry +HAS_REST_CONFIG=$(sonic-cfggen -d -v "1 if REST_SERVER and REST_SERVER['default']") +if [ "$HAS_REST_CONFIG" == "1" ]; then + SERVER_PORT=$(sonic-cfggen -d -v "REST_SERVER['default']['port']") + CLIENT_AUTH=$(sonic-cfggen -d -v "REST_SERVER['default']['client_auth']") + LOG_LEVEL=$(sonic-cfggen -d -v "REST_SERVER['default']['log_level']") +fi + +# Read certificate file paths from DEVICE_METADATA|x509 entry. +HAS_X509_CONFIG=$(sonic-cfggen -d -v "1 if DEVICE_METADATA and DEVICE_METADATA['x509']") +if [ "$HAS_X509_CONFIG" == "1" ]; then + SERVER_CRT=$(sonic-cfggen -d -v "DEVICE_METADATA['x509']['server_crt']") + SERVER_KEY=$(sonic-cfggen -d -v "DEVICE_METADATA['x509']['server_key']") + CA_CRT=$(sonic-cfggen -d -v "DEVICE_METADATA['x509']['ca_crt']") +fi + +# Create temporary server certificate if they not configured in ConfigDB +if [ -z $SERVER_CRT ] && [ -z $SERVER_KEY ]; then + echo "Generating temporary TLS server certificate ..." + (cd /tmp && /usr/sbin/generate_cert --host="localhost,127.0.0.1") + SERVER_CRT=/tmp/cert.pem + SERVER_KEY=/tmp/key.pem +fi + + +REST_SERVER_ARGS="-ui /rest_ui -logtostderr" +[ ! -z $SERVER_PORT ] && REST_SERVER_ARGS+=" -port $SERVER_PORT" +[ ! -z $LOG_LEVEL ] && REST_SERVER_ARGS+=" -v $LOG_LEVEL" +[ ! -z $CLIENT_AUTH ] && REST_SERVER_ARGS+=" -client_auth $CLIENT_AUTH" +[ ! -z $SERVER_CRT ] && REST_SERVER_ARGS+=" -cert $SERVER_CRT" +[ ! -z $SERVER_KEY ] && REST_SERVER_ARGS+=" -key $SERVER_KEY" +[ ! -z $CA_CRT ] && REST_SERVER_ARGS+=" -cacert $CA_CRT" + +echo "REST_SERVER_ARGS = $REST_SERVER_ARGS" + + +export CVL_SCHEMA_PATH=/usr/sbin/schema +export LIBYANG_EXTENSIONS_PLUGINS_DIR=/usr/lib/x86_64-linux-gnu/libyang/extensions +export LIBYANG_USER_TYPES_PLUGINS_DIR=/usr/lib/x86_64-linux-gnu/libyang/user_types + +exec /usr/sbin/rest_server ${REST_SERVER_ARGS} diff --git a/dockers/docker-sonic-mgmt-framework/start.sh b/dockers/docker-sonic-mgmt-framework/start.sh new file mode 100755 index 0000000000..24d355670e --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/start.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +mkdir -p /var/sonic +echo "# Config files managed by sonic-config-engine" > /var/sonic/config_status + +rm -f /var/run/rsyslogd.pid + +supervisorctl start rsyslogd + +supervisorctl start rest-server diff --git a/dockers/docker-sonic-mgmt-framework/supervisord.conf b/dockers/docker-sonic-mgmt-framework/supervisord.conf new file mode 100644 index 0000000000..e26f815f5f --- /dev/null +++ b/dockers/docker-sonic-mgmt-framework/supervisord.conf @@ -0,0 +1,28 @@ +[supervisord] +logfile_maxbytes=1MB +logfile_backups=2 +nodaemon=true + +[program:start.sh] +command=/usr/bin/start.sh +priority=1 +autostart=true +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:rsyslogd] +command=/usr/sbin/rsyslogd -n +priority=2 +autostart=false +autorestart=false +stdout_logfile=syslog +stderr_logfile=syslog + +[program:rest-server] +command=/usr/bin/rest-server.sh +priority=3 +autostart=false +autorestart=true +stdout_logfile=syslog +stderr_logfile=syslog diff --git a/dockers/docker-sonic-telemetry/dialout.sh b/dockers/docker-sonic-telemetry/dialout.sh index 8683e8edae..485c3292d0 100755 --- a/dockers/docker-sonic-telemetry/dialout.sh +++ b/dockers/docker-sonic-telemetry/dialout.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Start with default config - +export CVL_SCHEMA_PATH=/usr/sbin/schema exec /usr/sbin/dialout_client_cli -insecure -logtostderr -v 2 diff --git a/dockers/docker-sonic-telemetry/supervisord.conf b/dockers/docker-sonic-telemetry/supervisord.conf index b6a01de58a..e1346fe7db 100644 --- a/dockers/docker-sonic-telemetry/supervisord.conf +++ b/dockers/docker-sonic-telemetry/supervisord.conf @@ -6,8 +6,8 @@ nodaemon=true [eventlistener:supervisor-proc-exit-listener] command=/usr/bin/supervisor-proc-exit-listener events=PROCESS_STATE_EXITED -autostart=always -autorestart=unexpected +autostart=true +autorestart=false [program:start.sh] command=/usr/bin/start.sh diff --git a/dockers/docker-sonic-telemetry/telemetry.sh b/dockers/docker-sonic-telemetry/telemetry.sh index 8cfd8a531c..8b29b4d616 100755 --- a/dockers/docker-sonic-telemetry/telemetry.sh +++ b/dockers/docker-sonic-telemetry/telemetry.sh @@ -6,6 +6,7 @@ X509=`sonic-cfggen -d -v "DEVICE_METADATA['x509']"` TELEMETRY=`sonic-cfggen -d -v 'TELEMETRY.keys() | join(" ") if TELEMETRY'` TELEMETRY_ARGS=" -logtostderr" +export CVL_SCHEMA_PATH=/usr/sbin/schema if [ -n "$X509" ]; then SERVER_CRT=`sonic-cfggen -d -v "DEVICE_METADATA['x509']['server_crt']"` diff --git a/files/build_templates/mgmt-framework.service.j2 b/files/build_templates/mgmt-framework.service.j2 new file mode 100644 index 0000000000..d0a030347b --- /dev/null +++ b/files/build_templates/mgmt-framework.service.j2 @@ -0,0 +1,14 @@ +[Unit] +Description=Management Framework container +Requires=swss.service +After=swss.service +Before=ntp-config.service + +[Service] +User={{ sonicadmin_user }} +ExecStartPre=/usr/bin/{{docker_container_name}}.sh start +ExecStart=/usr/bin/{{docker_container_name}}.sh wait +ExecStop=/usr/bin/{{docker_container_name}}.sh stop + +[Install] +WantedBy=multi-user.target diff --git a/files/image_config/platform/rc.local b/files/image_config/platform/rc.local index b74e9f40c4..d64ec1bb79 100755 --- a/files/image_config/platform/rc.local +++ b/files/image_config/platform/rc.local @@ -331,4 +331,7 @@ if [ -f $FIRST_BOOT_FILE ]; then firsttime_exit fi +# Create dir where following scripts put their output files +mkdir -p /var/platform + exit 0 diff --git a/rules/config b/rules/config index 04af59ccd8..b9dba045fa 100644 --- a/rules/config +++ b/rules/config @@ -98,3 +98,6 @@ DEFAULT_VS_PREPARE_MEM = yes # ENABLE_SYSTEM_SFLOW - build docker-sonic-sflow for sFlow support ENABLE_SFLOW = y + +# ENABLE_MGMT_FRAMEWORK - build docker-sonic-mgt-framework for CLI and REST server support +ENABLE_MGMT_FRAMEWORK = y diff --git a/rules/docker-sonic-mgmt-framework.mk b/rules/docker-sonic-mgmt-framework.mk new file mode 100644 index 0000000000..eb99f37875 --- /dev/null +++ b/rules/docker-sonic-mgmt-framework.mk @@ -0,0 +1,34 @@ +# docker image for mgmt-framework + +DOCKER_MGMT_FRAMEWORK_STEM = docker-sonic-mgmt-framework +DOCKER_MGMT_FRAMEWORK = $(DOCKER_MGMT_FRAMEWORK_STEM).gz +DOCKER_MGMT_FRAMEWORK_DBG = $(DOCKER_MGMT_FRAMEWORK_STEM)-$(DBG_IMAGE_MARK).gz + +$(DOCKER_MGMT_FRAMEWORK)_PATH = $(DOCKERS_PATH)/$(DOCKER_MGMT_FRAMEWORK_STEM) + +$(DOCKER_MGMT_FRAMEWORK)_DEPENDS += $(REDIS_TOOLS) $(SONIC_MGMT_FRAMEWORK) +$(DOCKER_MGMT_FRAMEWORK)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_DEPENDS) +$(DOCKER_MGMT_FRAMEWORK)_DBG_DEPENDS += $(REDIS_TOOLS) $(SONIC_MGMT_FRAMEWORK_DBG) + +SONIC_DOCKER_IMAGES += $(DOCKER_MGMT_FRAMEWORK) +$(DOCKER_MGMT_FRAMEWORK)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_STRETCH) +$(DOCKER_MGMT_FRAMEWORK)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES) + +ifeq ($(ENABLE_MGMT_FRAMEWORK), y) +SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_MGMT_FRAMEWORK) +SONIC_STRETCH_DOCKERS += $(DOCKER_MGMT_FRAMEWORK) +endif + +SONIC_DOCKER_DBG_IMAGES += $(DOCKER_MGMT_FRAMEWORK_DBG) +ifeq ($(ENABLE_MGMT_FRAMEWORK), y) +SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_MGMT_FRAMEWORK_DBG) +SONIC_STRETCH_DBG_DOCKERS += $(DOCKER_MGMT_FRAMEWORK_DBG) +endif + +$(DOCKER_MGMT_FRAMEWORK)_CONTAINER_NAME = mgmt-framework +$(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += --net=host --privileged -t +$(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += -v /etc:/host_etc:ro +$(DOCKER_MGMT_FRAMEWORK)_RUN_OPT += --mount type=bind,source="/var/platform/",target="/mnt/platform/" + +$(DOCKER_MGMT_FRAMEWORK)_BASE_IMAGE_FILES += sonic-cli:/usr/bin/sonic-cli diff --git a/rules/docker-telemetry.mk b/rules/docker-telemetry.mk index defeb4d008..799bef1b17 100644 --- a/rules/docker-telemetry.mk +++ b/rules/docker-telemetry.mk @@ -27,5 +27,6 @@ endif $(DOCKER_TELEMETRY)_CONTAINER_NAME = telemetry $(DOCKER_TELEMETRY)_RUN_OPT += --net=host --privileged -t $(DOCKER_TELEMETRY)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_TELEMETRY)_RUN_OPT += --mount type=bind,source="/var/platform/",target="/mnt/platform/" $(DOCKER_TELEMETRY)_FILES += $(SUPERVISOR_PROC_EXIT_LISTENER_SCRIPT) diff --git a/rules/sonic-mgmt-framework.mk b/rules/sonic-mgmt-framework.mk new file mode 100644 index 0000000000..a57ce6b1b0 --- /dev/null +++ b/rules/sonic-mgmt-framework.mk @@ -0,0 +1,16 @@ +# SONiC mgmt-framework package + +ifeq ($(ENABLE_MGMT_FRAMEWORK), y) + +SONIC_MGMT_FRAMEWORK = sonic-mgmt-framework_1.0-01_amd64.deb +$(SONIC_MGMT_FRAMEWORK)_SRC_PATH = $(SRC_PATH)/sonic-mgmt-framework +$(SONIC_MGMT_FRAMEWORK)_DEPENDS = $(LIBYANG_DEV) $(LIBYANG) +$(SONIC_MGMT_FRAMEWORK)_RDEPENDS = $(LIBYANG) +SONIC_DPKG_DEBS += $(SONIC_MGMT_FRAMEWORK) + +SONIC_MGMT_FRAMEWORK_DBG = sonic-mgmt-framework-dbg_1.0-01_amd64.deb +$(SONIC_MGMT_FRAMEWORK_DBG)_DEPENDS += $(SONIC_MGMT_FRAMEWORK) +$(SONIC_MGMT_FRAMEWORK_DBG)_RDEPENDS += $(SONIC_MGMT_FRAMEWORK) +$(eval $(call add_derived_package,$(SONIC_MGMT_FRAMEWORK),$(SONIC_MGMT_FRAMEWORK_DBG))) + +endif diff --git a/rules/telemetry.mk b/rules/telemetry.mk index 1d903e6032..0b4421b119 100644 --- a/rules/telemetry.mk +++ b/rules/telemetry.mk @@ -1,5 +1,7 @@ # SONiC telemetry package SONIC_TELEMETRY = sonic-telemetry_0.1_$(CONFIGURED_ARCH).deb -$(SONIC_TELEMETRY)_SRC_PATH = $(SRC_PATH)/telemetry +$(SONIC_TELEMETRY)_SRC_PATH = $(SRC_PATH)/sonic-telemetry +$(SONIC_TELEMETRY)_DEPENDS = $(LIBYANG_DEV) $(LIBYANG) +$(SONIC_TELEMETRY)_RDEPENDS = $(LIBYANG) SONIC_DPKG_DEBS += $(SONIC_TELEMETRY) diff --git a/sonic-slave-stretch/Dockerfile.j2 b/sonic-slave-stretch/Dockerfile.j2 index 31be4f50d4..17da5ba9cf 100644 --- a/sonic-slave-stretch/Dockerfile.j2 +++ b/sonic-slave-stretch/Dockerfile.j2 @@ -354,6 +354,9 @@ RUN pip install mockredispy==2.9.3 RUN pip install pytest-runner==4.4 RUN pip install setuptools==40.8.0 +# For mgmt-framework build +RUN pip install mmh3 + # Install dependencies for isc-dhcp-relay build RUN apt-get -y build-dep isc-dhcp diff --git a/src/sonic-mgmt-framework b/src/sonic-mgmt-framework new file mode 160000 index 0000000000..8b199a9f82 --- /dev/null +++ b/src/sonic-mgmt-framework @@ -0,0 +1 @@ +Subproject commit 8b199a9f822ca42564a7a89da0cab3133684bd12 diff --git a/src/sonic-telemetry b/src/sonic-telemetry new file mode 160000 index 0000000000..aaa9188fda --- /dev/null +++ b/src/sonic-telemetry @@ -0,0 +1 @@ +Subproject commit aaa9188fda4b6f36dd5da00cdb240933fedae5ce diff --git a/src/telemetry/Makefile b/src/telemetry/Makefile deleted file mode 100644 index 91a822d4f9..0000000000 --- a/src/telemetry/Makefile +++ /dev/null @@ -1,20 +0,0 @@ -export GOPATH=/tmp/go - -INSTALL := /usr/bin/install - -all: sonic-telemetry - -sonic-telemetry: - /usr/local/go/bin/go get -v github.com/Azure/sonic-telemetry/telemetry - /usr/local/go/bin/go get -v github.com/Azure/sonic-telemetry/dialout/dialout_client_cli - -install: - $(INSTALL) -D ${GOPATH}/bin/telemetry $(DESTDIR)/usr/sbin/telemetry - $(INSTALL) -D ${GOPATH}/bin/dialout_client_cli $(DESTDIR)/usr/sbin/dialout_client_cli - -deinstall: - rm $(DESTDIR)/usr/sbin/telemetry - rm $(DESTDIR)/usr/sbin/dialout_client_cli - -clean: - rm -fr ${GOPATH} diff --git a/src/telemetry/debian/changelog b/src/telemetry/debian/changelog deleted file mode 100644 index 77dab62904..0000000000 --- a/src/telemetry/debian/changelog +++ /dev/null @@ -1,5 +0,0 @@ -sonic-telemetry (0.1) UNRELEASED; urgency=medium - - * Initial release. - - -- Jipan Yang Sat, 24 Mar 2018 12:48:22 -0700 diff --git a/src/telemetry/debian/compat b/src/telemetry/debian/compat deleted file mode 100644 index ec635144f6..0000000000 --- a/src/telemetry/debian/compat +++ /dev/null @@ -1 +0,0 @@ -9 diff --git a/src/telemetry/debian/control b/src/telemetry/debian/control deleted file mode 100644 index f1b6a7c072..0000000000 --- a/src/telemetry/debian/control +++ /dev/null @@ -1,17 +0,0 @@ -Source: sonic-telemetry -Section: devel -Priority: optional -Maintainer: Jipan Yang -Build-Depends: debhelper (>= 8.0.0), - dh-systemd -Standards-Version: 3.9.3 -Homepage: https://github.com/Azure/sonic-telemetry -XS-Go-Import-Path: github.com/Azure/sonic-telemetry - -Package: sonic-telemetry -Architecture: any -Built-Using: ${misc:Built-Using} -Depends: ${misc:Depends}, - ${shlibs:Depends} -Description: SONiC telemetry - sonic-telemetry \ No newline at end of file diff --git a/src/telemetry/debian/rules b/src/telemetry/debian/rules deleted file mode 100755 index 3995a26d7f..0000000000 --- a/src/telemetry/debian/rules +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/make -f -%: - dh $@ --with systemd diff --git a/src/telemetry/debian/telemetry.init.d b/src/telemetry/debian/telemetry.init.d deleted file mode 100644 index 2fea32e17c..0000000000 --- a/src/telemetry/debian/telemetry.init.d +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -# -### BEGIN INIT INFO -# Provides: sonic-telemetry -# Required-Start: $local_fs $network $remote_fs $syslog -# Required-Stop: $local_fs $network $remote_fs $syslog -# Default-Start: 2 3 4 5 -# Default-Stop: 0 1 6 -# Short-Description: start and stop the telemetry -# Description: sonic-telemetry is an implementation of sonic telemetry daemon in Go -### END INIT INFO -# - -exit 0