Squashed merge master

.gitignore

@@ -1,2 +1,14 @@
 deps/
-
+dockers/*/files/
+*.bin
+*.deb
+*.zip
+fsroot/
+fs.*
+src/hiredis/hiredis_*
+src/hiredis/hiredis-*/
+src/quagga/quagga_*
+src/quagga/quagga/
+src/initramfs-tools/initramfs-tools/
+src/redis/redis_*
+src/redis/redis-*/

README.md

@@ -39,7 +39,8 @@ The target directory is ./target, containing the NOS installer image and docker
 - docker-base.gz: base docker image where others are built from (gzip tar archive)
 - docker-fpm.gz: docker image for quagga with fpm module enabled (gzip tar archive)
 - docker-orchagent.gz: docker image for SWitch State Service (SWSS)
-- docker-syncd.gz: docker image for the daemon to sync database and switch ASIC
+- docker-syncd.gz: docker image for the daemon to sync database and Broadcom switch ASIC
+- docker-syncd-mlnx.gz: docker image for the daemon to sync database and Mellanox switch ASIC
 
 # Contribution guide
 

build_docker.sh

@@ -9,7 +9,7 @@ set -e
 usage() {
     cat >&2 <<EOF
 Usage:
-  sudo ./build_docker.sh -i=DOCKER_IMAGE_NAME DOCKER_BUILD_DIR [REGISTRY_SERVER REGISTRY_PORT]
+  sudo ./build_docker.sh -i=DOCKER_IMAGE_NAME DOCKER_BUILD_DIR [REGISTRY_SERVER REGISTRY_PORT REGISTRY_USERNAME REGISTRY_PASSWD]
   
 Description:
   -i DOCKER_IMAGE_NAME
@@ -125,4 +125,5 @@ fi
 
 mkdir -p target
 rm -f target/$docker_image_name.*.gz
-docker save $docker_image_name | gzip -c > target/$docker_image_name.$image_sha.gz
+docker save $docker_image_name | gzip -c > target/$docker_image_name.$build_version.gz
+echo "Image sha256: $image_sha"

dockers/docker-lldp/Dockerfile

@@ -10,23 +10,12 @@ COPY deps/*py2*.whl deps/lldpsyncd_*.deb deps/lldpd_*.deb /deps/
 RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; } &&        \
     dpkg_apt /deps/lldpd_*.deb &&                                                       \
     dpkg_apt /deps/lldpsyncd_*.deb &&                                                           \
-    apt-get install -y python-pip && \
+    apt-get install -y python-pip supervisor && \
     pip install /deps/*.whl && \
     apt-get remove -y python-pip && \
-    apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y &&                        \
+    apt-get purge -y && apt-get autoclean -y && apt-get autoremove -y &&                        \
     rm -rf /deps
 
-## There is a known bug: agetty processes at 100% cpu
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
-## When:
-##   1. running container in --privileged mode
-##   2. container runs /sbin/init
-## ref: https://github.com/docker/docker/issues/4040
-## Temporary solution:
-##   Disable tty services permanently
-RUN systemctl --no-pager list-unit-files --type=service | grep getty | awk '{print $1}' | xargs systemctl mask
-## Note: getty@.service in last grep output will not mask below cases
-RUN systemctl mask getty@tty1.service
 
-## Specify init as CMD to enable systemd
+ENTRYPOINT ["/usr/bin/supervisord"]
-## Note: don't provide ENTRYPOINT at the same time
-CMD ["/sbin/init"]

dockers/docker-lldp/supervisord.conf

@@ -0,0 +1,16 @@
+[supervisord]
+nodaemon=true
+
+[program:lldpd]
+## -d: Do not daemonize.  If this option is specified, lldpd will run in
+##     the foreground and log to stderr.  This option can be specified
+##     many times to increase verbosity
+command=/usr/sbin/lldpd -dd
+## route stderr to the syslog service instead of being saved to files
+stderr_logfile=syslog
+
+[program:lldpsyncd]
+command=/usr/sbin/lldpsyncd
+
+[program:rsyslogd]
+command=/usr/sbin/rsyslogd -n

dockers/docker-orchagent/Dockerfile

@@ -11,7 +11,7 @@ RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return
 RUN apt-get install -f -y ifupdown bridge-utils
 
 ## Copy executable binaries
-COPY ["deps/orchagent","deps/swssconfig","deps/portsyncd","deps/intfsyncd","deps/neighsyncd","/usr/local/bin/"]
+COPY ["deps/orchagent","deps/swssconfig","deps/portsyncd","deps/intfsyncd","deps/neighsyncd","/usr/bin/"]
 
 COPY start.sh /usr/bin/start.sh
 
@@ -19,5 +19,5 @@ COPY start.sh /usr/bin/start.sh
 RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
 RUN rm -rf /deps
 
-ENTRYPOINT /usr/bin/start.sh    \
+ENTRYPOINT ["/bin/bash"]
-        && /bin/bash
+CMD ["/usr/bin/start.sh"]

dockers/docker-orchagent/start.sh

@@ -1,5 +1,16 @@
 #!/bin/bash
 
+function clean_up {
+    kill -9 $ORCHAGENT_PID
+    kill -9 $PORTSYNCD_PID
+    kill -9 $INTFSYNCD_PID
+    kill -9 $NEIGHSYNCD_PID
+    service rsyslog stop
+    exit
+}
+
+trap clean_up SIGTERM SIGKILL
+
 . /host/machine.conf
 
 MAC_ADDRESS=`ip link show eth0 | grep ether | awk '{print $2}'`
@@ -17,9 +28,15 @@ fi
 
 service rsyslog start
 orchagent $ORCHAGENT_ARGS &
+ORCHAGENT_PID=$!
 sleep 5
 portsyncd $PORTSYNCD_ARGS &
+PORTSYNCD_PID=$!
 sleep 5
 intfsyncd &
+INTFSYNCD_PID=$!
 sleep 5
 neighsyncd &
+NEIGHSYNCD_PID=$!
+
+read

dockers/docker-ptf/Dockerfile

@@ -9,11 +9,11 @@ COPY deps /root/deps
 ENV DEBIAN_FRONTEND=noninteractive
 
 ## Set the apt source, update package cache and install necessary packages
-RUN echo "deb http://ftp.us.debian.org/debian/ jessie main contrib non-free" > /etc/apt/sources.list \
+RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' /etc/apt/sources.list \
-    && apt-get update           \
+    && apt-get update          \
-    && apt-get upgrade -y       \
+    && apt-get upgrade -y   \
     && apt-get dist-upgrade -y  \
-    && apt-get install -y --no-install-recommends \
+    && apt-get install -y   \
         openssh-server      \
         vim                 \
         python              \
@@ -47,14 +47,15 @@ RUN echo "deb http://ftp.us.debian.org/debian/ jessie main contrib non-free" > /
     && tar xvfz 1.0.0.tar.gz \
     && cd nanomsg-1.0.0    \
     && mkdir -p build      \
-    && cmake .             \
+    && cd build            \
+    && cmake ..            \
     && make install        \
     && ldconfig            \
-    && cd ..               \
+    && cd ../..            \
     && rm -fr nanomsg-1.0.0 \
     && rm -f 1.0.0.tar.gz  \
-    && pip install cffi    \
+    && pip install cffi==1.7.0    \
-    && pip install --upgrade cffi \
+    && pip install --upgrade cffi==1.7.0 \
     && pip install nnpy    \
     && mkdir -p /opt       \
     && cd /opt             \
@@ -69,7 +70,10 @@ RUN mkdir /var/run/sshd \
  && mkdir /root/deps
 
 COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+COPY sshd.conf /etc/supervisor/conf.d/sshd.conf
+COPY ptf_nn_agent.conf /etc/supervisor/conf.d/ptf_nn_agent.conf
 
 EXPOSE 22
 
-ENTRYPOINT ["/usr/bin/supervisord"]
+ENTRYPOINT while [ "x$( ifconfig eth3 | grep RUNNING )" = 'x' ] ; do sleep 1 ; done \
+           && /usr/bin/supervisord

dockers/docker-ptf/ptf_nn_agent.conf

@@ -0,0 +1,10 @@
+[program:ptf_nn_agent]
+command=/usr/bin/python /opt/ptf_nn_agent.py --device-socket 0@tcp://127.0.0.1:10900 -i 0-3@eth3
+process_name=ptf_nn_agent
+stdout_logfile=/tmp/ptf_nn_agent.out.log
+stderr_logfile=/tmp/ptf_nn_agent.err.log
+redirect_stderr=false
+autostart=true
+autorestart=true
+startsecs=1
+numprocs=1

dockers/docker-ptf/sshd.conf

@@ -0,0 +1,10 @@
+[program:sshd]
+command=/usr/sbin/sshd -D
+process_name=sshd
+stdout_logfile=/tmp/sshd.out.log
+stderr_logfile=/tmp/sshd.err.log
+redirect_stderr=false
+autostart=true
+autorestart=true
+startsecs=1
+numprocs=1

dockers/docker-ptf/supervisord.conf

@@ -1,13 +1,2 @@
 [supervisord]
 nodaemon=true
-
-[program:sshd]
-command=/usr/sbin/sshd -D
-process_name=sshd
-stdout_logfile=/tmp/sshd.out.log
-stderr_logfile=/tmp/sshd.err.log
-redirect_stderr=false
-autostart=true
-autorestart=true
-startsecs=1
-numprocs=1

dockers/docker-snmp/Dockerfile

@@ -19,7 +19,7 @@ RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return
     rm -rf /deps 
 
 # install subagent
-RUN apt-get -y install build-essential wget libssl-dev openssl && \
+RUN apt-get -y install build-essential wget libssl-dev openssl supervisor && \
     rm -rf /var/lib/apt/lists/* && \
     wget https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz && \
     tar xvf Python-3.5.2.tgz && cd Python-3.5.2 && \
@@ -36,27 +36,14 @@ RUN apt-get -y install build-essential wget libssl-dev openssl && \
     rm -rf /usr/lib/python3.5/idlelib && \
     rm -rf /usr/lib/python3.5/email && \
     rm -rf /usr/lib/python3.5/test && \
-    apt-get -y remove build-essential wget libssl-dev openssl && \
+    apt-get -y purge build-essential wget libssl-dev openssl && \
     apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y && \
-    python3 -m acs_ax_impl install && \
     find / | grep -E "__pycache__" | xargs rm -rf && \
-    rm -rf ~/.cache && \
+    rm -rf ~/.cache
-    systemctl enable acs-snmp-subagent.service
 
-## There is a known bug: agetty processes at 100% cpu
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
-## When:
-##   1. running container in --privileged mode
-##   2. container runs /sbin/init
-## ref: https://github.com/docker/docker/issues/4040
-## Temporary solution:
-##   Disable tty services permanently
-RUN systemctl --no-pager list-unit-files --type=service | grep getty | awk '{print $1}' | xargs systemctl mask
-## Note: getty@.service in last grep output will not mask below cases
-RUN systemctl mask getty@tty1.service
 
 ## Although exposing ports is not need for host net mode, keep it for possible bridge mode
 EXPOSE 161/udp 162/udp
 
-## Specify init as CMD to enable systemd
+ENTRYPOINT ["/usr/bin/supervisord"]
-## Note: don't provide ENTRYPOINT at the same time
-CMD ["/sbin/init"]

dockers/docker-snmp/supervisord.conf

@@ -0,0 +1,11 @@
+[supervisord]
+nodaemon=true
+
+[program:snmpd]
+command=/usr/sbin/snmpd -f -LS4d -u Debian-snmp -g Debian-snmp -I -smux,mteTrigger,mteTriggerConf,ifTable,ifXTable -p /run/snmpd.pid
+
+[program:acs-snmp-subagent]
+command=/usr/bin/env python3 -m acs_ax_impl
+
+[program:rsyslogd]
+command=/usr/sbin/rsyslogd -n

dockers/docker-syncd-mlnx-rpc/Dockerfile

@@ -3,6 +3,8 @@ FROM docker-syncd-mlnx
 ## Make apt-get non-interactive
 ENV DEBIAN_FRONTEND=noninteractive
 
+COPY deps /deps
+
 ## Pre-install the fundamental packages
 RUN apt-get update \
  && apt-get -y install  \
@@ -23,24 +25,24 @@ RUN apt-get update \
  && ldconfig            \
  && cd ..               \
  && rm -fr nanomsg-1.0.0 \
+ && rm -f 1.0.0.tar.gz  \
  && pip install cffi    \
  && pip install --upgrade cffi \
  && pip install nnpy    \
  && mkdir -p /opt       \
  && cd /opt             \
- && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py
+ && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py \
+ && dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; } \
+ && dpkg -r syncd \
+ && dpkg_apt /deps/syncd_*.deb \
+ && dpkg_apt /deps/libthrift-0.9.2_*.deb \
+ && apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y \
+ && rm -rf /deps
 
-COPY deps /deps
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
-
+COPY ptf_nn_agent.conf /etc/supervisor/conf.d/ptf_nn_agent.conf
-RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; } \
-    && dpkg -r syncd \
-    && dpkg_apt /deps/syncd_*.deb \
-    && dpkg_apt /deps/libthrift-0.9.2_*.deb
-
-## Clean up
-RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
-RUN rm -rf /deps
 
 ENTRYPOINT service rsyslog start \
     && service syncd start       \
-    && /bin/bash
+    && while [ "x$( ifconfig Ethernet12 | grep RUNNING )" = 'x' ] ; do sleep 1 ; done \
+    && /usr/bin/supervisord

dockers/docker-syncd-mlnx-rpc/ptf_nn_agent.conf

@@ -0,0 +1,10 @@
+[program:ptf_nn_agent]
+command=/usr/bin/python /opt/ptf_nn_agent.py --device-socket 1@tcp://0.0.0.0:10900   -i 1-3@Ethernet12
+process_name=ptf_nn_agent
+stdout_logfile=/tmp/ptf_nn_agent.out.log
+stderr_logfile=/tmp/ptf_nn_agent.err.log
+redirect_stderr=false
+autostart=true
+autorestart=true
+startsecs=1
+numprocs=1

dockers/docker-syncd-mlnx-rpc/supervisord.conf

@@ -0,0 +1,2 @@
+[supervisord]
+nodaemon=true

dockers/docker-syncd-mlnx/Dockerfile

@@ -24,10 +24,11 @@ RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return
 
 RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; dpkg_apt /deps/syncd_*.deb
 
+COPY ["start.sh", "/usr/bin/"]
+
 ## Clean up
 RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
 RUN rm -rf /deps
 
-ENTRYPOINT service rsyslog start \
+ENTRYPOINT ["/bin/bash"]
-    && service syncd start       \
+CMD ["/usr/bin/start.sh"]
-    && /bin/bash

dockers/docker-syncd-mlnx/start.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+function clean_up {
+    service syncd stop
+    service rsyslog stop
+    exit
+}
+
+trap clean_up SIGTERM SIGKILL
+
+service rsyslog start
+service syncd start
+
+read

dockers/docker-syncd-rpc/Dockerfile

@@ -0,0 +1,48 @@
+FROM docker-syncd
+
+## Make apt-get non-interactive
+ENV DEBIAN_FRONTEND=noninteractive
+
+COPY deps /deps
+
+## Pre-install the fundamental packages
+RUN apt-get update \
+ && apt-get -y install  \
+    net-tools           \
+    python-pip          \
+    build-essential     \
+    libssl-dev          \
+    libffi-dev          \
+    python-dev          \
+    wget                \
+    cmake               \
+ && wget https://github.com/nanomsg/nanomsg/archive/1.0.0.tar.gz \
+ && tar xvfz 1.0.0.tar.gz \
+ && cd nanomsg-1.0.0    \
+ && mkdir -p build      \
+ && cmake .             \
+ && make install        \
+ && ldconfig            \
+ && cd ..               \
+ && rm -fr nanomsg-1.0.0 \
+ && rm -f 1.0.0.tar.gz  \
+ && pip install cffi    \
+ && pip install --upgrade cffi \
+ && pip install nnpy    \
+ && mkdir -p /opt       \
+ && cd /opt             \
+ && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py \
+ && dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; } \
+ && dpkg -r syncd \
+ && dpkg_apt /deps/syncd_*.deb \
+ && dpkg_apt /deps/libthrift-0.9.2_*.deb \
+ && apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y \
+ && rm -rf /deps
+
+COPY supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+COPY ptf_nn_agent.conf /etc/supervisor/conf.d/ptf_nn_agent.conf
+
+ENTRYPOINT service rsyslog start \
+    && service syncd start       \
+    && while [ "x$( ifconfig Ethernet12 | grep RUNNING )" = 'x' ] ; do sleep 1 ; done \
+    && /usr/bin/supervisord

dockers/docker-syncd-rpc/ptf_nn_agent.conf

@@ -0,0 +1,10 @@
+[program:ptf_nn_agent]
+command=/usr/bin/python /opt/ptf_nn_agent.py --device-socket 1@tcp://0.0.0.0:10900   -i 1-3@Ethernet12
+process_name=ptf_nn_agent
+stdout_logfile=/tmp/ptf_nn_agent.out.log
+stderr_logfile=/tmp/ptf_nn_agent.err.log
+redirect_stderr=false
+autostart=true
+autorestart=true
+startsecs=1
+numprocs=1

dockers/docker-syncd-rpc/supervisord.conf

@@ -0,0 +1,2 @@
+[supervisord]
+nodaemon=true

dockers/docker-syncd/Dockerfile

@@ -21,5 +21,5 @@ COPY ["deps/dsserve", "deps/bcmcmd", "start.sh", "/usr/bin/"]
 RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y
 RUN rm -rf /deps
 
-ENTRYPOINT /usr/bin/start.sh      \
+ENTRYPOINT ["/bin/bash"]
-        && /bin/bash
+CMD ["/usr/bin/start.sh"]

dockers/docker-syncd/start.sh

@@ -1,4 +1,14 @@
 #!/bin/bash
 
+function clean_up {
+    service syncd stop
+    service rsyslog stop
+    exit
+}
+
+trap clean_up SIGTERM SIGKILL
+
 service rsyslog start
 service syncd start
+
+read

installer/x86_64/install.sh

@@ -51,15 +51,7 @@ CONSOLE_SPEED=9600
 
 # Get platform specific linux kernel command line arguments
 ONIE_PLATFORM_EXTRA_CMDLINE_LINUX=""
-# platform specific configurations
+source platforms/$onie_platform
-if [ "$onie_platform" == "x86_64-dell_s6000_s1220-r0" ]; then
-    `pwd`/dell-s6000-replace-reboot.sh
-elif [ "$onie_platform" == "x86_64-mlnx_x86-r5.0.1400" ]; then
-    ONIE_PLATFORM_EXTRA_CMDLINE_LINUX="acpi_enforce_resources=lax acpi=noirq"
-elif [ "$onie_platform" == "x86_64-dell_s6100_c2538-r0" ]; then
-    CONSOLE_PORT=0x2f8
-    CONSOLE_DEV=1
-fi
 
 # Install demo on same block device as ONIE
 onie_dev=$(blkid | grep ONIE-BOOT | head -n 1 | awk '{print $1}' |  sed -e 's/:.*$//')
@@ -362,7 +354,7 @@ demo_install_uefi_grub()
     grub_install_log=$(mktemp)
     grub-install \
         --no-nvram \
-        --bootloader-id="$onie_initrd_tmp/$demo_volume_label" \
+        --bootloader-id="$demo_volume_label" \
         --efi-directory="/boot/efi" \
         --boot-directory="$demo_mnt" \
         --recheck \

installer/x86_64/platforms/x86_64-dell_s6000_s1220-r0

@@ -0,0 +1,2 @@
+ONIE_PLATFORM_EXTRA_CMDLINE_LINUX="processor.max_cstate=1 intel_idle.max_cstate=0"
+./dell-s6000-replace-reboot.sh

installer/x86_64/platforms/x86_64-dell_s6100_c2538-r0

@@ -0,0 +1,2 @@
+CONSOLE_PORT=0x2f8
+CONSOLE_DEV=1

installer/x86_64/platforms/x86_64-mlnx_x86-r5.0.1400

@@ -0,0 +1 @@
+ONIE_PLATFORM_EXTRA_CMDLINE_LINUX="acpi_enforce_resources=lax acpi=noirq"

onie-mk-demo.sh

@@ -76,7 +76,7 @@ tmp_dir=$(mktemp --directory)
 tmp_installdir="$tmp_dir/installer"
 mkdir $tmp_installdir || clean_up 1
 
-cp $installer_dir/$arch/* $tmp_installdir || clean_up 1
+cp -r $installer_dir/$arch/* $tmp_installdir || clean_up 1
 cp onie-image.conf $tmp_installdir
 
 # Escape special chars in the user provide kernel cmdline string for use in