Build python3.6 deb packages and install them into docker-snmp-sv2 (#245)

dockers/docker-snmp-sv2/Dockerfile.j2

@@ -1,45 +1,39 @@
 FROM docker-config-engine
 
-COPY \
+COPY [ \
 {% for deb in docker_snmp_sv2_debs.split(' ') -%}
-debs/{{ deb }}{{' '}}
+    "debs/{{ deb }}",
-{%- endfor -%}
+{%- endfor %} \
-debs/
+    "/debs/"]
 
-COPY python-wheels /python-wheels
+COPY python-wheels/sswsdk-*-py3-*.whl /python-wheels/
+COPY python-wheels/asyncsnmp-*-py3-*.whl /python-wheels/
 
 # enable -O for all Python calls
 ENV PYTHONOPTIMIZE 1
 
-## Pre-install the fundamental packages
+# install supervisor
-## Install Python SSWSDK (SNMP subagent dependency)
+# install libsnmp30 dependencies
-## Install SNMP subagent
+# install libpython3.6-dev dependencies
-## Clean up
+# install pip dependencies
-RUN apt-get update && apt-get install -y libmysqlclient-dev libmysqld-dev libperl-dev libpci-dev libpci3 libsensors4 libsensors4-dev libwrap0-dev
+# TODO: remove libpython3.6-dev, its and pip's dependencies if we can get pip3 directly
-
-RUN dpkg -i \
-{% for deb in docker_snmp_sv2_debs.split(' ') -%}
-debs/{{ deb }}{{' '}}
-{%- endfor %}
-
-RUN rm -rf /debs
-
 # install subagent
-RUN apt-get -y install build-essential wget libssl-dev openssl supervisor && \
+# clean up
-    rm -rf /var/lib/apt/lists/* && \
+RUN apt-get update && apt-get install -y supervisor \
-    wget https://www.python.org/ftp/python/3.5.2/Python-3.5.2.tgz && \
+            libperl5.20 libpci3 libwrap0 \
-    tar xvf Python-3.5.2.tgz && cd Python-3.5.2 && \
+            libexpat1-dev \
-    ./configure --without-doc-strings --prefix=/usr --without-pymalloc --enable-shared && \
+            curl gcc && \
-    make && make install && \
+    dpkg -i \
-    ldconfig && \
+{% for deb in docker_snmp_sv2_debs.split(' ') -%}
-    cd .. && rm -rf Python-3.5.2 && rm Python-3.5.2.tgz && \
+        debs/{{ deb }}{{' '}}
-    pip3 install --no-cache-dir /python-wheels/*py3*.whl hiredis && \
+{%- endfor %} && \
+    rm -rf /debs && \
+    curl https://bootstrap.pypa.io/get-pip.py | python3.6 && \
+    python3.6 -m pip install --no-cache-dir /python-wheels/*py3*.whl hiredis && \
     rm -rf /python-wheels && \
-    python3 -m sonic_ax_impl install && \
+    python3.6 -m sonic_ax_impl install && \
-    python3 -m pip uninstall -y pip setuptools && \
+    apt-get -y purge libpython3.6-dev libexpat1-dev curl gcc && \
-    /bin/bash -c "rm -rf /usr/lib/python3.5/{unittest,lib2to3,tkinter,idlelib,email,test}" && \
+    apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y --purge && \
-    apt-get -y purge build-essential libssl-dev openssl && \
-    apt-get clean -y && apt-get autoclean -y && apt-get autoremove -y && \
     find / | grep -E "__pycache__" | xargs rm -rf && \
     rm -rf ~/.cache
 

dockers/docker-snmp-sv2/supervisord.conf

@@ -6,7 +6,7 @@ command=/usr/sbin/snmpd -f -LS4d -u Debian-snmp -g Debian-snmp -I -smux,mteTrigg
 priority=100
 
 [program:snmp-subagent]
-command=/usr/bin/env python3 -m sonic_ax_impl
+command=/usr/bin/env python3.6 -m sonic_ax_impl
 priority=200
 
 [program:rsyslogd]

rules/docker-snmp-sv2.mk

@@ -2,7 +2,8 @@
 
 DOCKER_SNMP_SV2 = docker-snmp-sv2.gz
 $(DOCKER_SNMP_SV2)_PATH = $(DOCKERS_PATH)/docker-snmp-sv2
-$(DOCKER_SNMP_SV2)_DEPENDS += $(SNMP) $(SNMPD)
+## TODO: remove LIBPY3_DEV if we can get pip3 directly
+$(DOCKER_SNMP_SV2)_DEPENDS += $(SNMP) $(SNMPD) $(PY3) $(LIBPY3_DEV)
 $(DOCKER_SNMP_SV2)_PYTHON_WHEELS += $(ASYNCSNMP_PY3)
 $(DOCKER_SNMP_SV2)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE)
 SONIC_DOCKER_IMAGES += $(DOCKER_SNMP_SV2)

rules/python3.mk

@@ -0,0 +1,36 @@
+PYTHON_VER=3.6.0-1
+PYTHON_PNAME=python3.6
+
+export PYTHON_VER
+export PYTHON_PNAME
+
+LIBPY3_MIN = lib$(PYTHON_PNAME)-minimal_$(PYTHON_VER)_amd64.deb
+$(LIBPY3_MIN)_SRC_PATH = $(SRC_PATH)/python3
+$(LIBPY3_MIN)_DEPENDS += 
+$(LIBPY3_MIN)_RDEPENDS += 
+SONIC_MAKE_DEBS += $(LIBPY3_MIN)
+
+LIBPY3_STD = lib$(PYTHON_PNAME)-stdlib_$(PYTHON_VER)_amd64.deb
+$(eval $(call add_derived_package,$(LIBPY3_MIN),$(LIBPY3_STD)))
+$(LIBPY3_STD)_DEPENDS += $(LIBMPDECIMAL)
+$(LIBPY3_STD)_RDEPENDS += $(LIBPY3_MIN) $(LIBMPDECIMAL)
+
+LIBPY3 = lib$(PYTHON_PNAME)_$(PYTHON_VER)_amd64.deb
+$(eval $(call add_derived_package,$(LIBPY3_MIN),$(LIBPY3)))
+$(LIBPY3)_DEPENDS += $(LIBPY3_STD)
+$(LIBPY3)_RDEPENDS += $(LIBPY3_MIN) $(LIBPY3_STD)
+
+PY3_MIN = $(PYTHON_PNAME)-minimal_$(PYTHON_VER)_amd64.deb
+$(eval $(call add_derived_package,$(LIBPY3_MIN),$(PY3_MIN)))
+$(PY3_MIN)_RDEPENDS += $(LIBPY3_MIN)
+
+PY3 = $(PYTHON_PNAME)_$(PYTHON_VER)_amd64.deb
+$(eval $(call add_derived_package,$(LIBPY3_MIN),$(PY3)))
+$(PY3)_DEPENDS += $(PY3_MIN) $(LIBPY3)
+$(PY3)_RDEPENDS += $(PY3_MIN) $(LIBPY3) $(LIBPY3_MIN)
+
+LIBPY3_DEV = lib$(PYTHON_PNAME)-dev_$(PYTHON_VER)_amd64.deb
+$(eval $(call add_derived_package,$(LIBPY3_MIN),$(LIBPY3_DEV)))
+$(LIBPY3_DEV)_DEPENDS += $(LIBPY3) $($(LIBPY3)_DEPENDS)
+$(LIBPY3_DEV)_RDEPENDS += $(LIBPY3) $($(LIBPY3)_RDEPENDS)
+

src/python3.5/build.sh

@@ -1,18 +0,0 @@
-#!/bin/bash
-
-PYTHON_VER=3.5.2
-PYTHON_DEB_VER=8
-
-wget -N http://http.debian.net/debian/pool/main/p/python3.5/python3.5_${PYTHON_VER}.orig.tar.xz
-wget -N http://http.debian.net/debian/pool/main/p/python3.5/python3.5_${PYTHON_VER}-${PYTHON_DEB_VER}.debian.tar.xz
-wget -N http://http.debian.net/debian/pool/main/p/python3.5/python3.5_${PYTHON_VER}-${PYTHON_DEB_VER}.dsc
-
-dpkg-source -x python3.5_${PYTHON_VER}-${PYTHON_DEB_VER}.dsc
-
-pushd python3.5-${PYTHON_VER}
-
-dpkg-buildpackage -us -uc -b
-
-popd
-
-cp *.deb ../

src/python3/Makefile

@@ -0,0 +1,34 @@
+.ONESHELL:
+SHELL = /bin/bash
+.SHELLFLAGS += -e
+
+PYTHON_VER=3.6.0
+PYTHON_DEB_VER=1
+PYTHON_PNAME=python3.6
+
+MAIN_TARGET = lib$(PYTHON_PNAME)-minimal_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb
+DERIVED_TARGETS = lib$(PYTHON_PNAME)-stdlib_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb \
+				  lib$(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb \
+				  $(PYTHON_PNAME)-minimal_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb \
+				  $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb \
+				  lib$(PYTHON_PNAME)-dev_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb
+				  #$(PYTHON_PNAME)-dev_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb
+
+$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
+	## Obtaining the python3
+	wget 'https://sonicstorage.blob.core.windows.net/packages/$(PYTHON_PNAME)_$(PYTHON_VER).orig.tar.xz?sv=2015-04-05&sr=b&sig=d42Wh1CA9NZvlskhW4fpWcHVgc7N3IKhdFzyeO2zbRA%3D&se=2027-02-02T01%3A00%3A57Z&sp=r' -O $(PYTHON_PNAME)_$(PYTHON_VER).orig.tar.xz
+	wget 'https://sonicstorage.blob.core.windows.net/packages/$(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).debian.tar.xz?sv=2015-04-05&sr=b&sig=KLX9pMJ3zpQvGBo6ZjzoZXgooMJRUUwMx8ZaTJtywK0%3D&se=2027-02-02T00%3A59%3A34Z&sp=r' -O $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).debian.tar.xz
+	wget 'https://sonicstorage.blob.core.windows.net/packages/$(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc?sv=2015-04-05&sr=b&sig=95s%2FC4vKY6bRKtkUTz%2BmHLqOllBOYbfP3zV5ayAuzSM%3D&se=2027-02-02T01%3A00%3A26Z&sp=r' -O $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc
+
+	## Build
+	dpkg-source -x $(PYTHON_PNAME)_$(PYTHON_VER)-$(PYTHON_DEB_VER).dsc
+	pushd $(PYTHON_PNAME)-$(PYTHON_VER)
+	sudo apt-get install devscripts
+	mk-build-deps
+	sudo dpkg -i $(PYTHON_PNAME)-build-deps_$(PYTHON_VER)-$(PYTHON_DEB_VER)_amd64.deb || sudo apt-get install -f
+	dpkg-buildpackage -us -uc -b
+	popd
+
+	cp $(DERIVED_TARGETS) $* $(DEST)/
+
+$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET)