From d2733917732fd8c73239f50407046604f03c838d Mon Sep 17 00:00:00 2001
From: Tobias Genannt <tobias.genannt@gmail.com>
Date: Tue, 10 Nov 2020 15:23:07 +0100
Subject: [PATCH] Gunicorn is replaced with nginx-unit

We now serve Netbox with an nginx-unit instance instead of Gunicorn.
This allows us to get rid of the extra Nginx container because Unit is
also serving the static files. The static files are now collected at container
buildtime instead of every startup.
---
 .github/ISSUE_TEMPLATE/bug_report.md | 10 ------
 .github/workflows/push.yml           |  2 +-
 Dockerfile                           | 48 +++++++++++--------------
 README.md                            |  2 +-
 build.sh                             | 14 ++++----
 docker-compose.test.yml              | 14 --------
 docker-compose.yml                   | 23 +++---------
 docker/configuration.docker.py       |  1 +
 docker/docker-entrypoint.sh          |  6 ++--
 docker/gunicorn_config.py            |  8 -----
 docker/launch-netbox.sh              | 53 ++++++++++++++++++++++++++++
 docker/nginx-unit.json               | 40 +++++++++++++++++++++
 docker/nginx.conf                    | 44 -----------------------
 requirements-container.txt           |  4 +++
 test.sh                              |  2 +-
 15 files changed, 135 insertions(+), 136 deletions(-)
 delete mode 100644 docker/gunicorn_config.py
 create mode 100755 docker/launch-netbox.sh
 create mode 100644 docker/nginx-unit.json
 delete mode 100644 docker/nginx.conf
 create mode 100644 requirements-container.txt

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 82f22b1..7bf0632 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -65,13 +65,3 @@ If your log is very long, create a Gist instead (and post the link to it): https
 ```text
 LOG LOG LOG
 ```
-
-The output of `docker-compose logs nginx`:
-<!--
-Only if you have gotten a 5xx http error, else delete this section.
-If your log is very long, create a Gist instead (and post the link to it): https://gist.github.com
--->
-
-```text
-LOG LOG LOG
-```
diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml
index e1780b7..09bc661 100644
--- a/.github/workflows/push.yml
+++ b/.github/workflows/push.yml
@@ -19,7 +19,7 @@ jobs:
         - ./build.sh develop
         docker_from:
         - '' # use the default of the build script
-        # - python:3.10-rc-alpine # disable until dependencies work
+        - alpine:edge
       fail-fast: false
     runs-on: ubuntu-latest
     name: Builds new Netbox Docker Images
diff --git a/Dockerfile b/Dockerfile
index 8db4cac..272f882 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -12,27 +12,15 @@ RUN apk add --no-cache \
       libffi-dev \
       libxslt-dev \
       openldap-dev \
-      postgresql-dev
-
-WORKDIR /install
-
-RUN pip install --prefix="/install" --no-warn-script-location \
-# gunicorn is used for launching netbox
-      gunicorn \
-      greenlet \
-      eventlet \
-# napalm is used for gathering information from network devices
-      napalm \
-# ruamel is used in startup_scripts
-      'ruamel.yaml>=0.15,<0.16' \
-# django_auth_ldap is required for ldap
-      django_auth_ldap \
-# django-storages was introduced in 2.7 and is optional
-      django-storages
+      postgresql-dev \
+      py3-pip \
+      python3-dev \
+      && python3 -m venv /opt/netbox/venv \
+      && /opt/netbox/venv/bin/python3 -m pip install --upgrade pip setuptools
 
 ARG NETBOX_PATH
-COPY ${NETBOX_PATH}/requirements.txt /
-RUN pip install --prefix="/install" --no-warn-script-location -r /requirements.txt
+COPY ${NETBOX_PATH}/requirements.txt requirements-container.txt /
+RUN /opt/netbox/venv/bin/pip install -r /requirements.txt -r /requirements-container.txt
 
 ###
 # Main stage
@@ -44,6 +32,7 @@ FROM ${FROM} as main
 RUN apk add --no-cache \
       bash \
       ca-certificates \
+      curl \
       graphviz \
       libevent \
       libffi \
@@ -51,35 +40,38 @@ RUN apk add --no-cache \
       libressl \
       libxslt \
       postgresql-libs \
-      ttf-ubuntu-font-family
+      python3 \
+      py3-pip \
+      ttf-ubuntu-font-family \
+      unit \
+      unit-python3
 
 WORKDIR /opt
 
-COPY --from=builder /install /usr/local
+COPY --from=builder /opt/netbox/venv /opt/netbox/venv
 
 ARG NETBOX_PATH
 COPY ${NETBOX_PATH} /opt/netbox
 
 COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
-COPY docker/gunicorn_config.py /etc/netbox/
-COPY docker/nginx.conf /etc/netbox-nginx/nginx.conf
 COPY docker/docker-entrypoint.sh /opt/netbox/docker-entrypoint.sh
+COPY docker/launch-netbox.sh /opt/netbox/launch-netbox.sh
 COPY startup_scripts/ /opt/netbox/startup_scripts/
 COPY initializers/ /opt/netbox/initializers/
 COPY configuration/ /etc/netbox/config/
+COPY docker/nginx-unit.json /etc/unit/
 
 WORKDIR /opt/netbox/netbox
 
-# Must set permissions for '/opt/netbox/netbox/static' directory
-# to g+w so that `./manage.py collectstatic` can be executed during
-# container startup.
 # Must set permissions for '/opt/netbox/netbox/media' directory
 # to g+w so that pictures can be uploaded to netbox.
-RUN mkdir static && chmod -R g+w static media
+RUN mkdir -p static /opt/unit/state/ /opt/unit/tmp/ \
+      && chmod -R g+w media /opt/unit/ \
+      && SECRET_KEY="dummy" /opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py collectstatic --no-input
 
 ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
 
-CMD ["gunicorn", "-c /etc/netbox/gunicorn_config.py", "netbox.wsgi"]
+CMD [ "/opt/netbox/launch-netbox.sh" ]
 
 LABEL ORIGINAL_TAG="" \
       NETBOX_GIT_BRANCH="" \
diff --git a/README.md b/README.md
index 11edf52..0125c29 100644
--- a/README.md
+++ b/README.md
@@ -56,7 +56,7 @@ cd netbox-docker
 tee docker-compose.override.yml <<EOF
 version: '3.4'
 services:
-  nginx:
+  netbox:
     ports:
       - 8000:8080
 EOF
diff --git a/build.sh b/build.sh
index 19c56c8..897f74e 100755
--- a/build.sh
+++ b/build.sh
@@ -49,7 +49,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "  DOCKERFILE  The name of Dockerfile to use."
   echo "              Default: Dockerfile"
   echo "  DOCKER_FROM The base image to use."
-  echo "              Default: 'python:3.9-alpine'"
+  echo "              Default: 'alpine:3.13'"
   echo "  DOCKER_TARGET A specific target to build."
   echo "              It's currently not possible to pass multiple targets."
   echo "              Default: main ldap"
@@ -106,7 +106,7 @@ else
 fi
 
 ###
-# Variables for fetching the source
+# Variables for fetching the Netbox source
 ###
 SRC_ORG="${SRC_ORG-netbox-community}"
 SRC_REPO="${SRC_REPO-netbox}"
@@ -115,10 +115,10 @@ URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}"
 NETBOX_PATH="${NETBOX_PATH-.netbox}"
 
 ###
-# Fetching the source
+# Fetching the Netbox source
 ###
 if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then
-  echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'"
+  echo "🌐 Checking out '${NETBOX_BRANCH}' of Netbox from the url '${URL}' into '${NETBOX_PATH}'"
   if [ ! -d "${NETBOX_PATH}" ]; then
     $DRY git clone -q --depth 10 -b "${NETBOX_BRANCH}" "${URL}" "${NETBOX_PATH}"
   fi
@@ -135,7 +135,7 @@ if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then
     $DRY git checkout -qf FETCH_HEAD
     $DRY git prune
   )
-  echo "✅ Checked out netbox"
+  echo "✅ Checked out Netbox"
 fi
 
 ###
@@ -157,7 +157,7 @@ fi
 # Determining the value for DOCKER_FROM
 ###
 if [ -z "$DOCKER_FROM" ]; then
-  DOCKER_FROM="python:3.9-alpine"
+  DOCKER_FROM="alpine:3.13"
 fi
 
 ###
@@ -271,7 +271,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
 
       if ! printf '%s\n' "${IMAGES_LAYERS_OLD[@]}" | grep -q -P "^${PYTHON_LAST_LAYER}\$"; then
         SHOULD_BUILD="true"
-        BUILD_REASON="${BUILD_REASON} python"
+        BUILD_REASON="${BUILD_REASON} alpine"
       fi
       if [ "${NETBOX_GIT_REF}" != "${NETBOX_GIT_REF_OLD}" ]; then
         SHOULD_BUILD="true"
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
index 23f74ca..a7fa7a3 100644
--- a/docker-compose.test.yml
+++ b/docker-compose.test.yml
@@ -14,19 +14,9 @@ services:
     - ./configuration:/etc/netbox/config:z,ro
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
-    - netbox-nginx-config:/etc/netbox-nginx:z
-    - netbox-static-files:/opt/netbox/netbox/static:z
     - netbox-media-files:/opt/netbox/netbox/media:z
-  nginx:
-    command: nginx -c /etc/netbox-nginx/nginx.conf
-    image: nginx:1.19-alpine
-    depends_on:
-    - netbox
     ports:
     - 8080
-    volumes:
-    - netbox-static-files:/opt/netbox/netbox/static:ro
-    - netbox-nginx-config:/etc/netbox-nginx/:ro
   postgres:
     image: postgres:12-alpine
     env_file: env/postgres.env
@@ -45,9 +35,5 @@ services:
     - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
     env_file: env/redis-cache.env
 volumes:
-  netbox-static-files:
-    driver: local
-  netbox-nginx-config:
-    driver: local
   netbox-media-files:
     driver: local
diff --git a/docker-compose.yml b/docker-compose.yml
index 8a2c575..4b0fda3 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -15,30 +15,19 @@ services:
     - ./configuration:/etc/netbox/config:z,ro
     - ./reports:/etc/netbox/reports:z,ro
     - ./scripts:/etc/netbox/scripts:z,ro
-    - netbox-nginx-config:/etc/netbox-nginx:z
-    - netbox-static-files:/opt/netbox/netbox/static:z
     - netbox-media-files:/opt/netbox/netbox/media:z
+    ports:
+    - "8080"
   netbox-worker:
     <<: *netbox
     depends_on:
     - redis
     entrypoint:
-    - python3
+    - /opt/netbox/venv/bin/python
     - /opt/netbox/netbox/manage.py
     command:
     - rqworker
-
-  # nginx
-  nginx:
-    command: nginx -c /etc/netbox-nginx/nginx.conf
-    image: nginx:1.19-alpine
-    depends_on:
-    - netbox
-    ports:
-    - 8080
-    volumes:
-    - netbox-static-files:/opt/netbox/netbox/static:ro
-    - netbox-nginx-config:/etc/netbox-nginx/:ro
+    ports: []
 
   # postgres
   postgres:
@@ -66,10 +55,6 @@ services:
     env_file: env/redis-cache.env
 
 volumes:
-  netbox-static-files:
-    driver: local
-  netbox-nginx-config:
-    driver: local
   netbox-media-files:
     driver: local
   netbox-postgres-data:
diff --git a/docker/configuration.docker.py b/docker/configuration.docker.py
index e7a7c55..0bd4d51 100644
--- a/docker/configuration.docker.py
+++ b/docker/configuration.docker.py
@@ -9,6 +9,7 @@ from os import scandir
 import importlib.util
 import sys
 
+
 def _filename(f):
   return f.name
 
diff --git a/docker/docker-entrypoint.sh b/docker/docker-entrypoint.sh
index 4887d96..0765a4a 100755
--- a/docker/docker-entrypoint.sh
+++ b/docker/docker-entrypoint.sh
@@ -7,6 +7,9 @@ set -e
 # Allows Netbox to be run as non-root users
 umask 002
 
+# Load correct Python3 env
+source /opt/netbox/venv/bin/activate
+
 # Try to connect to the DB
 DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3}
 MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30}
@@ -60,9 +63,6 @@ else
   echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python
 fi
 
-# Copy static files
-./manage.py collectstatic --no-input
-
 echo "✅ Initialisation is done."
 
 # Launch whatever is passed by docker
diff --git a/docker/gunicorn_config.py b/docker/gunicorn_config.py
deleted file mode 100644
index 4ce7763..0000000
--- a/docker/gunicorn_config.py
+++ /dev/null
@@ -1,8 +0,0 @@
-command = '/usr/bin/gunicorn'
-pythonpath = '/opt/netbox/netbox'
-bind = '0.0.0.0:8001'
-workers = 3
-errorlog = '-'
-accesslog = '-'
-capture_output = False
-loglevel = 'info'
diff --git a/docker/launch-netbox.sh b/docker/launch-netbox.sh
new file mode 100755
index 0000000..d0e52e3
--- /dev/null
+++ b/docker/launch-netbox.sh
@@ -0,0 +1,53 @@
+#!/bin/bash
+
+UNIT_CONFIG="${UNIT_CONFIG-/etc/unit/nginx-unit.json}"
+UNIT_SOCKET="/opt/unit/unit.sock"
+
+load_configuration() {
+  MAX_WAIT=10
+  WAIT_COUNT=0
+  while [ ! -S $UNIT_SOCKET ]; do
+    if [ $WAIT_COUNT -gte $MAX_WAIT ]; then
+      echo "⚠️ No control socket found; configuration will not be loaded."
+      return 1
+    fi
+
+    WAIT_COUNT=$((WAIT_COUNT + 1))
+    echo "⏳ Waiting for control socket to be created... (${WAIT_COUNT}/${MAX_WAIT})"
+
+    sleep 1
+  done
+
+  # even when the control socket exists, it does not mean unit has finished initialisation
+  # this curl call will get a reply once unit is fully launched
+  curl --silent --output /dev/null --request GET --unix-socket $UNIT_SOCKET http://localhost/
+
+  echo "⚙️ Applying configuration from $UNIT_CONFIG";
+
+  RESP_CODE=$(curl \
+                  --silent \
+                  --output /dev/null \
+                  --write-out '%{http_code}' \
+                  --request PUT \
+                  --data-binary "@${UNIT_CONFIG}" \
+                  --unix-socket $UNIT_SOCKET \
+                  http://localhost/config
+            )
+  if [ "$RESP_CODE" != "200" ]; then
+    echo "⚠️ Could no load Unit configuration"
+    kill "$(cat /opt/unit/unit.pid)"
+    return 1
+  fi
+
+  echo "✅ Unit configuration loaded successfully"
+}
+
+load_configuration &
+
+exec unitd \
+    --no-daemon \
+    --control unix:$UNIT_SOCKET \
+    --pid /opt/unit/unit.pid \
+    --log /dev/stdout \
+    --state /opt/unit/state/ \
+    --tmp /opt/unit/tmp/
diff --git a/docker/nginx-unit.json b/docker/nginx-unit.json
new file mode 100644
index 0000000..80673ae
--- /dev/null
+++ b/docker/nginx-unit.json
@@ -0,0 +1,40 @@
+{
+    "listeners": {
+        "*:8080": {
+            "pass": "routes"
+        }
+    },
+
+    "routes": [
+        {
+            "match": {
+                "uri": "/static/*"
+            },
+            "action": {
+                "share": "/opt/netbox/netbox"
+            }
+        },
+
+        {
+            "action": {
+                "pass": "applications/netbox"
+            }
+        }
+    ],
+
+    "applications": {
+        "netbox": {
+            "type": "python 3",
+            "path": "/opt/netbox/netbox/",
+            "module": "netbox.wsgi",
+            "home": "/opt/netbox/venv",
+            "processes": {
+                "max": 4,
+                "spare": 1,
+                "idle_timeout": 120
+            }
+        }
+    },
+
+    "access_log": "/dev/stdout"
+}
diff --git a/docker/nginx.conf b/docker/nginx.conf
deleted file mode 100644
index 75e3bfd..0000000
--- a/docker/nginx.conf
+++ /dev/null
@@ -1,44 +0,0 @@
-daemon off;
-worker_processes 1;
-
-error_log /dev/stderr info;
-
-events {
-    worker_connections 1024;
-}
-
-http {
-    include              /etc/nginx/mime.types;
-    default_type         application/octet-stream;
-    sendfile             on;
-    tcp_nopush           on;
-    keepalive_timeout    65;
-    gzip                 on;
-    server_tokens        off;
-    client_max_body_size 10M;
-
-    server {
-        listen      8080;
-        access_log  off;
-
-        location /static/ {
-            alias /opt/netbox/netbox/static/;
-        }
-
-        location / {
-            proxy_pass http://netbox:8001;
-            proxy_set_header X-Forwarded-Host $http_host;
-            proxy_set_header X-Real-IP $remote_addr;
-            add_header P3P 'CP="ALL DSP COR PSAa PSDa OUR NOR ONL UNI COM NAV"';
-        }
-    }
-
-    server {
-        listen      8081;
-        access_log  off;
-
-        location = /stub_status {
-            stub_status;
-        }
-    }
-}
diff --git a/requirements-container.txt b/requirements-container.txt
new file mode 100644
index 0000000..e011042
--- /dev/null
+++ b/requirements-container.txt
@@ -0,0 +1,4 @@
+napalm==3.2.0
+ruamel.yaml>=0.15,<0.16
+django-auth-ldap==2.2.0
+django-storages==1.10.1
\ No newline at end of file
diff --git a/test.sh b/test.sh
index fa6fa38..2249316 100755
--- a/test.sh
+++ b/test.sh
@@ -35,7 +35,7 @@ if [ -z "${IMAGE}" ]; then
 fi
 
 # The docker compose command to use
-doco="docker-compose -f docker-compose.test.yml"
+doco="docker-compose --file docker-compose.test.yml --project-name netbox_docker_test_${1}"
 
 INITIALIZERS_DIR=".initializers"