Building the Docker image with Github Actions

.github/workflows/push.yml

@@ -0,0 +1,31 @@
+on:
+  push:
+    branches-ignore:
+      - release
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        build_cmd:
+        - ./build-latest.sh
+        - PRERELEASE=true ./build-latest.sh
+        - ./build-branches.sh
+        docker_from:
+        - '' # use the default of the DOCKERFILE
+        - python:3.7-alpine
+        - python:3.8-alpine
+        - python:3.9-rc-alpine
+      fail-fast: false
+    runs-on: ubuntu-latest
+    name: Builds new Netbox Docker Images
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v1
+    - name: Build the image from '${{ matrix.docker_from }}' with '${{ matrix.build_cmd }}'
+      run: ${{ matrix.build_cmd }}
+      env:
+        DOCKER_FROM: ${{ matrix.docker_from }}
+        GH_ACTION: enable
+    - name: Test the image
+      run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh

.github/workflows/release.yml

@@ -0,0 +1,39 @@
+on:
+  push:
+    branches:
+      - release
+  schedule:
+    - cron: '45 5 * * *'
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        build_cmd:
+        - ./build-latest.sh
+        - PRERELEASE=true ./build-latest.sh
+        - ./build-branches.sh
+      fail-fast: false
+    runs-on: ubuntu-latest
+    name: Builds new Netbox Docker Images
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v1
+    - name: Build the image with '${{ matrix.build_cmd }}'
+      run: ${{ matrix.build_cmd }}
+      env:
+        GH_ACTION: enable
+    - name: Test the image
+      run: IMAGE="${FINAL_DOCKER_TAG}" ./test.sh
+    - name: Login to the Docker Registry
+      run: |
+        echo "::add-mask::$DOCKERHUB_USERNAME"
+        echo "::add-mask::$DOCKERHUB_PASSWORD"
+        docker login -u "$DOCKERHUB_USERNAME" --password "${DOCKERHUB_PASSWORD}" "${DOCKER_REGISTRY}"
+      env:
+        DOCKERHUB_USERNAME: ${{ secrets.dockerhub_username }}
+        DOCKERHUB_PASSWORD: ${{ secrets.dockerhub_password }}
+    - name: Push the image
+      run: ${{ matrix.build_cmd }} --push-only
+    - name: Logout of the Docker Registry
+      run: docker logout "${DOCKER_REGISTRY}"

build.sh

@@ -63,6 +63,10 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "              Default: undefined"
   echo "  DRY_RUN     Prints all build statements instead of running them."
   echo "              Default: undefined"
+  echo "  GH_ACTION   If defined, special 'echo' statements are enabled that set the"
+  echo "              following environment variables in Github Actions:"
+  echo "              - FINAL_DOCKER_TAG: The final value of the DOCKER_TAG env variable"
+  echo "              Default: undefined"
   echo ""
   echo "Examples:"
   echo "  ${0} master"
@@ -92,7 +96,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
 fi
 
 ###
-# Determining the build command to use
+# Enabling dry-run mode
 ###
 if [ -z "${DRY_RUN}" ]; then
   DRY=""
@@ -102,7 +106,7 @@ else
 fi
 
 ###
-# variables for fetching the source
+# Variables for fetching the source
 ###
 SRC_ORG="${SRC_ORG-netbox-community}"
 SRC_REPO="${SRC_REPO-netbox}"
@@ -111,7 +115,7 @@ URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}"
 NETBOX_PATH="${NETBOX_PATH-.netbox}"
 
 ###
-# fetching the source
+# Fetching the source
 ###
 if [ "${2}" != "--push-only" ] && [ -z "${SKIP_GIT}" ] ; then
   echo "🌐 Checking out '${NETBOX_BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'"
@@ -150,7 +154,7 @@ if [ ! -f "${DOCKERFILE}" ]; then
 fi
 
 ###
-# variables for labelling the docker image
+# Variables for labelling the docker image
 ###
 BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M+00:00')"
 
@@ -158,7 +162,7 @@ if [ -d ".git" ]; then
   GIT_REF="$(git rev-parse HEAD)"
 fi
 
-# read the project version from the `VERSION` file and trim it, see https://stackoverflow.com/a/3232433/172132
+# Read the project version from the `VERSION` file and trim it, see https://stackoverflow.com/a/3232433/172132
 PROJECT_VERSION="${PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}"
 
 # Get the Git information from the netbox directory
@@ -169,7 +173,7 @@ if [ -d "${NETBOX_PATH}/.git" ]; then
 fi
 
 ###
-# variables for tagging the docker image
+# Variables for tagging the docker image
 ###
 DOCKER_REGISTRY="${DOCKER_REGISTRY-docker.io}"
 DOCKER_ORG="${DOCKER_ORG-netboxcommunity}"
@@ -193,6 +197,7 @@ echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}"
 ###
 # Build each target
 ###
+export DOCKER_BUILDKIT=${DOCKER_BUILDKIT-1}
 for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
   echo "🏗 Building the target '${DOCKER_TARGET}'"
 
@@ -203,6 +208,9 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
   if [ "${DOCKER_TARGET}" != "main" ]; then
     TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}"
   fi
+  if [ -n "${GH_ACTION}" ]; then
+    echo "::set-env name=FINAL_DOCKER_TAG::${TARGET_DOCKER_TAG}"
+  fi
 
   ###
   # composing the additional DOCKER_SHORT_TAG,

docker-compose.test.yml

@@ -0,0 +1,45 @@
+version: '3.4'
+services:
+  netbox:
+    image: ${IMAGE-netboxcommunity/netbox:latest}
+    depends_on:
+    - postgres
+    - redis
+    env_file: env/netbox.env
+    user: '101'
+    volumes:
+    - ./startup_scripts:/opt/netbox/startup_scripts:z,ro
+    - ./initializers:/opt/netbox/initializers:z,ro
+    - ./configuration:/etc/netbox/config:z,ro
+    - ./reports:/etc/netbox/reports:z,ro
+    - ./scripts:/etc/netbox/scripts:z,ro
+    - netbox-nginx-config:/etc/netbox-nginx:z
+    - netbox-static-files:/opt/netbox/netbox/static:z
+    - netbox-media-files:/opt/netbox/netbox/media:z
+  nginx:
+    command: nginx -c /etc/netbox-nginx/nginx.conf
+    image: nginx:1.17-alpine
+    depends_on:
+    - netbox
+    ports:
+    - 8080
+    volumes:
+    - netbox-static-files:/opt/netbox/netbox/static:ro
+    - netbox-nginx-config:/etc/netbox-nginx/:ro
+  postgres:
+    image: postgres:11-alpine
+    env_file: env/postgres.env
+  redis:
+    image: redis:5-alpine
+    command:
+    - sh
+    - -c # this is to evaluate the $REDIS_PASSWORD from the env
+    - redis-server --requirepass $$REDIS_PASSWORD ## $$ because of docker-compose
+    env_file: env/redis.env
+volumes:
+  netbox-static-files:
+    driver: local
+  netbox-nginx-config:
+    driver: local
+  netbox-media-files:
+    driver: local

docker-compose.yml

@@ -57,8 +57,6 @@ volumes:
     driver: local
   netbox-media-files:
     driver: local
-  netbox-report-files:
-    driver: local
   netbox-postgres-data:
     driver: local
   netbox-redis-data:

docker/docker-entrypoint.sh

@@ -1,13 +1,27 @@
 #!/bin/bash
+# Runs on every start of the Netbox Docker container
+
+# Stop when an error occures
 set -e
+
+# Allows Netbox to be run as non-root users
 umask 002
 
-# wait shortly and then run db migrations (retry on error)
+# Try to connect to the DB
-while ! ./manage.py migrate 2>&1; do
+DB_WAIT_TIMEOUT=${DB_WAIT_TIMEOUT-3}
-  echo "⏳ Waiting on DB..."
+MAX_DB_WAIT_TIME=${MAX_DB_WAIT_TIME-30}
-  sleep 3
+CUR_DB_WAIT_TIME=0
+while ! ./manage.py migrate 2>&1 && [ "${CUR_DB_WAIT_TIME}" -lt "${MAX_DB_WAIT_TIME}" ]; do
+  echo "⏳ Waiting on DB... (${CUR_DB_WAIT_TIME}s / ${MAX_DB_WAIT_TIME}s)"
+  sleep "${DB_WAIT_TIMEOUT}"
+  CUR_DB_WAIT_TIME=$(( CUR_DB_WAIT_TIME + DB_WAIT_TIMEOUT ))
 done
+if [ "${CUR_DB_WAIT_TIME}" -ge "${MAX_DB_WAIT_TIME}" ]; then
+  echo "❌ Waited ${MAX_DB_WAIT_TIME}s or more for the DB to become ready."
+  exit 1
+fi
 
+# Create Superuser if required
 if [ "$SKIP_SUPERUSER" == "true" ]; then
   echo "↩️ Skip creating the superuser"
 else
@@ -43,18 +57,19 @@ END
   echo "💡 Superuser Username: ${SUPERUSER_NAME}, E-Mail: ${SUPERUSER_EMAIL}"
 fi
 
+# Run the startup scripts (and initializers)
 if [ "$SKIP_STARTUP_SCRIPTS" == "true" ]; then
   echo "↩️ Skipping startup scripts"
 else
   echo "import runpy; runpy.run_path('../startup_scripts')" | ./manage.py shell --interface python
 fi
 
-# copy static files
+# Copy static files
 ./manage.py collectstatic --no-input
 
 echo "✅ Initialisation is done."
 
-# launch whatever is passed by docker
+# Launch whatever is passed by docker
 # (i.e. the RUN instruction in the Dockerfile)
 #
 # shellcheck disable=SC2068

hooks/build

@@ -1,6 +0,0 @@
-#!/bin/bash
-
-. hooks/common
-
-# shellcheck disable=SC2119
-run_build

hooks/common

@@ -1,60 +0,0 @@
-#!/bin/bash
-
-ensure_jq() {
-  if [ ! -x "$(command -v jq)" ]; then
-    if [ -x "$(command -v apt-get)" ]; then
-      echo "🛠🛠🛠 Installing 'jq' via 'apt-get'"
-      apt-get update && apt-get install -y jq
-    else
-      echo "⚠️⚠️⚠️ apt-get not found, unable to automatically install 'jq'."
-    fi
-  fi
-}
-
-# Passes args to the scripts
-run_build() {
-  echo "🐳🐳🐳 Building '${BUILD}' images"
-  case ${BUILD} in
-    release)
-      # build the latest release
-      # shellcheck disable=SC2068
-      ./build-latest.sh $@
-      ;;
-    prerelease)
-      # build the latest pre-release
-      # shellcheck disable=SC2068
-      PRERELEASE=true ./build-latest.sh $@
-      ;;
-    branches)
-      # build all branches
-      # shellcheck disable=SC2068
-      ./build-branches.sh $@
-      ;;
-    this) # Pull Requests
-      # only build the 'master' branch of netbox
-      # (resulting in the 'latest' docker tag)
-      # and the 'main' target.
-      DOCKER_TARGET=main ./build.sh master
-      ;;
-    *)
-      echo "🚨 Unrecognized build '$BUILD'."
-
-      if [ -z "$DEBUG" ]; then
-        exit 1
-      else
-        echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
-      fi
-      ;;
-  esac
-}
-
-echo "🤖🤖🤖 Preparing build"
-export DOCKER_ORG=netboxcommunity
-export DOCKER_REPO=netbox
-export DOCKER_REGISTRY=docker.io
-# shellcheck disable=SC2153
-export BUILD="${DOCKER_TAG}"
-
-unset DOCKER_TAG
-
-ensure_jq

hooks/push

@@ -1,14 +0,0 @@
-#!/bin/bash
-
-. hooks/common
-
-if [ "${SOURCE_BRANCH}" == "release" ] || [ "${DEBUG}" == "true" ]; then
-  if [ "${SOURCE_BRANCH}" != "release" ]; then
-    echo "⚠️⚠️⚠️ Would exit, but DEBUG is '${DEBUG}'".
-  fi
-
-  run_build --push-only
-else
-  echo "⚠️⚠️⚠️ Only pushing on 'main' branch, but current branch is '${SOURCE_BRANCH}'"
-  exit 0
-fi

hooks/test

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-. hooks/common
-
-# test on builds of 'branches'
-if [ "${BUILD}" == "branches" ] \
-   || [ "${DEBUG}" == "true" ]; then
-  ./test.sh latest
-  ./test.sh snapshot
-# test on bulds of 'this' (i.e. pull request)
-elif [ "${BUILD}" == "this" ]; then
-  ./test.sh latest
-else
-  echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'."
-fi

test.sh

@@ -5,11 +5,14 @@ set -e
 
 # version is used by `docker-compose.yml` do determine the tag
 # of the Docker Image that is to be used
-export VERSION=${VERSION-latest}
+export IMAGE="${IMAGE-netboxcommunity/netbox:latest}"
+
+# The docker compose command to use
+doco="docker-compose -f docker-compose.test.yml"
 
 test_netbox_unit_tests() {
   echo "⏱ Running Netbox Unit Tests"
-  docker-compose run --rm netbox ./manage.py test
+  $doco run --rm netbox ./manage.py test
 }
 
 test_initializers() {
@@ -25,17 +28,20 @@ test_initializers() {
   mv initializers initializers_original
   mv initializers_test initializers
 
-  docker-compose run --rm netbox ./manage.py check
+  $doco run --rm netbox ./manage.py check
 }
 
 test_cleanup() {
   echo "💣 Cleaning Up"
-  docker-compose down -v
+  $doco down -v
-  rm -rf initializers
+
-  mv initializers_original initializers
+  if [ -d initializers_original ]; then
+    rm -rf initializers
+    mv initializers_original initializers
+  fi
 }
 
-echo "🐳🐳🐳 Start testing '${VERSION}'"
+echo "🐳🐳🐳 Start testing '${IMAGE}'"
 
 # Make sure the cleanup script is executed
 trap test_cleanup EXIT ERR
@@ -43,4 +49,4 @@ trap test_cleanup EXIT ERR
 test_netbox_unit_tests
 test_initializers
 
-echo "🐳🐳🐳 Done testing '${VERSION}'"
+echo "🐳🐳🐳 Done testing '${IMAGE}'"