Build system for hub.docker.com

DOCKER_HUB.md

@@ -0,0 +1,66 @@
+# cloud.docker.com Configuration
+
+The automatic build is configured in cloud.docker.com.
+
+The following build configuration is expected:
+
+```yaml
+Source Repository: github.com/netbox-community/netbox-docker
+Build Location: Build on Docker Hub's infrastructure
+Autotest: Internal and External Pull Requests
+Repository Links: Enable for Base Image
+Build Rules:
+- Source Type: Branch
+  Source: master
+  Docker Tag: branches-main
+  Dockerfile location: Dockerfile
+- Source Type: Branch
+  Source: master
+  Docker Tag: branches-ldap
+  Dockerfile location: Dockerfile.ldap
+- Source Type: Branch
+  Source: master
+  Docker Tag: prerelease-main
+  Dockerfile location: Dockerfile
+- Source Type: Branch
+  Source: master
+  Docker Tag: prerelease-ldap
+  Dockerfile location: Dockerfile.ldap
+- Source Type: Branch
+  Source: master
+  Docker Tag: release-main
+  Dockerfile location: Dockerfile
+- Source Type: Branch
+  Source: master
+  Docker Tag: release-ldap
+  Dockerfile location: Dockerfile.ldap
+Build Environment Variables:
+# Create an app on Github and use it's OATH credentials here
+- Key: GITHUB_OAUTH_CLIENT_ID
+  Value: <secret>
+- Key: GITHUB_OAUTH_CLIENT_SECRET
+  Value: <secret>
+Build Triggers:
+- Name: Cron Trigger
+# Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds
+```
+
+## Background Knowledge
+
+The build system of cloud.docker.com is not made for this kind of project.
+But we found a way to make it work, and this is how:
+
+1. The docker hub build system [allows to overwrite the scripts that get executed
+   for `build`, `test` and `push`](overwrite). See `hooks/*`.
+2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `hooks/common`.
+3. The `build` script runs `run_build()` from `hooks/common`.
+   This triggers either `build-branches.sh`, `build-latest.sh` or directly `build.sh`.
+4. The `test` script just invokes `docker-compose` commands.
+5. The `push` script runs `run_build()` from `hooks/common` with a `--push-only` flag.
+   This causes the `build.sh` script to not re-build the Docker image, but just the just built image.
+
+The _Docker Tag_ configuration setting is misused to select the type (_release_, _prerelease_, _branches_) of the build as well as the variant (_main_, _ldap_).
+
+The _Dockerfile location_ configuration setting is completely ignored by the build scripts.
+
+[overwrite]: https://docs.docker.com/docker-hub/builds/advanced/#override-build-test-or-push-commands

build-all.sh

@@ -12,9 +12,9 @@ BUILDS=("${BUILD:-"${ALL_BUILDS[@]}"}")
 
 echo "⚙️ Configured builds: ${BUILDS[*]}"
 
-VARIANTS=("" "ldap")
+VARIANTS=("main" "ldap")
 
-if [ ! -z "${DEBUG}" ]; then
+if [ -n "${DEBUG}" ]; then
   export DEBUG
 fi
 
@@ -22,16 +22,14 @@ ERROR=0
 
 # Don't build if not on `master` and don't build if on a pull request,
 # but build when DEBUG is not empty
-if [ ! -z "${DEBUG}" ] || \
+for VARIANT in "${VARIANTS[@]}"; do
-  ( [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ] ); then
-  for VARIANT in "${VARIANTS[@]}"; do
   export VARIANT
 
   # Checking which VARIANT to build
-    if [ -z "$VARIANT" ]; then
+  if [ "${VARIANT}" == "main" ]; then
-      DOCKERFILE="Dockerfile"
+    DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}"
   else
-      DOCKERFILE="Dockerfile.${VARIANT}"
+    DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}.${VARIANT}"
 
     # Fail fast
     if [ ! -f "${DOCKERFILE}" ]; then
@@ -68,6 +66,7 @@ if [ ! -z "${DEBUG}" ] || \
         # special build
         # shellcheck disable=SC2068
         #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ || ERROR=1
+        echo "✅ No special builds today."
         ;;
       *)
         echo "🚨 Unrecognized build '$BUILD'."
@@ -80,9 +79,6 @@ if [ ! -z "${DEBUG}" ] || \
         ;;
     esac
   done
-  done
+done
-else
-  echo "❎ Not building anything."
-fi
 
 exit $ERROR

build-branches.sh

@@ -3,7 +3,7 @@
 
 echo "▶️ $0 $*"
 
-if [ ! -z "${GITHUB_OAUTH_CLIENT_ID}" ] && [ ! -z "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
+if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
   echo "🗝 Performing authenticated Github API calls."
   GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
 else

build-latest.sh

@@ -3,7 +3,7 @@
 
 echo "▶️ $0 $*"
 
-if [ ! -z "${GITHUB_OAUTH_CLIENT_ID}" ] && [ ! -z "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
+if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
   echo "🗝 Performing authenticated Github API calls."
   GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
 else

build.sh

@@ -6,9 +6,10 @@ echo "▶️ $0 $*"
 set -e
 
 if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
-  echo "Usage: ${0} <branch> [--push]"
+  echo "Usage: ${0} <branch> [--push|--push-only]"
   echo "  branch       The branch or tag to build. Required."
-  echo "  --push  Pushes built Docker image to docker hub."
+  echo "  --push       Pushes built the Docker image to the registry."
+  echo "  --push-only  Does not build. Only pushes the Docker image to the registry."
   echo ""
   echo "You can use the following ENV variables to customize the build:"
   echo "  DEBUG    If defined, the script does not stop when certain checks are unsatisfied."
@@ -49,8 +50,9 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
   echo "             ARG FROM_TAG=latest"
   echo "             FROM \$DOCKER_ORG/\$DOCKER_REPO:\$FROM_TAG"
   echo "           Example: VARIANT=ldap will result in the tag 'latest-ldap' and the"
-  echo "            Dockerfile 'Dockerfile.ldap' being used."
+  echo "           Dockerfile './Dockerfile.ldap' being used."
-  echo "           Default: empty"
+  echo "           Exception: VARIANT=main will use the './Dockerfile' Dockerfile"
+  echo "           Default: main"
   echo "  HTTP_PROXY The proxy to use for http requests."
   echo "           Example: http://proxy.domain.tld:3128"
   echo "           Default: empty"
@@ -95,7 +97,7 @@ esac
 DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}"
 
 # Checking which VARIANT to build
-if [ -z "$VARIANT" ]; then
+if [ "$VARIANT" == "main" ]; then
   DOCKERFILE="Dockerfile"
 else
   DOCKERFILE="Dockerfile.${VARIANT}"
@@ -152,11 +154,13 @@ else
   DOCKER_CMD="echo docker"
 fi
 
-echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'."
+if [ "${2}" != "--push-only" ] ; then
-$DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" .
+  echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'."
-echo "✅ Finished building the Docker images '${DOCKER_TAG}'"
+  $DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" .
+  echo "✅ Finished building the Docker images '${DOCKER_TAG}'"
+fi
 
-if [ "${2}" == "--push" ] ; then
+if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then
   echo "⏫ Pushing '${DOCKER_TAG}"
   $DOCKER_CMD push "${DOCKER_TAG}"
   echo "✅ Finished pushing the Docker image '${DOCKER_TAG}'."

hooks/build

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. hooks/common
+
+run_build

hooks/common

@@ -0,0 +1,82 @@
+#!/bin/bash
+
+ensure_jq() {
+  echo "🛠🛠🛠 Installing JQ via apt-get"
+  [ -x "$(command -v jq)" ] || ( apt-get update && apt-get install -y jq )
+}
+
+ensure_dockerfile_present() {
+  if [ "${VARIANT}" == "main" ]; then
+    DOCKERFILE="Dockerfile"
+  else
+    DOCKERFILE="Dockerfile.${VARIANT}"
+
+    # Fail fast
+    if [ ! -f "${DOCKERFILE}" ]; then
+      echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist."
+
+      if [ -z "$DEBUG" ]; then
+        exit 1
+      else
+        echo "⚠️ Would skip this, but DEBUG is enabled."
+      fi
+    fi
+
+    if [ "${DOCKERFILE}" != "${DOCKERFILE_PATH}" ]; then
+      echo "⚠️ The specified Dockerfile '${DOCKERFILE_PATH}' does not match the expected Dockerfile '${DOCKERFILE}'."
+      echo "   This script will use '${DOCKERFILE}' and ignore '${DOCKERFILE_PATH}'."
+    fi
+  fi
+}
+
+# Passes args to the scripts
+run_build() {
+  echo "🐳🐳🐳 Building '${BUILD}' images, the '${VARIANT:-main}' variant"
+  case $BUILD in
+    release)
+      # build the latest release
+      # shellcheck disable=SC2068
+      ./build-latest.sh $@
+      ;;
+    prerelease)
+      # build the latest pre-release
+      # shellcheck disable=SC2068
+      PRERELEASE=true ./build-latest.sh $@
+      ;;
+    branches)
+      # build all branches
+      # shellcheck disable=SC2068
+      ./build-branches.sh $@
+      ;;
+    special)
+      # special build
+      # shellcheck disable=SC2068
+      #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@
+      echo "✅ No special builds today."
+      ;;
+    *)
+      echo "🚨 Unrecognized build '$BUILD'."
+
+      if [ -z "$DEBUG" ]; then
+        exit 1
+      else
+        echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
+      fi
+      ;;
+  esac
+}
+
+echo "🤖🤖🤖 Preparing build"
+export DOCKER_ORG="index.docker.io/cimnine"
+export DOCKER_REPO=netbox-test
+export DOCKERHUB_REPO=cimnine/netbox-test
+
+# mis-using the "${DOCKER_TAG}" variable as "branch to build"
+export BUILD="${DOCKER_TAG%-*}"
+export VARIANT="${DOCKER_TAG#*-}"
+
+unset DOCKER_TAG
+
+ensure_dockerfile_present
+
+ensure_jq

hooks/push

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+. hooks/common
+
+run_build --push-only

hooks/test

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+. hooks/common
+
+if [ "${VARIANT}" == "main" ] && [ "${BUILD}" == "BRANCHES" ]; then
+  echo "🐳🐳🐳 Testing"
+  docker-compose pull --parallel
+  docker-compose build
+  docker-compose run netbox ./manage.py test
+else
+  echo "🐳🐳🐳 No tests are implemented for build '${BUILD}' with variant '${VARIANT}'."
+fi