From 4ef420d4435b91b9c77992290cb5e4a43814a900 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Ma=CC=88der?= Date: Fri, 1 Feb 2019 09:58:07 +0100 Subject: [PATCH] Build system for hub.docker.com --- DOCKER_HUB.md | 66 ++++++++++++++++++++++++++++ build-all.sh | 108 ++++++++++++++++++++++------------------------ build-branches.sh | 2 +- build-latest.sh | 2 +- build.sh | 28 ++++++------ hooks/build | 5 +++ hooks/common | 82 +++++++++++++++++++++++++++++++++++ hooks/push | 5 +++ hooks/test | 12 ++++++ 9 files changed, 240 insertions(+), 70 deletions(-) create mode 100644 DOCKER_HUB.md create mode 100755 hooks/build create mode 100755 hooks/common create mode 100755 hooks/push create mode 100755 hooks/test diff --git a/DOCKER_HUB.md b/DOCKER_HUB.md new file mode 100644 index 0000000..6149c3c --- /dev/null +++ b/DOCKER_HUB.md @@ -0,0 +1,66 @@ +# cloud.docker.com Configuration + +The automatic build is configured in cloud.docker.com. + +The following build configuration is expected: + +```yaml +Source Repository: github.com/netbox-community/netbox-docker +Build Location: Build on Docker Hub's infrastructure +Autotest: Internal and External Pull Requests +Repository Links: Enable for Base Image +Build Rules: +- Source Type: Branch + Source: master + Docker Tag: branches-main + Dockerfile location: Dockerfile +- Source Type: Branch + Source: master + Docker Tag: branches-ldap + Dockerfile location: Dockerfile.ldap +- Source Type: Branch + Source: master + Docker Tag: prerelease-main + Dockerfile location: Dockerfile +- Source Type: Branch + Source: master + Docker Tag: prerelease-ldap + Dockerfile location: Dockerfile.ldap +- Source Type: Branch + Source: master + Docker Tag: release-main + Dockerfile location: Dockerfile +- Source Type: Branch + Source: master + Docker Tag: release-ldap + Dockerfile location: Dockerfile.ldap +Build Environment Variables: +# Create an app on Github and use it's OATH credentials here +- Key: GITHUB_OAUTH_CLIENT_ID + Value: +- Key: GITHUB_OAUTH_CLIENT_SECRET + Value: +Build Triggers: +- Name: Cron Trigger +# Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds +``` + +## Background Knowledge + +The build system of cloud.docker.com is not made for this kind of project. +But we found a way to make it work, and this is how: + +1. The docker hub build system [allows to overwrite the scripts that get executed + for `build`, `test` and `push`](overwrite). See `hooks/*`. +2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `hooks/common`. +3. The `build` script runs `run_build()` from `hooks/common`. + This triggers either `build-branches.sh`, `build-latest.sh` or directly `build.sh`. +4. The `test` script just invokes `docker-compose` commands. +5. The `push` script runs `run_build()` from `hooks/common` with a `--push-only` flag. + This causes the `build.sh` script to not re-build the Docker image, but just the just built image. + +The _Docker Tag_ configuration setting is misused to select the type (_release_, _prerelease_, _branches_) of the build as well as the variant (_main_, _ldap_). + +The _Dockerfile location_ configuration setting is completely ignored by the build scripts. + +[overwrite]: https://docs.docker.com/docker-hub/builds/advanced/#override-build-test-or-push-commands diff --git a/build-all.sh b/build-all.sh index 0c1bd53..d88c5af 100755 --- a/build-all.sh +++ b/build-all.sh @@ -12,9 +12,9 @@ BUILDS=("${BUILD:-"${ALL_BUILDS[@]}"}") echo "⚙️ Configured builds: ${BUILDS[*]}" -VARIANTS=("" "ldap") +VARIANTS=("main" "ldap") -if [ ! -z "${DEBUG}" ]; then +if [ -n "${DEBUG}" ]; then export DEBUG fi @@ -22,67 +22,63 @@ ERROR=0 # Don't build if not on `master` and don't build if on a pull request, # but build when DEBUG is not empty -if [ ! -z "${DEBUG}" ] || \ - ( [ "$TRAVIS_BRANCH" = "master" ] && [ "$TRAVIS_PULL_REQUEST" = "false" ] ); then - for VARIANT in "${VARIANTS[@]}"; do - export VARIANT +for VARIANT in "${VARIANTS[@]}"; do + export VARIANT - # Checking which VARIANT to build - if [ -z "$VARIANT" ]; then - DOCKERFILE="Dockerfile" - else - DOCKERFILE="Dockerfile.${VARIANT}" + # Checking which VARIANT to build + if [ "${VARIANT}" == "main" ]; then + DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}" + else + DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}.${VARIANT}" - # Fail fast - if [ ! -f "${DOCKERFILE}" ]; then - echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist." - ERROR=1 + # Fail fast + if [ ! -f "${DOCKERFILE}" ]; then + echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist." + ERROR=1 - if [ -z "$DEBUG" ]; then - continue - else - echo "⚠️ Would skip this, but DEBUG is enabled." - fi + if [ -z "$DEBUG" ]; then + continue + else + echo "⚠️ Would skip this, but DEBUG is enabled." fi fi + fi - for BUILD in "${BUILDS[@]}"; do - echo "🛠 Building '$BUILD' from '$DOCKERFILE'" - case $BUILD in - release) - # build the latest release - # shellcheck disable=SC2068 - ./build-latest.sh $@ || ERROR=1 - ;; - prerelease) - # build the latest pre-release - # shellcheck disable=SC2068 - PRERELEASE=true ./build-latest.sh $@ || ERROR=1 - ;; - branches) - # build all branches - # shellcheck disable=SC2068 - ./build-branches.sh $@ || ERROR=1 - ;; - special) - # special build - # shellcheck disable=SC2068 - #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ || ERROR=1 - ;; - *) - echo "🚨 Unrecognized build '$BUILD'." + for BUILD in "${BUILDS[@]}"; do + echo "🛠 Building '$BUILD' from '$DOCKERFILE'" + case $BUILD in + release) + # build the latest release + # shellcheck disable=SC2068 + ./build-latest.sh $@ || ERROR=1 + ;; + prerelease) + # build the latest pre-release + # shellcheck disable=SC2068 + PRERELEASE=true ./build-latest.sh $@ || ERROR=1 + ;; + branches) + # build all branches + # shellcheck disable=SC2068 + ./build-branches.sh $@ || ERROR=1 + ;; + special) + # special build + # shellcheck disable=SC2068 + #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ || ERROR=1 + echo "✅ No special builds today." + ;; + *) + echo "🚨 Unrecognized build '$BUILD'." - if [ -z "$DEBUG" ]; then - exit 1 - else - echo "⚠️ Would exit here with code '1', but DEBUG is enabled." - fi - ;; - esac - done + if [ -z "$DEBUG" ]; then + exit 1 + else + echo "⚠️ Would exit here with code '1', but DEBUG is enabled." + fi + ;; + esac done -else - echo "❎ Not building anything." -fi +done exit $ERROR diff --git a/build-branches.sh b/build-branches.sh index 3bd93a9..91dbf22 100755 --- a/build-branches.sh +++ b/build-branches.sh @@ -3,7 +3,7 @@ echo "▶️ $0 $*" -if [ ! -z "${GITHUB_OAUTH_CLIENT_ID}" ] && [ ! -z "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then +if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then echo "🗝 Performing authenticated Github API calls." GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}" else diff --git a/build-latest.sh b/build-latest.sh index d903f8f..5732bd2 100755 --- a/build-latest.sh +++ b/build-latest.sh @@ -3,7 +3,7 @@ echo "▶️ $0 $*" -if [ ! -z "${GITHUB_OAUTH_CLIENT_ID}" ] && [ ! -z "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then +if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then echo "🗝 Performing authenticated Github API calls." GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}" else diff --git a/build.sh b/build.sh index ecd680d..a400f6f 100755 --- a/build.sh +++ b/build.sh @@ -6,9 +6,10 @@ echo "▶️ $0 $*" set -e if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then - echo "Usage: ${0} [--push]" - echo " branch The branch or tag to build. Required." - echo " --push Pushes built Docker image to docker hub." + echo "Usage: ${0} [--push|--push-only]" + echo " branch The branch or tag to build. Required." + echo " --push Pushes built the Docker image to the registry." + echo " --push-only Does not build. Only pushes the Docker image to the registry." echo "" echo "You can use the following ENV variables to customize the build:" echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied." @@ -49,8 +50,9 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then echo " ARG FROM_TAG=latest" echo " FROM \$DOCKER_ORG/\$DOCKER_REPO:\$FROM_TAG" echo " Example: VARIANT=ldap will result in the tag 'latest-ldap' and the" - echo " Dockerfile 'Dockerfile.ldap' being used." - echo " Default: empty" + echo " Dockerfile './Dockerfile.ldap' being used." + echo " Exception: VARIANT=main will use the './Dockerfile' Dockerfile" + echo " Default: main" echo " HTTP_PROXY The proxy to use for http requests." echo " Example: http://proxy.domain.tld:3128" echo " Default: empty" @@ -95,7 +97,7 @@ esac DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}" # Checking which VARIANT to build -if [ -z "$VARIANT" ]; then +if [ "$VARIANT" == "main" ]; then DOCKERFILE="Dockerfile" else DOCKERFILE="Dockerfile.${VARIANT}" @@ -117,8 +119,8 @@ DOCKER_OPTS=("${DOCKER_OPTS[@]}") # caching is only ok for version tags case "${TAG}" in - v*) ;; - *) DOCKER_OPTS+=( "--no-cache" ) ;; + v*) ;; + *) DOCKER_OPTS+=( "--no-cache" ) ;; esac DOCKER_OPTS+=( "--pull" ) @@ -152,11 +154,13 @@ else DOCKER_CMD="echo docker" fi -echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'." -$DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . -echo "✅ Finished building the Docker images '${DOCKER_TAG}'" +if [ "${2}" != "--push-only" ] ; then + echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'." + $DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" . + echo "✅ Finished building the Docker images '${DOCKER_TAG}'" +fi -if [ "${2}" == "--push" ] ; then +if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then echo "⏫ Pushing '${DOCKER_TAG}" $DOCKER_CMD push "${DOCKER_TAG}" echo "✅ Finished pushing the Docker image '${DOCKER_TAG}'." diff --git a/hooks/build b/hooks/build new file mode 100755 index 0000000..85bad75 --- /dev/null +++ b/hooks/build @@ -0,0 +1,5 @@ +#!/bin/bash + +. hooks/common + +run_build diff --git a/hooks/common b/hooks/common new file mode 100755 index 0000000..ba9a3b8 --- /dev/null +++ b/hooks/common @@ -0,0 +1,82 @@ +#!/bin/bash + +ensure_jq() { + echo "🛠🛠🛠 Installing JQ via apt-get" + [ -x "$(command -v jq)" ] || ( apt-get update && apt-get install -y jq ) +} + +ensure_dockerfile_present() { + if [ "${VARIANT}" == "main" ]; then + DOCKERFILE="Dockerfile" + else + DOCKERFILE="Dockerfile.${VARIANT}" + + # Fail fast + if [ ! -f "${DOCKERFILE}" ]; then + echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist." + + if [ -z "$DEBUG" ]; then + exit 1 + else + echo "⚠️ Would skip this, but DEBUG is enabled." + fi + fi + + if [ "${DOCKERFILE}" != "${DOCKERFILE_PATH}" ]; then + echo "⚠️ The specified Dockerfile '${DOCKERFILE_PATH}' does not match the expected Dockerfile '${DOCKERFILE}'." + echo " This script will use '${DOCKERFILE}' and ignore '${DOCKERFILE_PATH}'." + fi + fi +} + +# Passes args to the scripts +run_build() { + echo "🐳🐳🐳 Building '${BUILD}' images, the '${VARIANT:-main}' variant" + case $BUILD in + release) + # build the latest release + # shellcheck disable=SC2068 + ./build-latest.sh $@ + ;; + prerelease) + # build the latest pre-release + # shellcheck disable=SC2068 + PRERELEASE=true ./build-latest.sh $@ + ;; + branches) + # build all branches + # shellcheck disable=SC2068 + ./build-branches.sh $@ + ;; + special) + # special build + # shellcheck disable=SC2068 + #SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ + echo "✅ No special builds today." + ;; + *) + echo "🚨 Unrecognized build '$BUILD'." + + if [ -z "$DEBUG" ]; then + exit 1 + else + echo "⚠️ Would exit here with code '1', but DEBUG is enabled." + fi + ;; + esac +} + +echo "🤖🤖🤖 Preparing build" +export DOCKER_ORG="index.docker.io/cimnine" +export DOCKER_REPO=netbox-test +export DOCKERHUB_REPO=cimnine/netbox-test + +# mis-using the "${DOCKER_TAG}" variable as "branch to build" +export BUILD="${DOCKER_TAG%-*}" +export VARIANT="${DOCKER_TAG#*-}" + +unset DOCKER_TAG + +ensure_dockerfile_present + +ensure_jq diff --git a/hooks/push b/hooks/push new file mode 100755 index 0000000..0c0b65f --- /dev/null +++ b/hooks/push @@ -0,0 +1,5 @@ +#!/bin/bash + +. hooks/common + +run_build --push-only diff --git a/hooks/test b/hooks/test new file mode 100755 index 0000000..6c89c87 --- /dev/null +++ b/hooks/test @@ -0,0 +1,12 @@ +#!/bin/bash + +. hooks/common + +if [ "${VARIANT}" == "main" ] && [ "${BUILD}" == "BRANCHES" ]; then + echo "🐳🐳🐳 Testing" + docker-compose pull --parallel + docker-compose build + docker-compose run netbox ./manage.py test +else + echo "🐳🐳🐳 No tests are implemented for build '${BUILD}' with variant '${VARIANT}'." +fi