Merge pull request #159 from netbox-community/multistage-build

Refactor to multistage builds
This commit is contained in:
Christian Mäder 2019-10-15 11:47:09 +02:00 committed by GitHub
commit ef989284c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 411 additions and 350 deletions

View File

@ -2,3 +2,4 @@
.travis.yml
build*
*.env
.git

1
.gitignore vendored
View File

@ -1 +1,2 @@
*.sql.gz
.netbox

View File

@ -12,28 +12,25 @@ Repository Links: Enable for Base Image
Build Rules:
- Source Type: Branch
Source: master
Docker Tag: branches-main
Docker Tag: branches
Dockerfile location: Dockerfile
Build Context: /
Autobuild: on
Build Caching: on
- Source Type: Branch
Source: master
Docker Tag: branches-ldap
Dockerfile location: Dockerfile.ldap
- Source Type: Branch
Source: master
Docker Tag: prerelease-main
Docker Tag: prerelease
Dockerfile location: Dockerfile
Build Context: /
Autobuild: on
Build Caching: on
- Source Type: Branch
Source: master
Docker Tag: prerelease-ldap
Dockerfile location: Dockerfile.ldap
- Source Type: Branch
Source: master
Docker Tag: release-main
Docker Tag: release
Dockerfile location: Dockerfile
- Source Type: Branch
Source: master
Docker Tag: release-ldap
Dockerfile location: Dockerfile.ldap
Build Context: /
Autobuild: on
Build Caching: on
Build Environment Variables:
# Create an app on Github and use it's OATH credentials here
- Key: GITHUB_OAUTH_CLIENT_ID
@ -42,6 +39,7 @@ Build Environment Variables:
Value: <secret>
Build Triggers:
- Name: Cron Trigger
Trigger URL: <generated>
# Use this trigger in combination with e.g. https://cron-job.org in order to regularly schedule builds
```
@ -51,16 +49,15 @@ The build system of cloud.docker.com is not made for this kind of project.
But we found a way to make it work, and this is how:
1. The docker hub build system [allows to overwrite the scripts that get executed
for `build`, `test` and `push`](overwrite). See `hooks/*`.
2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `hooks/common`.
3. The `build` script runs `run_build()` from `hooks/common`.
This triggers either `build-branches.sh`, `build-latest.sh` or directly `build.sh`.
for `build`, `test` and `push`](overwrite). See `/hooks/*`.
2. Shared functionality of the scripts `build`, `test` and `push` is extracted to `/hooks/common`.
3. The `build` script runs `run_build()` from `/hooks/common`.
This triggers either `/build-branches.sh`, `/build-latest.sh` or directly `/build.sh`.
4. The `test` script just invokes `docker-compose` commands.
5. The `push` script runs `run_build()` from `hooks/common` with a `--push-only` flag.
This causes the `build.sh` script to not re-build the Docker image, but just the just built image.
The _Docker Tag_ configuration setting is misused to select the type (_release_, _prerelease_, _branches_) of the build as well as the variant (_main_, _ldap_).
The _Dockerfile location_ configuration setting is completely ignored by the build scripts.
The _Docker Tag_ configuration setting (`$DOCKER_TAG`) is only used to select the type (_release_, _prerelease_, _branches_) of the build in `hooks/common`.
Because it has a different meaning in all the other build scripts, it is `unset` after it has served it's purpose.
[overwrite]: https://docs.docker.com/docker-hub/builds/advanced/#override-build-test-or-push-commands

View File

@ -1,4 +1,5 @@
FROM python:3.7-alpine3.10
ARG FROM=python:3.7-alpine
FROM ${FROM} as builder
RUN apk add --no-cache \
bash \
@ -7,51 +8,55 @@ RUN apk add --no-cache \
cyrus-sasl-dev \
graphviz \
jpeg-dev \
libevent-dev \
libffi-dev \
libxml2-dev \
libxslt-dev \
openldap-dev \
postgresql-dev \
ttf-ubuntu-font-family \
wget
postgresql-dev
RUN pip install \
WORKDIR /install
RUN pip install --install-option="--prefix=/install" \
# gunicorn is used for launching netbox
gunicorn \
greenlet \
eventlet \
# napalm is used for gathering information from network devices
napalm \
# ruamel is used in startup_scripts
'ruamel.yaml>=0.15,<0.16' \
# pinning django to the version required by netbox
# adding it here, to install the correct version of
# django-rq
'Django>=2.2,<2.3' \
# django-rq is used for webhooks
django-rq
# django_auth_ldap is required for ldap
django_auth_ldap
ARG BRANCH=master
ARG NETBOX_PATH
COPY ${NETBOX_PATH}/requirements.txt /
RUN pip install --install-option="--prefix=/install" -r /requirements.txt
WORKDIR /tmp
###
# Main stage
###
# As the requirements don't change very often,
# and as they take some time to compile,
# we try to cache them very agressively.
ARG REQUIREMENTS_URL=https://raw.githubusercontent.com/netbox-community/netbox/$BRANCH/requirements.txt
ADD ${REQUIREMENTS_URL} requirements.txt
RUN pip install -r requirements.txt
ARG FROM
FROM ${FROM} as main
# Cache bust when the upstream branch changes:
# ADD will fetch the file and check if it has changed
# If not, Docker will use the existing build cache.
# If yes, Docker will bust the cache and run every build step from here on.
ARG REF_URL=https://api.github.com/repos/netbox-community/netbox/contents?ref=$BRANCH
ADD ${REF_URL} version.json
RUN apk add --no-cache \
bash \
ca-certificates \
graphviz \
libevent \
libffi \
libjpeg-turbo \
libressl \
libxslt \
postgresql-libs \
ttf-ubuntu-font-family
WORKDIR /opt
ARG URL=https://github.com/netbox-community/netbox/archive/$BRANCH.tar.gz
RUN wget -q -O - "${URL}" | tar xz \
&& mv netbox* netbox
COPY --from=builder /install /usr/local
ARG NETBOX_PATH
COPY ${NETBOX_PATH} /opt/netbox
COPY docker/configuration.docker.py /opt/netbox/netbox/netbox/configuration.py
COPY configuration/gunicorn_config.py /etc/netbox/config/
@ -67,7 +72,22 @@ ENTRYPOINT [ "/opt/netbox/docker-entrypoint.sh" ]
CMD ["gunicorn", "-c /etc/netbox/config/gunicorn_config.py", "netbox.wsgi"]
LABEL SRC_URL="$URL"
LABEL NETBOX_DOCKER_PROJECT_VERSION="custom build" \
NETBOX_BRANCH="custom build" \
ORIGINAL_DOCKER_TAG="custom build" \
NETBOX_GIT_COMMIT="not built from git" \
NETBOX_GIT_URL="not built from git"
ARG NETBOX_DOCKER_PROJECT_VERSION=snapshot
LABEL NETBOX_DOCKER_PROJECT_VERSION="$NETBOX_DOCKER_PROJECT_VERSION"
#####
## LDAP specific configuration
#####
FROM main as ldap
RUN apk add --no-cache \
libsasl \
libldap \
util-linux
COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py
COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py

View File

@ -1,9 +0,0 @@
ARG DOCKER_ORG=netboxcommunity
ARG DOCKER_REPO=netbox
ARG FROM_TAG=latest
FROM $DOCKER_ORG/$DOCKER_REPO:$FROM_TAG
RUN pip install django_auth_ldap
COPY docker/ldap_config.docker.py /opt/netbox/netbox/netbox/ldap_config.py
COPY configuration/ldap_config.py /etc/netbox/config/ldap_config.py

View File

@ -49,8 +49,8 @@ Default credentials:
This project relies only on *Docker* and *docker-compose* meeting this requirements:
* The *Docker version* must be at least `1.13.0`.
* The *docker-compose version* must be at least `1.10.0`.
* The *Docker version* must be at least `17.05`.
* The *docker-compose version* must be at least `1.17.0`.
To ensure this, compare the output of `docker --version` and `docker-compose --version` with the requirements above.
@ -69,24 +69,20 @@ To use this feature, set the environment-variable `VERSION` before launching `do
[any tag of the `netboxcommunity/netbox` Docker image on Docker Hub][netbox-dockerhub].
```bash
export VERSION=v2.2.6
export VERSION=v2.6.6
docker-compose pull netbox
docker-compose up -d
```
You can also build a specific version of the Netbox image. This time, `VERSION` indicates any valid
[Git Reference][git-ref] declared on [the 'netbox-community/netbox' Github repository][netbox-github].
Most commonly you will specify a tag or branch name.
You can also build a specific version of the Netbox Docker image yourself.
`VERSION` can be any valid [git ref][git-ref] in that case.
```bash
export VERSION=develop
docker-compose build --no-cache netbox
export VERSION=v2.6.6
./build.sh $VERSION
docker-compose up -d
```
Hint: If you're building a specific version by tag name, the `--no-cache` argument is not strictly necessary.
This can increase the build speed if you're just adjusting the config, for example.
[git-ref]: https://git-scm.com/book/en/v2/Git-Internals-Git-References
[netbox-github]: https://github.com/netbox-community/netbox/releases

View File

@ -12,73 +12,40 @@ BUILDS=("${BUILD:-"${ALL_BUILDS[@]}"}")
echo "⚙️ Configured builds: ${BUILDS[*]}"
VARIANTS=("main" "ldap")
if [ -n "${DEBUG}" ]; then
export DEBUG
fi
ERROR=0
# Don't build if not on `master` and don't build if on a pull request,
# but build when DEBUG is not empty
for VARIANT in "${VARIANTS[@]}"; do
export VARIANT
# Checking which VARIANT to build
if [ "${VARIANT}" == "main" ]; then
DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}"
else
DOCKERFILE="${DOCKERFILE_PATH-Dockerfile}.${VARIANT}"
# Fail fast
if [ ! -f "${DOCKERFILE}" ]; then
echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist."
ERROR=1
for BUILD in "${BUILDS[@]}"; do
echo "🛠 Building '$BUILD' from '$DOCKERFILE'"
case $BUILD in
release)
# build the latest release
# shellcheck disable=SC2068
./build-latest.sh $@ || ERROR=1
;;
prerelease)
# build the latest pre-release
# shellcheck disable=SC2068
PRERELEASE=true ./build-latest.sh $@ || ERROR=1
;;
branches)
# build all branches
# shellcheck disable=SC2068
./build-branches.sh $@ || ERROR=1
;;
*)
echo "🚨 Unrecognized build '$BUILD'."
if [ -z "$DEBUG" ]; then
continue
exit 1
else
echo "⚠️ Would skip this, but DEBUG is enabled."
echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
fi
fi
fi
for BUILD in "${BUILDS[@]}"; do
echo "🛠 Building '$BUILD' from '$DOCKERFILE'"
case $BUILD in
release)
# build the latest release
# shellcheck disable=SC2068
./build-latest.sh $@ || ERROR=1
;;
prerelease)
# build the latest pre-release
# shellcheck disable=SC2068
PRERELEASE=true ./build-latest.sh $@ || ERROR=1
;;
branches)
# build all branches
# shellcheck disable=SC2068
./build-branches.sh $@ || ERROR=1
;;
special)
# special build
# shellcheck disable=SC2068
#SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@ || ERROR=1
echo "✅ No special builds today."
;;
*)
echo "🚨 Unrecognized build '$BUILD'."
if [ -z "$DEBUG" ]; then
exit 1
else
echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
fi
;;
esac
done
;;
esac
done
exit $ERROR

View File

@ -3,6 +3,10 @@
echo "▶️ $0 $*"
###
# Checking for the presence of GITHUB_OAUTH_CLIENT_ID
# and GITHUB_OAUTH_CLIENT_SECRET
###
if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
echo "🗝 Performing authenticated Github API calls."
GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
@ -11,18 +15,33 @@ else
GITHUB_OAUTH_PARAMS=""
fi
###
# Calling Github to get the all branches
###
ORIGINAL_GITHUB_REPO="${SRC_ORG-netbox-community}/${SRC_REPO-netbox}"
GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}"
# Composing the JQ commans to extract the most recent version number
JQ_BRANCHES='map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string'
CURL="curl -sS"
BRANCHES=$($CURL "${URL_RELEASES}" | jq -r 'map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string')
# Querying the Github API to fetch all branches
BRANCHES=$($CURL "${URL_RELEASES}" | jq -r "$JQ_BRANCHES")
###
# Building each branch
###
# keeping track whether an error occured
ERROR=0
# calling build.sh for each branch
for BRANCH in $BRANCHES; do
# shellcheck disable=SC2068
./build.sh "${BRANCH}" $@ || ERROR=1
done
# returning whether an error occured
exit $ERROR

View File

@ -3,6 +3,10 @@
echo "▶️ $0 $*"
###
# Checking for the presence of GITHUB_OAUTH_CLIENT_ID
# and GITHUB_OAUTH_CLIENT_SECRET
###
if [ -n "${GITHUB_OAUTH_CLIENT_ID}" ] && [ -n "${GITHUB_OAUTH_CLIENT_SECRET}" ]; then
echo "🗝 Performing authenticated Github API calls."
GITHUB_OAUTH_PARAMS="client_id=${GITHUB_OAUTH_CLIENT_ID}&client_secret=${GITHUB_OAUTH_CLIENT_SECRET}"
@ -11,17 +15,38 @@ else
GITHUB_OAUTH_PARAMS=""
fi
###
# Checking if PRERELEASE is either unset, 'true' or 'false'
###
if [ -n "${PRERELEASE}" ] &&
{ [ "${PRERELEASE}" != "true" ] && [ "${PRERELEASE}" != "false" ]; }; then
if [ -z "${DEBUG}" ]; then
echo "⚠️ PRERELEASE must be either unset, 'true' or 'false', but was '${PRERELEASE}'!"
exit 1
else
echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
fi
fi
###
# Calling Github to get the latest version
###
ORIGINAL_GITHUB_REPO="netbox-community/netbox"
GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/releases?${GITHUB_OAUTH_PARAMS}"
# Composing the JQ commans to extract the most recent version number
JQ_LATEST="group_by(.prerelease) | .[] | sort_by(.published_at) | reverse | .[0] | select(.prerelease==${PRERELEASE-false}) | .tag_name"
CURL="curl -sS"
# Querying the Github API to fetch the most recent version number
VERSION=$($CURL "${URL_RELEASES}" | jq -r "${JQ_LATEST}")
###
# Check if the prerelease version is actually higher than stable version
###
if [ "${PRERELEASE}" == "true" ]; then
JQ_STABLE="group_by(.prerelease) | .[] | sort_by(.published_at) | reverse | .[0] | select(.prerelease==false) | .tag_name"
STABLE_VERSION=$($CURL "${URL_RELEASES}" | jq -r "${JQ_STABLE}")
@ -35,9 +60,11 @@ if [ "${PRERELEASE}" == "true" ]; then
# shellcheck disable=SC2003
MINOR_UNSTABLE=$(expr match "${VERSION}" 'v[0-9]\+\.\([0-9]\+\)')
if ( [ "$MAJOR_STABLE" -eq "$MAJOR_UNSTABLE" ] && [ "$MINOR_STABLE" -ge "$MINOR_UNSTABLE" ] ) \
|| [ "$MAJOR_STABLE" -gt "$MAJOR_UNSTABLE" ]; then
echo "❎ Latest unstable version ('$VERSION') is not higher than the latest stable version ('$STABLE_VERSION')."
if { [ "${MAJOR_STABLE}" -eq "${MAJOR_UNSTABLE}" ] \
&& [ "${MINOR_STABLE}" -ge "${MINOR_UNSTABLE}" ];
} || [ "${MAJOR_STABLE}" -gt "${MAJOR_UNSTABLE}" ]; then
echo "❎ Latest unstable version '${VERSION}' is not higher than the latest stable version '$STABLE_VERSION'."
if [ -z "$DEBUG" ]; then
exit 0
else
@ -46,32 +73,6 @@ if [ "${PRERELEASE}" == "true" ]; then
fi
fi
# Check if that version is not already available on docker hub:
ORIGINAL_DOCKERHUB_REPO="${DOCKER_ORG-netboxcommunity}/${DOCKER_REPO-netbox}"
DOCKERHUB_REPO="${DOCKERHUB_REPO-$ORIGINAL_DOCKERHUB_REPO}"
URL_DOCKERHUB_TOKEN="https://auth.docker.io/token?service=registry.docker.io&scope=repository:${DOCKERHUB_REPO}:pull"
BEARER_TOKEN="$($CURL "${URL_DOCKERHUB_TOKEN}" | jq -r .token)"
URL_DOCKERHUB_TAG="https://registry.hub.docker.com/v2/${DOCKERHUB_REPO}/tags/list"
AUTHORIZATION_HEADER="Authorization: Bearer ${BEARER_TOKEN}"
if [ -z "$VARIANT" ] || [ "$VARIANT" == "main" ]; then
DOCKER_TAG="${VERSION}"
else
DOCKER_TAG="${VERSION}-${VARIANT}"
fi
ALREADY_BUILT="$($CURL -H "${AUTHORIZATION_HEADER}" "${URL_DOCKERHUB_TAG}" | jq -e ".tags | any(.==\"${DOCKER_TAG}\")")"
if [ -n "$DEBUG" ] || [ "$ALREADY_BUILT" == "false" ]; then
if [ -n "$DEBUG" ]; then
echo "⚠️ Would not build, because ${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}, but DEBUG is enabled."
fi
# shellcheck disable=SC2068
./build.sh "${VERSION}" $@
exit $?
else
echo "${DOCKER_TAG} already exists on https://hub.docker.com/r/${DOCKERHUB_REPO}"
exit 0
fi
# shellcheck disable=SC2068
./build.sh "${VERSION}" $@
exit $?

350
build.sh
View File

@ -1,5 +1,5 @@
#!/bin/bash
# Builds the Dockerfile[.variant] and injects tgz'ed Netbox code from Github
# Clones the Netbox repository with git from Github and builds the Dockerfile
echo "▶️ $0 $*"
@ -8,66 +8,78 @@ set -e
if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
echo "Usage: ${0} <branch> [--push|--push-only]"
echo " branch The branch or tag to build. Required."
echo " --push Pushes built the Docker image to the registry."
echo " --push-only Does not build. Only pushes the Docker image to the registry."
echo " --push Pushes the built Docker image to the registry."
echo " --push-only Only pushes the Docker image to the registry, but does not build it."
echo ""
echo "You can use the following ENV variables to customize the build:"
echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied."
echo " DRY_RUN Prints all build statements instead of running them."
echo " DOCKER_OPTS Add parameters to Docker."
echo " Default:"
echo " When <TAG> starts with 'v': \"\""
echo " Else: \"--no-cache\""
echo " BRANCH The branch to build."
echo " Also used for tagging the image."
echo " TAG The version part of the docker tag."
echo " Default:"
echo " When <BRANCH>=master: latest"
echo " When <BRANCH>=develop: snapshot"
echo " Else: same as <BRANCH>"
echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r/<DOCKER_ORG>/<DOCKER_REPO>) "
echo " Also used for tagging the image."
echo " Default: netboxcommunity"
echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r/<DOCKER_ORG>/<DOCKER_REPO>) "
echo " Also used for tagging the image."
echo " Default: netbox"
echo " DOCKER_TAG The name of the tag which is applied to the image."
echo " Useful for pushing into another registry than hub.docker.com."
echo " Default: <DOCKER_ORG>/<DOCKER_REPO>:<BRANCH>"
echo " DOCKER_SHORT_TAG The name of the short tag which is applied to the image."
echo " This is used to tag all patch releases to their containing version e.g. v2.5.1 -> v2.5"
echo " Default: <DOCKER_ORG>/<DOCKER_REPO>:\$MAJOR.\$MINOR"
echo " SRC_ORG Which fork of netbox to use (i.e. github.com/<SRC_ORG>/<SRC_REPO>)."
echo " Default: netbox-community"
echo " SRC_REPO The name of the netbox for to use (i.e. github.com/<SRC_ORG>/<SRC_REPO>)."
echo " Default: netbox"
echo " URL Where to fetch the package from."
echo " Must be a tar.gz file of the source code."
echo " Default: https://github.com/<SRC_ORG>/<SRC_REPO>/archive/\$BRANCH.tar.gz"
echo " VARIANT The variant to build."
echo " The value will be used as a suffix to the \$TAG and for the Dockerfile"
echo " selection. The TAG being build must exist for the base variant and"
echo " corresponding Dockerfile must start with the following lines:"
echo " ARG DOCKER_ORG=netboxcommunity"
echo " ARG DOCKER_REPO=netbox"
echo " ARG FROM_TAG=latest"
echo " FROM \$DOCKER_ORG/\$DOCKER_REPO:\$FROM_TAG"
echo " Example: VARIANT=ldap will result in the tag 'latest-ldap' and the"
echo " Dockerfile './Dockerfile.ldap' being used."
echo " Exception: VARIANT=main will use the './Dockerfile' Dockerfile"
echo " Default: main"
echo " HTTP_PROXY The proxy to use for http requests."
echo " Example: http://proxy.domain.tld:3128"
echo " Default: empty"
echo " HTTPS_PROXY The proxy to use for https requests."
echo " Example: http://proxy.domain.tld:3128"
echo " Default: empty"
echo " FTP_PROXY The proxy to use for ftp requests."
echo " Example: http://proxy.domain.tld:3128"
echo " Default: empty"
echo " NO_PROXY Comma-separated list of domain extensions proxy should not be used for."
echo " Example: .domain1.tld,.domain2.tld"
echo " Default: empty"
echo " SRC_ORG Which fork of netbox to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})."
echo " Default: netbox-community"
echo " SRC_REPO The name of the repository to use (i.e. github.com/\${SRC_ORG}/\${SRC_REPO})."
echo " Default: netbox"
echo " URL Where to fetch the code from."
echo " Must be a git repository. Can be private."
echo " Default: https://github.com/\${SRC_ORG}/\${SRC_REPO}.git"
echo " NETBOX_PATH The path where netbox will be checkout out."
echo " Must not be outside of the netbox-docker repository (because of Docker)!"
echo " Default: .netbox"
echo " SKIP_GIT If defined, git is not invoked and \${NETBOX_PATH} will not be altered."
echo " This may be useful, if you are manually managing the NETBOX_PATH."
echo " Default: undefined"
echo " TAG The version part of the docker tag."
echo " Default:"
echo " When \${BRANCH}=master: latest"
echo " When \${BRANCH}=develop: snapshot"
echo " Else: same as \${BRANCH}"
echo " DOCKER_ORG The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})"
echo " Also used for tagging the image."
echo " Default: netboxcommunity"
echo " DOCKER_REPO The Docker registry (i.e. hub.docker.com/r/\${DOCKER_ORG}/\${DOCKER_REPO})"
echo " Also used for tagging the image."
echo " Default: netbox"
echo " DOCKER_FROM The base image to use."
echo " Default: Whatever is defined as default in the Dockerfile."
echo " DOCKER_TAG The name of the tag which is applied to the image."
echo " Useful for pushing into another registry than hub.docker.com."
echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:\${TAG}"
echo " DOCKER_SHORT_TAG The name of the short tag which is applied to the"
echo " image. This is used to tag all patch releases to their"
echo " containing version e.g. v2.5.1 -> v2.5"
echo " Default: \${DOCKER_ORG}/\${DOCKER_REPO}:<MAJOR>.<MINOR>"
echo " DOCKERFILE The name of Dockerfile to use."
echo " Default: Dockerfile"
echo " DOCKER_TARGET A specific target to build."
echo " It's currently not possible to pass multiple targets."
echo " Default: main ldap"
echo " HTTP_PROXY The proxy to use for http requests."
echo " Example: http://proxy.domain.tld:3128"
echo " Default: undefined"
echo " NO_PROXY Comma-separated list of domain extensions proxy should not be used for."
echo " Example: .domain1.tld,.domain2.tld"
echo " Default: undefined"
echo " DEBUG If defined, the script does not stop when certain checks are unsatisfied."
echo " Default: undefined"
echo " DRY_RUN Prints all build statements instead of running them."
echo " Default: undefined"
echo ""
echo "Examples:"
echo " ${0} master"
echo " This will fetch the latest 'master' branch, build a Docker Image and tag it"
echo " 'netboxcommunity/netbox:latest'."
echo " ${0} develop"
echo " This will fetch the latest 'develop' branch, build a Docker Image and tag it"
echo " 'netboxcommunity/netbox:snapshot'."
echo " ${0} v2.6.6"
echo " This will fetch the 'v2.6.6' tag, build a Docker Image and tag it"
echo " 'netboxcommunity/netbox:v2.6.6' and 'netboxcommunity/netbox:v2.6'."
echo " ${0} develop-2.7"
echo " This will fetch the 'develop-2.7' branch, build a Docker Image and tag it"
echo " 'netboxcommunity/netbox:develop-2.7'."
echo " SRC_ORG=cimnine ${0} feature-x"
echo " This will fetch the 'feature-x' branch from https://github.com/cimnine/netbox.git,"
echo " build a Docker Image and tag it 'netboxcommunity/netbox:feature-x'."
echo " SRC_ORG=cimnine DOCKER_ORG=cimnine ${0} feature-x"
echo " This will fetch the 'feature-x' branch from https://github.com/cimnine/netbox.git,"
echo " build a Docker Image and tag it 'cimnine/netbox:feature-x'."
if [ "${1}x" == "x" ]; then
exit 1
@ -76,36 +88,73 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
fi
fi
# read the project version and trim it
###
# Determining the build command to use
###
if [ -z "${DRY_RUN}" ]; then
DRY=""
else
echo "⚠️ DRY_RUN MODE ON ⚠️"
DRY="echo"
fi
###
# read the project version from the `VERSION` file and trim it
# see https://stackoverflow.com/a/3232433/172132
###
NETBOX_DOCKER_PROJECT_VERSION="${NETBOX_DOCKER_PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' VERSION)}"
###
# variables for fetching the source
###
SRC_ORG="${SRC_ORG-netbox-community}"
SRC_REPO="${SRC_REPO-netbox}"
BRANCH="${1}"
URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}/archive/$BRANCH.tar.gz}"
URL="${URL-https://github.com/${SRC_ORG}/${SRC_REPO}.git}"
# Checking which VARIANT to build
VARIANT="${VARIANT-main}"
if [ "$VARIANT" == "main" ]; then
DOCKERFILE="Dockerfile"
else
DOCKERFILE="Dockerfile.${VARIANT}"
###
# fetching the source
###
if [ "${2}" != "--push-only" ] ; then
NETBOX_PATH="${NETBOX_PATH-.netbox}"
echo "🌐 Checking out '${BRANCH}' of netbox from the url '${URL}' into '${NETBOX_PATH}'"
if [ ! -d "${NETBOX_PATH}" ]; then
$DRY git clone -q --depth 10 -b "${BRANCH}" "${URL}" "${NETBOX_PATH}"
fi
(
$DRY cd "${NETBOX_PATH}"
if [ -n "${HTTP_PROXY}" ]; then
git config http.proxy "${HTTP_PROXY}"
fi
$DRY git remote set-url origin "${URL}"
$DRY git fetch -qp --depth 10 origin "${BRANCH}"
$DRY git checkout -qf FETCH_HEAD
$DRY git prune
)
echo "✅ Checked out netbox"
fi
# Fail fast
###
# Determining the value for DOCKERFILE
# and checking whether it exists
###
DOCKERFILE="${DOCKERFILE-Dockerfile}"
if [ ! -f "${DOCKERFILE}" ]; then
echo "🚨 The Dockerfile ${DOCKERFILE} for variant '${VARIANT}' doesn't exist."
echo "🚨 The Dockerfile ${DOCKERFILE} doesn't exist."
if [ -z "$DEBUG" ]; then
if [ -z "${DEBUG}" ]; then
exit 1
else
echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
fi
fi
###
# variables for tagging the docker image
###
DOCKER_ORG="${DOCKER_ORG-netboxcommunity}"
DOCKER_REPO="${DOCKER_REPO-netbox}"
case "${BRANCH}" in
@ -117,81 +166,106 @@ case "${BRANCH}" in
TAG="${TAG-$BRANCH}";;
esac
DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}"
if [ "$VARIANT" != "main" ]; then
DOCKER_TAG="${DOCKER_TAG}-${VARIANT}"
fi
###
# Determine targets to build
###
DEFAULT_DOCKER_TARGETS=("main" "ldap")
DOCKER_TARGETS=( "${DOCKER_TARGET:-"${DEFAULT_DOCKER_TARGETS[@]}"}")
echo "🏭 Building the following targets:" "${DOCKER_TARGETS[@]}"
if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then
MAJOR=${BASH_REMATCH[1]}
MINOR=${BASH_REMATCH[2]}
###
# Build each target
###
for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
echo "🏗 Building the target '${DOCKER_TARGET}'"
DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}"
if [ "$VARIANT" != "main" ]; then
DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${VARIANT}"
###
# composing the final TARGET_DOCKER_TAG
###
TARGET_DOCKER_TAG="${DOCKER_TAG-${DOCKER_ORG}/${DOCKER_REPO}:${TAG}}"
if [ "${DOCKER_TARGET}" != "main" ]; then
TARGET_DOCKER_TAG="${TARGET_DOCKER_TAG}-${DOCKER_TARGET}"
fi
fi
DOCKER_OPTS=("${DOCKER_OPTS[@]}")
###
# composing the additional DOCKER_SHORT_TAG,
# i.e. "v2.6.1" becomes "v2.6",
# which is only relevant for version tags
###
if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then
MAJOR=${BASH_REMATCH[1]}
MINOR=${BASH_REMATCH[2]}
# caching is only ok for version tags
case "${TAG}" in
v*) ;;
*) DOCKER_OPTS+=( "--no-cache" ) ;;
esac
DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}"
DOCKER_OPTS+=( "--pull" )
# Build args
DOCKER_BUILD_ARGS=(
--build-arg "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}"
--build-arg "FROM_TAG=${TAG}"
--build-arg "BRANCH=${BRANCH}"
--build-arg "URL=${URL}"
--build-arg "DOCKER_ORG=${DOCKER_ORG}"
--build-arg "DOCKER_REPO=${DOCKER_REPO}"
)
if [ -n "$HTTP_PROXY" ]; then
DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" )
fi
if [ -n "$HTTPS_PROXY" ]; then
DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" )
fi
if [ -n "$FTP_PROXY" ]; then
DOCKER_BUILD_ARGS+=( --build-arg "ftp_proxy=${FTP_PROXY}" )
fi
if [ -n "$NO_PROXY" ]; then
DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" )
fi
if [ -z "$DRY_RUN" ]; then
DOCKER_CMD="docker"
else
echo "⚠️ DRY_RUN MODE ON ⚠️"
DOCKER_CMD="echo docker"
fi
if [ "${2}" != "--push-only" ] ; then
echo "🐳 Building the Docker image '${DOCKER_TAG}' from the url '${URL}'."
$DOCKER_CMD build -t "${DOCKER_TAG}" "${DOCKER_BUILD_ARGS[@]}" "${DOCKER_OPTS[@]}" -f "${DOCKERFILE}" .
echo "✅ Finished building the Docker images '${DOCKER_TAG}'"
if [ -n "$DOCKER_SHORT_TAG" ]; then
echo "🐳 Tagging image '${DOCKER_SHORT_TAG}'."
$DOCKER_CMD tag "${DOCKER_TAG}" "${DOCKER_SHORT_TAG}"
echo "✅ Tagged image '${DOCKER_SHORT_TAG}'"
if [ "${DOCKER_TARGET}" != "main" ]; then
DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG}-${DOCKER_TARGET}"
fi
fi
fi
if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then
echo "⏫ Pushing '${DOCKER_TAG}"
$DOCKER_CMD push "${DOCKER_TAG}"
echo "✅ Finished pushing the Docker image '${DOCKER_TAG}'."
###
# Proceeding to buils stage, except if `--push-only` is passed
###
if [ "${2}" != "--push-only" ] ; then
###
# Composing all arguments for `docker build`
###
DOCKER_BUILD_ARGS=(
--pull
--target "${DOCKER_TARGET}"
-f "${DOCKERFILE}"
-t "${TARGET_DOCKER_TAG}"
)
if [ -n "${DOCKER_SHORT_TAG}" ]; then
DOCKER_BUILD_ARGS+=( -t "${DOCKER_SHORT_TAG}" )
fi
if [ -n "$DOCKER_SHORT_TAG" ]; then
echo "⏫ Pushing '${DOCKER_SHORT_TAG}'"
$DOCKER_CMD push "${DOCKER_SHORT_TAG}"
echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'."
# --label
DOCKER_BUILD_ARGS+=(
--label "NETBOX_DOCKER_PROJECT_VERSION=${NETBOX_DOCKER_PROJECT_VERSION}"
--label "NETBOX_BRANCH=${BRANCH}"
--label "ORIGINAL_DOCKER_TAG=${TARGET_DOCKER_TAG}"
)
if [ -d "${NETBOX_PATH}/.git" ]; then
DOCKER_BUILD_ARGS+=(
--label "NETBOX_GIT_COMMIT=$($DRY cd "${NETBOX_PATH}"; $DRY git rev-parse HEAD)"
--label "NETBOX_GIT_URL=$($DRY cd "${NETBOX_PATH}"; $DRY git remote get-url origin)"
)
fi
# --build-arg
DOCKER_BUILD_ARGS+=( --build-arg "NETBOX_PATH=${NETBOX_PATH}" )
if [ -n "${DOCKER_FROM}" ]; then
DOCKER_BUILD_ARGS+=( --build-arg "FROM=${DOCKER_FROM}" )
fi
if [ -n "${HTTP_PROXY}" ]; then
DOCKER_BUILD_ARGS+=( --build-arg "http_proxy=${HTTP_PROXY}" )
DOCKER_BUILD_ARGS+=( --build-arg "https_proxy=${HTTPS_PROXY}" )
fi
if [ -n "${NO_PROXY}" ]; then
DOCKER_BUILD_ARGS+=( --build-arg "no_proxy=${NO_PROXY}" )
fi
###
# Building the docker image
###
echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'."
$DRY docker build "${DOCKER_BUILD_ARGS[@]}" .
echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'"
fi
fi
###
# Pushing the docker images if either `--push` or `--push-only` are passed
###
if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then
echo "⏫ Pushing '${TARGET_DOCKER_TAG}"
$DRY docker push "${TARGET_DOCKER_TAG}"
echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_TAG}'."
if [ -n "$DOCKER_SHORT_TAG" ]; then
echo "⏫ Pushing '${DOCKER_SHORT_TAG}'"
$DRY docker push "${DOCKER_SHORT_TAG}"
echo "✅ Finished pushing the Docker image '${DOCKER_SHORT_TAG}'."
fi
fi
done

View File

@ -1,10 +1,6 @@
version: '3'
version: '3.4'
services:
netbox: &netbox
build:
context: .
args:
- BRANCH=${VERSION-master}
image: netboxcommunity/netbox:${VERSION-latest}
depends_on:
- postgres

View File

@ -2,4 +2,5 @@
. hooks/common
# shellcheck disable=SC2119
run_build

View File

@ -1,37 +1,19 @@
#!/bin/bash
ensure_jq() {
echo "🛠🛠🛠 Installing JQ via apt-get"
[ -x "$(command -v jq)" ] || ( apt-get update && apt-get install -y jq )
}
ensure_dockerfile_present() {
if [ "${VARIANT}" == "main" ]; then
DOCKERFILE="Dockerfile"
else
DOCKERFILE="Dockerfile.${VARIANT}"
# Fail fast
if [ ! -f "${DOCKERFILE}" ]; then
echo "🚨 The Dockerfile '${DOCKERFILE}' for variant '${VARIANT}' doesn't exist."
if [ -z "$DEBUG" ]; then
exit 1
else
echo "⚠️ Would skip this, but DEBUG is enabled."
fi
fi
if [ "${DOCKERFILE}" != "${DOCKERFILE_PATH}" ]; then
echo "⚠️ The specified Dockerfile '${DOCKERFILE_PATH}' does not match the expected Dockerfile '${DOCKERFILE}'."
echo " This script will use '${DOCKERFILE}' and ignore '${DOCKERFILE_PATH}'."
if [ ! -x "$(command -v jq)" ]; then
if [ -x "$(command -v apt-get)" ]; then
echo "🛠🛠🛠 Installing 'jq' via 'apt-get'"
apt-get update && apt-get install -y jq
else
echo "⚠️⚠️⚠️ apt-get not found, unable to automatically install 'jq'."
fi
fi
}
# Passes args to the scripts
run_build() {
echo "🐳🐳🐳 Building '${BUILD}' images, the '${VARIANT:-main}' variant"
echo "🐳🐳🐳 Building '${BUILD}' images"
case $BUILD in
release)
# build the latest release
@ -48,11 +30,11 @@ run_build() {
# shellcheck disable=SC2068
./build-branches.sh $@
;;
special)
# special build
# shellcheck disable=SC2068
#SRC_ORG=lampwins TAG=webhooks-backend ./build.sh "feature/webhooks-backend" $@
echo "✅ No special builds today."
this) # Pull Requests
# only build the 'master' branch
# (resulting in the 'latest' docker tag)
# and the 'main' target.
DOCKER_TARGET=main ./build.sh master
;;
*)
echo "🚨 Unrecognized build '$BUILD'."
@ -70,13 +52,9 @@ echo "🤖🤖🤖 Preparing build"
export DOCKER_ORG="index.docker.io/netboxcommunity"
export DOCKER_REPO=netbox
export DOCKERHUB_REPO=netboxcommunity/netbox
# mis-using the "${DOCKER_TAG}" variable as "branch to build"
export BUILD="${DOCKER_TAG%-*}"
export VARIANT="${DOCKER_TAG#*-}"
# shellcheck disable=SC2153
export BUILD="${DOCKER_TAG}"
unset DOCKER_TAG
ensure_dockerfile_present
ensure_jq

View File

@ -2,4 +2,13 @@
. hooks/common
run_build --push-only
if [ "${SOURCE_BRANCH}" == "main" ] || [ "${DEBUG}" == "true" ]; then
if [ "${SOURCE_BRANCH}" != "main" ]; then
echo "⚠️⚠️⚠️ Would exit, but DEBUG is '${DEBUG}'".
fi
run_build --push-only
else
echo "⚠️⚠️⚠️ Only pushing on 'main' branch, but current branch is '${SOURCE_BRANCH}'"
exit 0
fi

View File

@ -2,11 +2,21 @@
. hooks/common
if [ "${VARIANT}" == "main" ] && [ "${BUILD}" == "BRANCHES" ]; then
echo "🐳🐳🐳 Testing"
docker-compose pull --parallel
docker-compose build
docker-compose run netbox ./manage.py test
run_test() {
echo "🐳🐳🐳 Testing '${1}'"
VERSION="${1}" docker-compose run netbox ./manage.py test
docker-compose down -v
echo "🐳🐳🐳 Done testing '${1}'"
}
# test on builds of 'branches'
if [ "${BUILD}" == "branches" ] \
|| [ "${DEBUG}" == "true" ]; then
run_test latest
run_test snapshot
# test on bulds of 'this' (i.e. pull request)
elif [ "${BUILD}" == "this" ]; then
run_test latest
else
echo "🐳🐳🐳 No tests are implemented for build '${BUILD}' with variant '${VARIANT}'."
echo "🐳🐳🐳 No tests are implemented for build '${BUILD}'."
fi