Merge pull request #274 from tobiasge/optimise-builds

Optimise builds
This commit is contained in:
Tobias Genannt 2020-05-15 13:57:43 +02:00 committed by GitHub
commit a6584d2874
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 180 additions and 118 deletions

View File

@ -15,7 +15,8 @@ jobs:
build_cmd:
- ./build-latest.sh
- PRERELEASE=true ./build-latest.sh
- ./build-branches.sh
- ./build-next.sh
- ./build.sh develop
docker_from:
- '' # use the default of the DOCKERFILE
- python:3.7-alpine

View File

@ -14,14 +14,15 @@ jobs:
build_cmd:
- ./build-latest.sh
- PRERELEASE=true ./build-latest.sh
- ./build-branches.sh
- ./build-next.sh
- ./build.sh develop
fail-fast: false
runs-on: ubuntu-latest
name: Builds new Netbox Docker Images
steps:
- id: git-checkout
name: Checkout
uses: actions/checkout@v1
uses: actions/checkout@v2
- id: docker-build
name: Build the image with '${{ matrix.build_cmd }}'
run: ${{ matrix.build_cmd }}

View File

@ -1,32 +0,0 @@
sudo: required
language: python
env:
- BUILD=release
- BUILD=prerelease
- BUILD=branches
- BUILD=special
git:
depth: 5
services:
- docker
install:
- docker-compose pull --parallel
- docker-compose build
script:
- docker-compose run netbox ./manage.py test
after_script:
- docker-compose down
after_success:
- docker login -u="$DOCKER_USERNAME" -p="$DOCKER_PASSWORD"
- ./build-all.sh --push
notifications:
slack:
secure: F3VsWcvU/XYyjGjU8ZAVGpREe7F1NjKq6LuMRzhQORbXUvanxDQtLzEe0Y5zm/6+gHkx6t8cX/v2PiCI+v46pkapYMUimd+QEOL1WxbUdnw2kQxcgw/R3wX34l2FHXbG3/a+TmH3euqbSCTIrPy9ufju948i+Q0E0u0fyInmozl8qOT23C4joQOpVAq7y+wHxTxsEg46ZzL2Ties+dmqjMsvHocv7mPI2IWzAWA8SJZxS82Amoapww++QjgEmoY+tMimLkdeXCRgeoj41UGHDg54rbEXh/PTaWiuzyzTr1WLmsGRScC57fDRivp3mSM37/MlNxsRj1z+j4zrvWFQgNfJ2yMjBHroc1jOX/uCY4dwbpSPqUCpc4idMGCGZFItgzTQ3lAPYAsom0C6n8C08Xk8EsNKWwXrDSd4ZUhIwptkNPCFK+kXbLFsMzSApnaBYW0T+wba57nZdiWjOPYmvJr49MDm5NHv2KaRBX2gpw7t7ZLhTgwGEWcZvcDebiLneXcXY5hZ7v2NHJkx/2x1yNXo85xZDy0wK1FGoOOHwPhvqOB+pcQZ/pUOSPTKqGw5l/CexoRm1shFsK+19FnSgimqTHjcuCo4lFW3JlEvlFhtfFXIte2Wjp1ALZgTrSq8zSD5rRxYCUKmM7b3EJwdaIgbvKWPdS4sCXlXU1bHx0g=

View File

@ -1,4 +1,4 @@
ARG FROM=python:3.7-alpine
ARG FROM
FROM ${FROM} as builder
RUN apk add --no-cache \

View File

@ -1,51 +0,0 @@
#!/bin/bash
# Builds all Docker images this project provides
# Arguments:
# BUILD: The release to build.
# Allowed: release, prerelease, branches, special
# Default: undefined
echo "▶️ $0 $*"
ALL_BUILDS=("release" "prerelease" "branches" "special")
BUILDS=("${BUILD:-"${ALL_BUILDS[@]}"}")
echo "⚙️ Configured builds: ${BUILDS[*]}"
if [ -n "${DEBUG}" ]; then
export DEBUG
fi
ERROR=0
for BUILD in "${BUILDS[@]}"; do
echo "🛠 Building '$BUILD' from '$DOCKERFILE'"
case $BUILD in
release)
# build the latest release
# shellcheck disable=SC2068
./build-latest.sh $@ || ERROR=1
;;
prerelease)
# build the latest pre-release
# shellcheck disable=SC2068
PRERELEASE=true ./build-latest.sh $@ || ERROR=1
;;
branches)
# build all branches
# shellcheck disable=SC2068
./build-branches.sh $@ || ERROR=1
;;
*)
echo "🚨 Unrecognized build '$BUILD'."
if [ -z "$DEBUG" ]; then
exit 1
else
echo "⚠️ Would exit here with code '1', but DEBUG is enabled."
fi
;;
esac
done
exit $ERROR

View File

@ -0,0 +1,8 @@
#!/bin/bash
push_image_to_registry() {
local target_tag=$1
echo "⏫ Pushing '${target_tag}'"
$DRY docker push "${target_tag}"
echo "✅ Finished pushing the Docker image '${target_tag}'."
}

View File

@ -0,0 +1,82 @@
#!/bin/bash
# Retrieves image configuration from public images in DockerHub
# Functions from https://gist.github.com/cirocosta/17ea17be7ac11594cb0f290b0a3ac0d1
# Optimised for our use case
get_image_label() {
local label=$1
local image=$2
local tag=$3
local token
token=$(_get_token "$image")
local digest
digest=$(_get_digest "$image" "$tag" "$token")
local retval="null"
if [ "$digest" != "null" ]; then
retval=$(_get_image_configuration "$image" "$token" "$digest" "$label")
fi
echo "$retval"
}
get_image_layers() {
local image=$1
local tag=$2
local token
token=$(_get_token "$image")
_get_layers "$image" "$tag" "$token"
}
get_image_last_layer() {
local image=$1
local tag=$2
local token
token=$(_get_token "$image")
local layers
mapfile -t layers < <(_get_layers "$image" "$tag" "$token")
echo "${layers[-1]}"
}
_get_image_configuration() {
local image=$1
local token=$2
local digest=$3
local label=$4
curl \
--silent \
--location \
--header "Authorization: Bearer $token" \
"https://registry-1.docker.io/v2/$image/blobs/$digest" \
| jq -r ".config.Labels.\"$label\""
}
_get_token() {
local image=$1
curl \
--silent \
"https://auth.docker.io/token?scope=repository:$image:pull&service=registry.docker.io" \
| jq -r '.token'
}
_get_digest() {
local image=$1
local tag=$2
local token=$3
curl \
--silent \
--header "Accept: application/vnd.docker.distribution.manifest.v2+json" \
--header "Authorization: Bearer $token" \
"https://registry-1.docker.io/v2/$image/manifests/$tag" \
| jq -r '.config.digest'
}
_get_layers() {
local image=$1
local tag=$2
local token=$3
curl \
--silent \
--header "Accept: application/vnd.docker.distribution.manifest.v2+json" \
--header "Authorization: Bearer $token" \
"https://registry-1.docker.io/v2/$image/manifests/$tag" \
| jq -r '.layers[].digest'
}

View File

@ -23,25 +23,17 @@ GITHUB_REPO="${GITHUB_REPO-$ORIGINAL_GITHUB_REPO}"
URL_RELEASES="https://api.github.com/repos/${GITHUB_REPO}/branches?${GITHUB_OAUTH_PARAMS}"
# Composing the JQ commans to extract the most recent version number
JQ_BRANCHES='map(.name) | .[] | scan("^[^v].+") | match("^(master|develop).*") | .string'
JQ_NEXT='map(.name) | .[] | scan("^[^v].+") | match("^(develop-).*") | .string'
CURL="curl -sS"
# Querying the Github API to fetch all branches
BRANCHES=$($CURL "${URL_RELEASES}" | jq -r "$JQ_BRANCHES")
NEXT=$($CURL "${URL_RELEASES}" | jq -r "$JQ_NEXT")
###
# Building each branch
###
# keeping track whether an error occured
ERROR=0
# calling build.sh for each branch
for BRANCH in $BRANCHES; do
if [ -n "$NEXT" ]; then
# shellcheck disable=SC2068
./build.sh "${BRANCH}" $@ || ERROR=1
done
# returning whether an error occured
exit $ERROR
./build.sh "${NEXT}" $@
else
echo "No branch matching 'develop-*' found"
echo "::set-output name=skipped::true"
fi

View File

@ -49,7 +49,7 @@ if [ "${1}x" == "x" ] || [ "${1}" == "--help" ] || [ "${1}" == "-h" ]; then
echo " DOCKERFILE The name of Dockerfile to use."
echo " Default: Dockerfile"
echo " DOCKER_FROM The base image to use."
echo " Default: Whatever is defined as default in the Dockerfile."
echo " Default: 'python:3.7-alpine'"
echo " DOCKER_TARGET A specific target to build."
echo " It's currently not possible to pass multiple targets."
echo " Default: main ldap"
@ -153,6 +153,13 @@ if [ ! -f "${DOCKERFILE}" ]; then
fi
fi
###
# Determining the value for DOCKER_FROM
###
if [ -z "$DOCKER_FROM" ]; then
DOCKER_FROM="python:3.7-alpine"
fi
###
# Variables for labelling the docker image
###
@ -167,9 +174,9 @@ PROJECT_VERSION="${PROJECT_VERSION-$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]
# Get the Git information from the netbox directory
if [ -d "${NETBOX_PATH}/.git" ]; then
NETBOX_GIT_REF=$(cd ${NETBOX_PATH}; git rev-parse HEAD)
NETBOX_GIT_BRANCH=$(cd ${NETBOX_PATH}; git rev-parse --abbrev-ref HEAD)
NETBOX_GIT_URL=$(cd ${NETBOX_PATH}; git remote get-url origin)
NETBOX_GIT_REF=$(cd "${NETBOX_PATH}"; git rev-parse HEAD)
NETBOX_GIT_BRANCH=$(cd "${NETBOX_PATH}"; git rev-parse --abbrev-ref HEAD)
NETBOX_GIT_URL=$(cd "${NETBOX_PATH}"; git remote get-url origin)
fi
###
@ -217,15 +224,18 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
# composing the additional DOCKER_SHORT_TAG,
# i.e. "v2.6.1" becomes "v2.6",
# which is only relevant for version tags
# Also let "latest" follow the highest version
###
if [[ "${TAG}" =~ ^v([0-9]+)\.([0-9]+)\.[0-9]+$ ]]; then
MAJOR=${BASH_REMATCH[1]}
MINOR=${BASH_REMATCH[2]}
TARGET_DOCKER_SHORT_TAG="${DOCKER_SHORT_TAG-${DOCKER_REGISTRY}/${DOCKER_ORG}/${DOCKER_REPO}:v${MAJOR}.${MINOR}}"
TARGET_DOCKER_LATEST_TAG="${DOCKER_REGISTRY}/${DOCKER_ORG}/${DOCKER_REPO}:latest"
if [ "${DOCKER_TARGET}" != "main" ]; then
TARGET_DOCKER_SHORT_TAG="${TARGET_DOCKER_SHORT_TAG}-${DOCKER_TARGET}"
TARGET_DOCKER_LATEST_TAG="${TARGET_DOCKER_LATEST_TAG}-${DOCKER_TARGET}"
fi
fi
@ -233,6 +243,48 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
# Proceeding to buils stage, except if `--push-only` is passed
###
if [ "${2}" != "--push-only" ] ; then
###
# Checking if the build is necessary,
# meaning build only if one of those values changed:
# - Python base image digest (Label: PYTHON_BASE_DIGEST)
# - netbox git ref (Label: NETBOX_GIT_REF)
# - netbox-docker git ref (Label: org.label-schema.vcs-ref)
###
# Load information from registry (only for docker.io)
SHOULD_BUILD="false"
BUILD_REASON=""
if [ -z "${GH_ACTION}" ]; then
# Asuming non Github builds should always proceed
SHOULD_BUILD="true"
BUILD_REASON="${BUILD_REASON} interactive"
elif [ "$DOCKER_REGISTRY" = "docker.io" ]; then
source ./build-functions/get-public-image-config.sh
IFS=':' read -ra DOCKER_FROM_SPLIT <<< "${DOCKER_FROM}"
if ! [[ ${DOCKER_FROM_SPLIT[0]} =~ .*/.* ]]; then
# Need to use "library/..." for images the have no two part name
DOCKER_FROM_SPLIT[0]="library/${DOCKER_FROM_SPLIT[0]}"
fi
PYTHON_LAST_LAYER=$(get_image_last_layer "${DOCKER_FROM_SPLIT[0]}" "${DOCKER_FROM_SPLIT[1]}")
mapfile -t IMAGES_LAYERS_OLD < <(get_image_layers "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}")
NETBOX_GIT_REF_OLD=$(get_image_label NETBOX_GIT_REF "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}")
GIT_REF_OLD=$(get_image_label org.label-schema.vcs-ref "${DOCKER_ORG}"/"${DOCKER_REPO}" "${TAG}")
if ! printf '%s\n' "${IMAGES_LAYERS_OLD[@]}" | grep -q -P "^${PYTHON_LAST_LAYER}\$"; then
SHOULD_BUILD="true"
BUILD_REASON="${BUILD_REASON} python"
fi
if [ "${NETBOX_GIT_REF}" != "${NETBOX_GIT_REF_OLD}" ]; then
SHOULD_BUILD="true"
BUILD_REASON="${BUILD_REASON} netbox"
fi
if [ "${GIT_REF}" != "${GIT_REF_OLD}" ]; then
SHOULD_BUILD="true"
BUILD_REASON="${BUILD_REASON} netbox-docker"
fi
else
SHOULD_BUILD="true"
BUILD_REASON="${BUILD_REASON} no-check"
fi
###
# Composing all arguments for `docker build`
###
@ -244,6 +296,7 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
)
if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then
DOCKER_BUILD_ARGS+=( -t "${TARGET_DOCKER_SHORT_TAG}" )
DOCKER_BUILD_ARGS+=( -t "${TARGET_DOCKER_LATEST_TAG}" )
fi
# --label
@ -269,6 +322,10 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
--label "NETBOX_GIT_URL=${NETBOX_GIT_URL}"
)
fi
if [ -n "${BUILD_REASON}" ]; then
BUILD_REASON=$(sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' <<< "$BUILD_REASON")
DOCKER_BUILD_ARGS+=( --label "BUILD_REASON=${BUILD_REASON}" )
fi
# --build-arg
DOCKER_BUILD_ARGS+=( --build-arg "NETBOX_PATH=${NETBOX_PATH}" )
@ -287,25 +344,29 @@ for DOCKER_TARGET in "${DOCKER_TARGETS[@]}"; do
###
# Building the docker image
###
if [ "${SHOULD_BUILD}" == "true" ]; then
echo "🐳 Building the Docker image '${TARGET_DOCKER_TAG}'."
echo " Build reason set to: ${BUILD_REASON}"
$DRY docker build "${DOCKER_BUILD_ARGS[@]}" .
echo "✅ Finished building the Docker images '${TARGET_DOCKER_TAG}'"
echo "🔎 Inspecting labels on '${TARGET_DOCKER_TAG}'"
$DRY docker inspect "${TARGET_DOCKER_TAG}" --format "{{json .Config.Labels}}"
else
echo "Build skipped because sources didn't change"
echo "::set-output name=skipped::true"
fi
fi
###
# Pushing the docker images if either `--push` or `--push-only` are passed
###
if [ "${2}" == "--push" ] || [ "${2}" == "--push-only" ] ; then
echo "⏫ Inspecting labels on '${TARGET_DOCKER_TAG}'"
$DRY docker inspect "${TARGET_DOCKER_TAG}" --format "{{json .Config.Labels}}"
echo "⏫ Pushing '${TARGET_DOCKER_TAG}'"
$DRY docker push "${TARGET_DOCKER_TAG}"
echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_TAG}'."
source ./build-functions/docker-functions.sh
push_image_to_registry "${TARGET_DOCKER_TAG}"
if [ -n "${TARGET_DOCKER_SHORT_TAG}" ]; then
echo "⏫ Pushing '${TARGET_DOCKER_SHORT_TAG}'"
$DRY docker push "${TARGET_DOCKER_SHORT_TAG}"
echo "✅ Finished pushing the Docker image '${TARGET_DOCKER_SHORT_TAG}'."
push_image_to_registry "${TARGET_DOCKER_SHORT_TAG}"
push_image_to_registry "${TARGET_DOCKER_LATEST_TAG}"
fi
fi
done