ISSUE-313

Dockerfile

@@ -10,6 +10,8 @@ ADD . /go/src/github.com/dutchcoders/transfer.sh
 WORKDIR /go/src/github.com/dutchcoders/transfer.sh
 
 ENV GO111MODULE=on
+ENV GOOS=${GOOS}
+ENV GOARCH=${GOARCH}
 
 # build & install server
 RUN go get -u ./... && CGO_ENABLED=0 go build -ldflags -a -tags netgo -ldflags '-w -extldflags "-static"' -o /go/bin/transfersh github.com/dutchcoders/transfer.sh

hooks/.config

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+set +u
+echo "variables (see https://docs.docker.com/docker-hub/builds/advanced/):"
+echo "SOURCE_BRANCH:   $SOURCE_BRANCH"
+echo "SOURCE_COMMIT:   $SOURCE_COMMIT"
+echo "COMMIT_MSG:      $COMMIT_MSG"
+echo "DOCKER_REPO:     $DOCKER_REPO"
+echo "DOCKERFILE_PATH: $DOCKERFILE_PATH"
+echo "CACHE_TAG:       $CACHE_TAG"
+echo "IMAGE_NAME:      $IMAGE_NAME"
+echo
+
+: "${DOCKERFILE_PATH:=./Dockerfile}"
+: "${IMAGE_NAME:=dutchcoders/transer.sh}"
+
+echo "variables after applying defaults:"
+echo "DOCKERFILE_PATH: $DOCKERFILE_PATH"
+echo "IMAGE_NAME:      $IMAGE_NAME"
+echo
+
+export PATH="$PWD/docker:$PATH"
+
+# =>
+# https://hub.docker.com/u/arm64v8/
+# https://hub.docker.com/u/arm32v7/
+# https://hub.docker.com/u/arm32v6/
+# https://hub.docker.com/u/arm32v5/
+declare -A base_image_prefix_map=( ["aarch64"]="arm64v8/" ["arm"]="arm32v5/" ["amd64"]="")
+
+# => dpkg -L qemu-user-static | grep /usr/bin/
+declare -A docker_qemu_arch_map=( ["aarch64"]="aarch64" ["arm"]="arm" ["amd64"]="x86_64")
+
+# => https://github.com/docker/docker-ce/blob/76ac3a4952a9c03f04f26fc88d3160acd51d1702/components/cli/cli/command/manifest/util.go#L22
+declare -A docker_to_manifest_map=( ["aarch64"]="arm64" ["arm"]="arm" ["amd64"]="amd64")
+
+# what we want to build
+build_architectures=(amd64 aarch64 arm)
+verified_build_architectures=()
+verified_build_architectures+=("$(docker version -f '{{.Server.Arch}}')")
+
+# what we can build
+for arch in ${build_architectures[@]}; do
+  if [ -f "qemu-${docker_qemu_arch_map[${arch}]}-static" ]; then
+    echo "qemu binary for $arch found";
+    verified_build_architectures+=($arch)
+  fi
+done
+
+echo $verified_build_architectures
+set -u
+
+docker -v
+echo

hooks/build

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+set -eu
+
+echo "build"
+source hooks/.config
+
+echo "Will build the following architectures: $verified_build_architectures"
+echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+
+for arch in ${verified_build_architectures[@]}; do
+  echo "building $arch"
+  echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+
+  BASE_IMAGE_PREFIX="${base_image_prefix_map[${arch}]}"
+  docker build \
+    --build-arg GOOS=linux \
+    --build-arg GOARCH=${arch} \
+    --file $DOCKERFILE_PATH \
+    --tag "${IMAGE_NAME}-${arch}"  \
+    .
+done
+
+echo "images built:"
+echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+docker image ls
+
+# https://github.com/moby/moby/issues/36552
+#
+tempdir=$(mktemp -d -t yolo.XXXXXXXX)
+cd $tempdir
+
+for arch in ${verified_build_architectures[@]}; do
+  echo "yolo fixing platform $arch"
+  echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+
+  manifest_arch=${docker_to_manifest_map[${arch}]}
+  docker save "${IMAGE_NAME}-${arch}"| tar xv
+
+  for filename in */json; do
+    [ -e "$filename" ] || continue
+    jq --compact-output 'del(.architecture)' < "$filename" | sponge "$filename"
+  done
+
+  for filename in *.json; do
+    [ -e "$filename" ] || continue
+    ! [ $filename = "manifest.json" ] || continue
+
+    jq --arg architecture "$manifest_arch" \
+       --compact-output '.architecture=$architecture' < "$filename" | sponge "$filename"
+  done
+
+  tar cv . | docker load
+  rm -rf $tempdir/*
+done
+
+trap "exit 1"          HUP INT PIPE QUIT TERM
+trap "rm -rf $tempdir" EXIT

hooks/get_qemu.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -ex
+
+# NOTE: this url will change regularly because it's unstable
+PACKAGE=http://ftp.de.debian.org/debian/pool/main/q/qemu/qemu-user-static_4.2-2_amd64.deb
+
+mkdir tmp/
+cd tmp/
+
+curl $PACKAGE -o $(basename ${PACKAGE})
+dpkg-deb -X $(basename ${PACKAGE}) .
+cp usr/bin/qemu-aarch64-static ..
+cp usr/bin/qemu-arm-static ..
+cd ..
+rm -rf tmp

hooks/post_checkout

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+set -eu
+
+echo "post_checkout"
+source hooks/.config
+
+echo "Install qemu + binfmt support"
+echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+# it's an Ubuntu VM and you can install stuff.
+apt-get update
+apt-get install -y curl qemu-user-static binfmt-support jq moreutils
+
+# Sadly docker itself uses Docker EE 17.06 on Dockerhub which does not support
+# manifests.
+echo "Install a fresh docker cli binary"
+echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+
+curl https://download.docker.com/linux/static/stable/x86_64/docker-19.03.9.tgz | \
+  tar xvz docker/docker
+
+echo "Build a usable config.json file"
+echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+# Manifests are still experimental and enabled by a config file flag.
+# Interestingly, there is no config file and the credential parts to push
+# images is available in an environment variable. Let's create a config file to
+# combine the two things:
+#
+mkdir -p ~/.docker
+jq --null-input --argjson auths "$DOCKERCFG" '. + {auths: $auths}' | \
+jq --arg experimental enabled '. + {experimental: $experimental}' | \
+sponge ~/.docker/config.json
+
+echo "copy qemu binaries into docker build context"
+echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+# The current setup copies the qemu binary into the image (see Dockerfile)
+# Pro:
+#      - it's easy to run non-amd64 images on amd64 systems for debugging
+# Contra:
+#      - it's dead weight in the "destination" architecture and consumes space
+# Alternative:
+#      - use a multistage Dockerfile (no RUN in the last stage possible of course)
+#      - wait for https://github.com/moby/moby/issues/14080
+#
+for arch in ${build_architectures[@]}; do
+  cp /usr/bin/qemu-${docker_qemu_arch_map[${arch}]}-static qemu-${arch}-static
+done
+
+ls -la

hooks/pre_build

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -eu
+
+echo "pre_build"
+source hooks/.config
+
+echo "Register qemu-*-static for all supported processors except current"
+echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+
+docker run --rm --privileged multiarch/qemu-user-static:register --reset

hooks/push

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -eu
+
+echo "push"
+source hooks/.config
+
+# 1. push all images
+IMAGE_NAME="${IMAGE_NAME//index.docker.io\/}"
+
+for arch in ${verified_build_architectures[@]}; do
+  echo "Pushing ${IMAGE_NAME}-${arch}"
+  echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+  echo
+  docker push ${IMAGE_NAME}-${arch}
+done
+
+docker image ls
+
+# 2. build and push manifest
+#DOCKER_REPO="index.docker.io/${DOCKER_REPO}"
+manifests=""
+
+for arch in ${verified_build_architectures[@]}; do
+  manifests="${manifests} ${IMAGE_NAME}-${arch}"
+done
+
+echo "Creating manifest ${IMAGE_NAME}"
+echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+docker manifest create ${IMAGE_NAME} \
+                            $manifests
+echo
+
+echo "Annotating manifest"
+echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+for arch in ${verified_build_architectures[@]}; do
+    docker manifest annotate ${IMAGE_NAME} \
+    ${IMAGE_NAME}-${arch} \
+    --os linux \
+    --arch ${docker_to_manifest_map[${arch}]}
+done
+
+echo "Inspecting manifest ${IMAGE_NAME}-${arch}"
+echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+docker manifest inspect ${IMAGE_NAME}-${arch}
+echo
+
+echo "Pushing manifest ${IMAGE_NAME}"
+echo "⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯⎯"
+docker manifest push --purge "${IMAGE_NAME}"
+echo
+
+echo
+echo "Done"