build-system: Docker build for 64bit MXE

We previously tried to build the MXE Docker container on GitHub using
an Action, but that really didn't work well and was a lot more trouble
than it was worth.

So this goes back to an offline build mechanism where I simply create
an updated Docker image when needed and push that to Docker Hub.

But this nearly hides the most interesting change here - we are finally
switching to using 64bit binaries on Windows. It's 2020 and fewer than
1% of our users use 32bit Windows machines. We'll need to expand this
to be able to have both a 32bit and a 64bit version of Subsurface for
Windows. But for now, this solves the problem for 99% of our users.

Signed-off-by: Dirk Hohndel <dirk@hohndel.org>

.github/workflows/mxe-dockerimage-stage1.yml

@@ -1,53 +0,0 @@
-name: MXE Docker Image CI
-
-on:
-  push:
-    paths:
-    - scripts/docker/mxe-build-container/*
-    - .github/workflows/mxe-docker*
-
-jobs:
-  mxe-build-container-stage1:
-    runs-on: ubuntu-latest
-    env:
-      # IMPORTANT: use the second digit to test new versions, each new official build should be n.0 with the first one being 1.0
-      #            as you test changes toward a new release, call those 1.1, 1.2, 1.3, etc
-      #            VERSION is just that version presented as a string constant
-      #
-      VERSION: ${{ '1.0' }}
-
-    steps:
-    - uses: actions/checkout@v1
-
-      # Because, reasons, we can't really do anything in YAML, so do this crazy shell callout thingy in order to assemble
-      # sane variables to use later in this job - if someone knows an easier way to do this, please let me know!
-      # This abomination below assembles the docker image NAME and appends ".stage1" to the VERSION
-    - name: set env
-      run: |
-        v=${{ env.VERSION }}
-        s=".stage1"
-        b=${{ github.ref }} # -BRANCH suffix, unless the branch is master
-        b=${b/refs\/heads\//}
-        b=${b,,} # the name needs to be all lower case
-        if [ $b = "master" ] ; then b="" ; else b="-$b" ; fi
-        echo "::set-env name=NAME::subsurface/mxe-build-container${b}:${v}${s}"
-
-    - name: Build and Publish stage 1 Docker image to Dockerhub
-      uses: elgohr/Publish-Docker-Github-Action@master
-      with:
-        name:       ${{ env.NAME }}
-        username:   ${{ secrets.DOCKER_USERNAME }}
-        password:   ${{ secrets.DOCKER_PASSWORD }}
-        dockerfile: 'Dockerfile-stage1'
-        workdir:    './scripts/docker/mxe-build-container/'
-
-    - name: Trigger stage 2 to run
-      # this triggers a dispatch event in our own repository - this allows us to work around the
-      # 6h runtime max (as on a two core system the total build time of this container is closer to 7h)
-      # we use the event_type to pass the version that we are building to the second stage
-      run: |
-        curl -XPOST -H 'authorization: token ${{ secrets.ACCESS_TOKEN }}' \
-                    -H "Accept: application/vnd.github.everest-preview+json" \
-                    -H "Content-Type: application/json" \
-                    https://api.github.com/repos/subsurface-divelog/subsurface/dispatches \
-                    --data '{"event_type": "${{ env.VERSION }}", "client_payload": { "target_branch": "${{ github.ref }}" } }'

.github/workflows/mxe-dockerimage-stage2.yml

@@ -1,40 +0,0 @@
-name: MXE stage 2 Docker Image CI
-
-# trigger this second stage via a repository dispaych event
-# the version that we are building is passed in via the event type that is available to us here
-# as github.event.action
-
-on: repository_dispatch
-
-jobs:
-  mxe-build-container-stage2:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-      with:
-        ref: ${{ github.event.client_payload.target_branch }}
-
-      # Grab the version from the event name that we were triggered by and add ".stage1" to find the docker image to start FROM
-      # And create the NAME of the final docker image (including the branch if not master)
-    - name: set env
-      run: |
-        v=${{ github.event.action }}
-        s=".stage1"
-        b=${{ github.event.client_payload.target_branch }} # -BRANCH suffix, unless the branch is master
-        b=${b/refs\/heads\//}
-        b=${b,,} # the name needs to be all lower case
-        if [ "$b" = "master" ] || [ "$b" = "" ] ; then b="" ; else b="-$b" ; fi
-        echo "::set-env name=VERSION::${v}${s}"
-        echo "::set-env name=NAME::subsurface/mxe-build-container${b}:${v}"
-
-    - name: Build and Publish stage 2 Docker image to Dockerhub
-      uses: elgohr/Publish-Docker-Github-Action@master
-      with:
-        name:       ${{ env.NAME }}
-        username:   ${{ secrets.DOCKER_USERNAME }}
-        password:   ${{ secrets.DOCKER_PASSWORD }}
-        dockerfile: 'Dockerfile-stage2'
-        workdir:    './scripts/docker/mxe-build-container/'
-        buildargs:  VERSION
-

packaging/windows/README.md

@@ -12,7 +12,7 @@ git clone git://github.com/subsurface/subsurface
 cd subsurface
 git submodule init
 git submodule update
-docker run -v /some/path/windows:/__w subsurface/mxe-build-container:1.1 /bin/bash /__w/subsurface/packaging/windows/create-win-installer.sh
+docker run -v /some/path/windows:/__w subsurface/mxe-build-container:2.0 /bin/bash /__w/subsurface/packaging/windows/create-win-installer.sh
 ```
 
 This will result in subsurface-VERSION.exe and smtk2ssrf-VERSION.exe to be created.

scripts/docker/mxe-build-container/Dockerfile

@@ -3,13 +3,15 @@
 #
 
 # Start from Ubuntu
-From ubuntu:18.04
+From ubuntu:20.04
+
+# very often master is broken, so we pass in a known good SHA
 ARG mxe_sha=master
 ENV _ver=${mxe_sha}
 
 # update and set up the packages we need for this cross build
 RUN apt-get update  &&  apt-get upgrade -y && \
-apt-get install -y \
+DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
     autoconf \
     automake \
     autopoint \
@@ -47,18 +49,26 @@ apt-get install -y \
     lzip \
     scons
 
-# copy over the partial MXE settings (simply missing QtWebKit so we don't exceed the 6h limit)
+# checkout MXE at the right version
 RUN mkdir -p /win
-ADD settings-stage1.mk /win/settings.mk
 RUN cd /win ; git clone git://github.com/mxe/mxe ; \
     cd mxe ; \
     git checkout ${_ver} ;
+
 # Patch the qtconnectivity build to explicilty enable native-win32-bluetooth and ensure another
 # backend is not picked
-ADD qtconnectivity-1.patch /win/qtconnectivity-1.patch
+ADD qtconnectivity-1.patch /win/mxe/src/qtconnectivity-1.patch
-RUN mv /win/qtconnectivity-1.patch /win/mxe/src
+
-RUN mv /win/settings.mk /win/mxe
+# Move the settings into place to build everything that we need
+# separate download from build so that we can redo the build
+ADD settings.mk /win/mxe/settings.mk
 RUN cd /win/mxe ; \
-    make -j 6 2>&1 | tee build.log ;
+    make -j 6 download 2>&1 | tee mxe-build.log
 RUN cd /win/mxe ; \
-    make MXE_TARGETS=i686-w64-mingw32.static glib -j 6 2>&1 | tee -a build.log ;
+    make -j 6 2>&1 | tee -a mxe-build.log ;
+
+# for some reason smtk2ssrf needs a static build of mdbtools
+RUN cd /win/mxe ; \
+    make MXE_TARGETS=x86_64-w64-mingw32.static glib mdbtools -j 6 2>&1 | tee -a mxe-build.log ;
+
+RUN apt-get remove -y bison flex gperf libc6-dev-i386 libgdk-pixbuf2.0-dev libxml-parser-perl python ruby xz-utils scons && rm -rf /var/lib/apt/lists/*

scripts/docker/mxe-build-container/Dockerfile-stage2

@@ -1,13 +0,0 @@
-ARG VERSION
-
-# Start from the container that we created in stage 1
-From subsurface/mxe-build-container:$VERSION
-
-# this time the settings include QtWebKit - since everything else was built
-# during the first stage, QtWebKit will be the only thing that gets built
-ADD settings-stage2.mk /win/mxe/settings.mk
-RUN cd /win/mxe ; \
-    make -j 2 2>&1 | tee build.log ;
-RUN cd /win/mxe ; \
-    make MXE_TARGETS=i686-w64-mingw32.static glib mdbtools -j 6 2>&1 | tee -a build.log ;
-

scripts/docker/mxe-build-container/build-container.sh

@@ -2,10 +2,13 @@
 set -x
 set -e
 
+# known good MXE sha
+MXE_SHA="8966a64"
 SCRIPTPATH=$(dirname $0)
 
-export VERSION=1.1
+# version of the docker image
+VERSION=2.0
+
 pushd $SCRIPTPATH
-docker build -t subsurface/mxe-build-container:$VERSION --build-arg=mxe_sha=1ee37f8 -f Dockerfile-stage1 .
+docker build --squash -t subsurface/mxe-build-container:$VERSION --build-arg=mxe_sha=$MXE_SHA -f Dockerfile .
-docker build -t subsurface/mxe-build-container:$VERSION --build-arg=VERSION=$VERSION -f Dockerfile-stage2 .
 popd

scripts/docker/mxe-build-container/instructions.md

@@ -2,16 +2,14 @@
 
 This document assumes you have alreay installed docker and have checked out subsurface according to the instructions in the INSTALL document.
 
-If you are just wantint to build with the current mxe build container then starting from the folder above subsurface run
+If you just want to build with the current mxe build container then starting from the folder above subsurface run
 
 ```bash
-docker run -v $PWD/win32:/win/win32 -v $PWD/subsurface:/win/subsurface --name=mybuilder -w /win -d subsurface/mxe-build-container:1.x /bin/sleep 60m
+docker run -v $PWD/win32:/win/win32 -v $PWD/subsurface:/win/subsurface --name=mybuilder -w /win -d subsurface/mxe-build-container:x.y /bin/sleep 60m
 ```
 
-replacing the x in the mxe-build-container tag with the current version e.g.
+replacing the x.y in the mxe-build-container tag with the current version e.g. 2.0
-```bash
+
-docker run -v $PWD/win32:/win/win32 -v $PWD/subsurface:/win/subsurface --name=mybuilder -w /win -d subsurface/mxe-build-container:1.0 /bin/sleep 60m
-```
 Next you need to prep the container by installing some prerequisites
 
 ```bash

scripts/docker/mxe-build-container/settings-stage1.mk

@@ -1,15 +0,0 @@
-# This is a template of configuration file for MXE. See
-# index.html for more extensive documentations.
-
-# This variable controls the number of compilation processes
-# within one package ("intra-package parallelism").
-JOBS := 2
-
-# This variable controls the targets that will build.
-MXE_TARGETS :=  i686-w64-mingw32.shared
-
-# The three lines below makes `make` build these "local packages" instead of all packages.
-LOCAL_PKG_LIST := qtbase qtconnectivity qtdeclarative qtimageformats qtlocation qtmultimedia qtquickcontrols qtquickcontrols2 qtscript qtsvg qttools qttranslations qtwebview libxml2 libxslt libusb1 libgit2 nsis curl libzip libftdi1
-.DEFAULT local-pkg-list:
-local-pkg-list: $(LOCAL_PKG_LIST)
-

scripts/docker/mxe-build-container/settings.mk

@@ -3,23 +3,31 @@
 
 # This variable controls the number of compilation processes
 # within one package ("intra-package parallelism").
-JOBS := 2
+JOBS := 8
 
 # This variable controls the targets that will build.
-MXE_TARGETS :=  i686-w64-mingw32.shared
+MXE_TARGETS :=  x86_64-w64-mingw32.shared
 
 # The three lines below makes `make` build these "local packages" instead of all packages.
-LOCAL_PKG_LIST := curl \
+# The ordering of the list appears weird, but this seems to help to get the build done
-                  hidapi \
+# faster on a massively parallel machine to get some of the bottleneck packages built as
-                  libftdi1 \
+# early as possible
-                  libgit2 \
+LOCAL_PKG_LIST := gcc \
-                  libusb1 \
+                  openssl \
+                  libmysqlclient \
+                  postgresql \
+                  qtbase \
+                  qtwebkit \
+                  nsis \
+                  curl \
                   libxml2 \
                   libxslt \
                   libzip \
+                  libusb1 \
+                  hidapi \
+                  libgit2 \
+                  libftdi1 \
                   mdbtools \
-                  nsis \
-                  qtbase \
                   qtconnectivity \
                   qtdeclarative \
                   qtimageformats \
@@ -27,13 +35,10 @@ LOCAL_PKG_LIST := curl \
                   qtmultimedia \
                   qtquickcontrols \
                   qtquickcontrols2 \
-                  qtscript \
+                  qtcharts \
                   qtsvg \
                   qttools \
                   qttranslations \
-                  qtwebkit \
-                  qtwebview \
                   zstd
 .DEFAULT local-pkg-list:
 local-pkg-list: $(LOCAL_PKG_LIST)
-