build-system/MXE: build with more up to date MXE tools.

- use hidapi grantlee and mdbtools from MXE
- update MXE version to use QT 5.15, and pull in libzstd and  CMake 3.17.3
- fix linking of winmm on windows build with new mxe
- add some instructions on building the container
- add some new dependancies from QT 5.15 to the packaging
- add a patch to MXE to Build qtconnectivity with native-win32-bluetooth

[Dirk Hohndel: small refactor]

Signed-off-by: Paul Buxton <paulbuxton.mail@googlemail.com>
Signed-off-by: Dirk Hohndel <dirk@hohndel.org>

.github/workflows/scripts/windows-container-prep.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+# abstract the prepare commands for the windows build into a script that can be reused
+# instead of a yaml file
+cd /win
+ln -s /__w/subsurface/subsurface .
+echo "downloading sources for fresh build"
+bash subsurface/scripts/get-dep-lib.sh single . libzip
+bash subsurface/scripts/get-dep-lib.sh single . googlemaps

.github/workflows/windows.yml

@@ -19,20 +19,8 @@ jobs:
 
     - name: get other dependencies
       run: |
-        echo "creating the link from /win/subsurface"
-        cd /win
-        ln -s /__w/subsurface/subsurface .
-        ls -l
-        ls -l subsurface/scripts
-        echo "installing missing container components"
-        apt-get install -y ca-certificates libtool
-        echo "downloading sources for fresh build"
-        bash subsurface/scripts/get-dep-lib.sh single . libzip
-        bash subsurface/scripts/get-dep-lib.sh single . hidapi
-        bash subsurface/scripts/get-dep-lib.sh single . googlemaps
-        bash subsurface/scripts/get-dep-lib.sh single . grantlee
-        bash subsurface/scripts/get-dep-lib.sh single . mdbtools
-
+        echo "Running script to install additional dependancies into container"
+        bash -x subsurface/.github/workflows/scripts/windows-container-prep.sh 2>&1 | tee pre-build.log
     - name: run build
       run: |
         cd /win

INSTALL

@@ -320,6 +320,9 @@ Please read through the explanations and instructions in
 packaging/windows/mxe-based-build.sh if you want to build the Windows
 version on your Linux system.
 
+In addition you can use the Docker container the same way that CI builds do modify/test 
+ modifications to the container build environment following instructions in 
+/scripts/docker/mxe-build-container/instructions.md
 
 Building Subsurface on Windows
 ------------------------------

packaging/windows/mxe-based-build.sh

@@ -138,41 +138,6 @@ else
 	touch Release
 fi
 
-# grantlee
-
-cd "$BUILDDIR"
-if [[ ! -d grantlee || -f build.grantlee ]] ; then
-	rm -f build.grantlee
-	mkdir -p grantlee
-	cd grantlee
-	"$MXEBUILDTYPE"-cmake \
-		-DCMAKE_BUILD_TYPE=$RELEASE \
-		-DBUILD_TESTS=OFF \
-		"$BASEDIR"/grantlee
-
-	make $JOBS
-	make install
-fi
-
-# hidapi for libdivecomputer (if available)
-
-if [[ -d "$BASEDIR"/hidapi ]] ; then
-	cd "$BUILDDIR"
-	if [[ ! -d hidapi || -f build.hidapi ]] ; then
-		rm -f build.hidapi
-		mkdir -p hidapi
-		pushd "$BASEDIR"/hidapi
-		bash ./bootstrap
-		popd
-		cd hidapi
-		"$BASEDIR"/hidapi/configure \
-			CC="$MXEBUILDTYPE"-gcc \
-			--host="$MXEBUILDTYPE" \
-			--prefix="$BASEDIR"/"$MXEDIR"/usr/"$MXEBUILDTYPE"
-		make $JOBS
-		make install
-	fi
-fi
 
 
 # libdivecomputer
@@ -284,7 +249,8 @@ done
 # for some reason we aren't installing Qt5Xml.dll and Qt5Location.dll
 # I need to figure out why and fix that, but for now just manually copy that as well
 EXTRA_MANUAL_DEPENDENCIES="$BASEDIR/"$MXEDIR"/usr/"$MXEBUILDTYPE"/qt5/bin/Qt5Xml$DLL_SUFFIX.dll \
-$BASEDIR/"$MXEDIR"/usr/"$MXEBUILDTYPE"/qt5/bin/Qt5Location$DLL_SUFFIX.dll"
+$BASEDIR/"$MXEDIR"/usr/"$MXEBUILDTYPE"/qt5/bin/Qt5Location$DLL_SUFFIX.dll \
+$BASEDIR/"$MXEDIR"/usr/"$MXEBUILDTYPE"/qt5/bin/Qt5QmlWorkerScript$DLL_SUFFIX.dll"
 
 for f in $EXTRA_MANUAL_DEPENDENCIES
 do

packaging/windows/smtk2ssrf-mxe-build.sh

@@ -142,32 +142,6 @@ export PKG_CONFIG_PATH_i686_w64_mingw32_shared="$BASEDIR/mxe/usr/i686-w64-mingw3
 export PKG_CONFIG_PATH="$PKG_CONFIG_PATH_i686_w64_mingw32_static":"$PKG_CONFIG_PATH_i686_w64_mingw32_shared"
 
 #
-# mdbtools
-# build from sources. If build fails, fallback to prebuilt mxe binaries.
-#
-echo -e "$BLUE---> Building mdbtools ... $DEFAULT "
-mkdir -p --verbose "$BASEDIR"/mxe/usr/i686-w64-mingw32.static/include
-mkdir -p --verbose "$BASEDIR"/mxe/usr/i686-w64-mingw32.static/lib
-cd "$BUILDDIR"
-[[ -d mdbtools ]] && rm -rf mdbtools
-mkdir -p mdbtools
-cd mdbtools
-if [ ! -f "$BASEDIR"/mdbtools/configure ] ; then
-	( cd "$BASEDIR"/mdbtools
-	autoreconf -v -f -i )
-fi
-"$BASEDIR"/mdbtools/configure CC=i686-w64-mingw32.static-gcc \
-		     --host=i686-w64-mingw32.static \
-		     --prefix="$BASEDIR"/mxe/usr/i686-w64-mingw32.static \
-		     --enable-shared=no \
-		     --disable-man \
-		     --disable-gmdb2
-# hack to make mdbtools build outsource
-ln -vs "$BUILDDIR"/mdbtools/include/mdbver.h "$BASEDIR"/mdbtools/include/mdbver.h
-
-make $JOBS >/dev/null && make install || \
-	echo -e "$RED---> Building mdbtools failed ...$LIGHT_GRAY Trying to build with precompiled mxe binaries$DEFAULT"
-
 # Subsurface
 #
 if [ "$AUTO" = "false" ]; then

scripts/docker/mxe-build-container/Dockerfile-stage1

@@ -8,8 +8,8 @@ ARG mxe_sha=master
 ENV _ver=${mxe_sha}
 
 # update and set up the packages we need for this cross build
-RUN apt-get update  &&  apt-get upgrade -y
-RUN apt-get install -y \
+RUN apt-get update  &&  apt-get upgrade -y && \
+apt-get install -y \
     autoconf \
     automake \
     autopoint \
@@ -17,6 +17,7 @@ RUN apt-get install -y \
     binutils \
     bison \
     bzip2 \
+    ca-certificates \
     flex \
     g++ \
     g++-multilib \
@@ -28,6 +29,7 @@ RUN apt-get install -y \
     libgdk-pixbuf2.0-dev \
     libltdl-dev \
     libssl-dev \
+    libtool \
     libtool-bin \
     libxml-parser-perl \
     make \
@@ -51,6 +53,10 @@ ADD settings-stage1.mk /win/settings.mk
 RUN cd /win ; git clone git://github.com/mxe/mxe ; \
     cd mxe ; \
     git checkout ${_ver} ;
+# Patch the qtconnectivity build to explicilty enable native-win32-bluetooth and ensure another
+# backend is not picked
+ADD qtconnectivity-1.patch /win/qtconnectivity-1.patch
+RUN mv /win/qtconnectivity-1.patch /win/mxe/src
 RUN mv /win/settings.mk /win/mxe
 RUN cd /win/mxe ; \
     make -j 6 2>&1 | tee build.log ;

scripts/docker/mxe-build-container/Dockerfile-stage2

@@ -9,14 +9,5 @@ ADD settings-stage2.mk /win/mxe/settings.mk
 RUN cd /win/mxe ; \
     make -j 2 2>&1 | tee build.log ;
 RUN cd /win/mxe ; \
-    make MXE_TARGETS=i686-w64-mingw32.static glib -j 6 2>&1 | tee -a build.log ;
+    make MXE_TARGETS=i686-w64-mingw32.static glib mdbtools -j 6 2>&1 | tee -a build.log ;
 
-# manually build the Win BLE version of QtConnectivity (we can drop this with Qt 5.14)
-RUN cd /win/mxe ; \
-    mkdir -p neolit ; cd neolit ; git clone -b wip/win git://github.com/qt/qtconnectivity ; \
-    sed -i 's/SetupAPI.h/setupapi.h/' qtconnectivity/src/bluetooth/qlowenergycontroller_win.cpp
-RUN cd /win/mxe/neolit/qtconnectivity ; \
-    PATH=/win/mxe/usr/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin /win/mxe/usr/i686-w64-mingw32.shared/qt5/bin/qmake qtconnectivity.pro ; \
-    PATH=/win/mxe/usr/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin make -j 6 ; \
-    PATH=/win/mxe/usr/bin/:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin make install ;
-RUN rm -rf /win/mxe/pkg

scripts/docker/mxe-build-container/build-container.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -x
+set -e
+
+SCRIPTPATH=$(dirname $0)
+
+export VERSION=1.1
+pushd $SCRIPTPATH
+docker build -t subsurface/mxe-build-container:$VERSION --build-arg=mxe_sha=1ee37f8 -f Dockerfile-stage1 .
+docker build -t subsurface/mxe-build-container:$VERSION --build-arg=VERSION=$VERSION -f Dockerfile-stage2 .
+popd

scripts/docker/mxe-build-container/instructions.md

@@ -0,0 +1,38 @@
+# Instructions for building the container environment and using it to build/package subsurface.
+
+This document assumes you have alreay installed docker and have checked out subsurface according to the instructions in the INSTALL document.
+
+If you are just wantint to build with the current mxe build container then starting from the folder above subsurface run
+
+```bash
+docker run -v $PWD/win32:/win/win32 -v $PWD/subsurface:/win/subsurface --name=mybuilder -w /win -d subsurface/mxe-build-container:1.x /bin/sleep 60m
+```
+
+replacing the x in the mxe-build-container tag with the current version e.g.
+```bash
+docker run -v $PWD/win32:/win/win32 -v $PWD/subsurface:/win/subsurface --name=mybuilder -w /win -d subsurface/mxe-build-container:1.0 /bin/sleep 60m
+```
+Next you need to prep the container by installing some prerequisites
+
+```bash
+docker exec -t mybuilder bash subsurface/.github/workflows/scripts/windows-container-prep.sh 2>&1 | tee pre-build.log
+```
+
+Finaly the actual build is done with
+```bash
+docker exec -t mybuilder bash subsurface/.github/workflows/scripts/windows-in-container-build.sh 2>&1 | tee build.log
+```
+
+To get the built binary out of the container
+```
+docker exec -t mybuilder bash cp /subsurface-installer.exe /win/win32
+```
+Which will copy the installer into the win32 folder which will be a sibling of the subsurface folder.
+
+## Modifying the container
+If you want to make changes to the build environment used in the conatiner
+The script scripts/docker/mxe-build-container.sh will build the Docker image itself.
+The sha of the version of MXE we are using is built into this, so you can update that to whatever version is required, and modify dockerfiles and settings-stage1.mk and settings-stage2.mk to pull in any other prerequisites as required.
+
+If you are working on updating the container then you should incrment the minor version of the variable VERSION in the script as otherwise it will clash with the version used in production
+

scripts/docker/mxe-build-container/qtconnectivity-1.patch

@@ -0,0 +1,12 @@
+diff --git a/src/bluetooth/configure.json b/src/bluetooth/configure.json
+index 3bd95903..0d41d88f 100644
+--- a/src/bluetooth/configure.json
++++ b/src/bluetooth/configure.json
+@@ -63,6 +63,7 @@
+             "purpose": "Uses the native Win32 Bluetooth backend.",
+             "section": "Qt Bluetooth",
+             "autoDetect": false,
++            "enable": true,
+             "output": [ "publicFeature", "feature" ]
+         },
+         "winrt_bt": {

scripts/docker/mxe-build-container/settings-stage2.mk

@@ -9,7 +9,32 @@ JOBS := 2
 MXE_TARGETS :=  i686-w64-mingw32.shared
 
 # The three lines below makes `make` build these "local packages" instead of all packages.
-LOCAL_PKG_LIST := qtbase qtconnectivity qtdeclarative qtimageformats qtlocation qtmultimedia qtquickcontrols qtquickcontrols2 qtscript qtsvg qttools qttranslations qtwebview qtwebkit libxml2 libxslt libusb1 libgit2 nsis curl libzip libftdi1
+LOCAL_PKG_LIST := curl \
+                  grantlee \
+                  hidapi \
+                  libftdi1 \
+                  libgit2 \
+                  libusb1 \
+                  libxml2 \
+                  libxslt \
+                  libzip \
+                  mdbtools \
+                  nsis \
+                  qtbase \
+                  qtconnectivity \
+                  qtdeclarative \
+                  qtimageformats \
+                  qtlocation \
+                  qtmultimedia \
+                  qtquickcontrols \
+                  qtquickcontrols2 \
+                  qtscript \
+                  qtsvg \
+                  qttools \
+                  qttranslations \
+                  qtwebkit \
+                  qtwebview \
+                  zstd
 .DEFAULT local-pkg-list:
 local-pkg-list: $(LOCAL_PKG_LIST)
 

smtk-import/CMakeLists.txt

@@ -92,7 +92,6 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
 if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
         set(SMTK_LINK_LIBRARIES ${SMTK_LINK_LIBRARIES} -lssh2 -lz -lpthread)
 elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
-        set(SMTK_LINK_LIBRARIES ${SMTK_LINK_LIBRARIES} -lwsock32 -lws2_32 -lole32 -limm32 -lwinmm)
 	remove_definitions(-DUNICODE)
         add_definitions(-mwindows -D_WIN32)
 endif()
@@ -109,6 +108,13 @@ set(SMTK_LINK_LIBRARIES ${SMTK_LINK_LIBRARIES} ${SUBSURFACE_LINK_LIBRARIES} ${SS
 source_group("SmartTrak Import libs" FILES ${SMTK_IMPORT_SRCS})
 set(SMTK_IMPORT_TARGET smtk2ssrf)
 add_library(smtk_import STATIC ${SMTK_IMPORT_SRCS})
+
+
+if(CMAKE_SYSTEM_NAME STREQUAL "Windows")
+	# Setting the link target this way resolves an issue with not importing timegettime correctly fron winmm
+	target_link_libraries(smtk_import PRIVATE wsock32 ws2_32 ole32 imm32 winmm)
+endif()
+
 add_executable(${SMTK_IMPORT_TARGET} smtk_standalone.cpp ${SUBSURFACE_RESOURCES})
 
 # We just want CLI mode on Linux. Silently drop it if cross building to Windows.