WIP: single stage Docker build for 64bit MXE

Signed-off-by: Dirk Hohndel <dirk@hohndel.org>

.github/workflows/codeql-analysis.yml

@@ -47,7 +47,7 @@ jobs:
         qml-module-qtquick2 qt5-default qt5-qmake qtchooser qtconnectivity5-dev \
         qtdeclarative5-dev qtdeclarative5-private-dev qtlocation5-dev \
         qtpositioning5-dev qtscript5-dev qttools5-dev qttools5-dev-tools \
-        qtquickcontrols2-5-dev xvfb libbluetooth-dev
+        qtquickcontrols2-5-dev xvfb libbluetooth-dev libmtp-dev
 
 
     # Initializes the CodeQL tools for scanning.

.github/workflows/coverity-scan.yml

@@ -25,7 +25,7 @@ jobs:
         qml-module-qtquick2 qt5-default qt5-qmake qtchooser qtconnectivity5-dev \
         qtdeclarative5-dev qtdeclarative5-private-dev qtlocation5-dev \
         qtpositioning5-dev qtscript5-dev qttools5-dev qttools5-dev-tools \
-        qtquickcontrols2-5-dev wget curl libbluetooth-dev
+        qtquickcontrols2-5-dev wget curl libbluetooth-dev libmtp-dev
 
     - name: Download Coverity Build Tool
       run: |

.github/workflows/linux-bionic-5.9.yml

@@ -29,7 +29,7 @@ jobs:
         qml-module-qtquick2 qt5-default qt5-qmake qtchooser qtconnectivity5-dev \
         qtdeclarative5-dev qtdeclarative5-private-dev qtlocation5-dev \
         qtpositioning5-dev qtscript5-dev qttools5-dev qttools5-dev-tools \
-        qtquickcontrols2-5-dev xvfb libbluetooth-dev
+        qtquickcontrols2-5-dev xvfb libbluetooth-dev libmtp-dev
 
     - name: build Subsurface
       run: |

.github/workflows/linux-eoan-5.12.yml

@@ -33,7 +33,7 @@ jobs:
         qml-module-qtquick2 qt5-default qt5-qmake qtchooser qtconnectivity5-dev \
         qtdeclarative5-dev qtdeclarative5-private-dev qtlocation5-dev \
         qtpositioning5-dev qtscript5-dev qttools5-dev qttools5-dev-tools \
-        qtquickcontrols2-5-dev xvfb libbluetooth-dev
+        qtquickcontrols2-5-dev xvfb libbluetooth-dev libmtp-dev
 
     - name: build Subsurface-mobile
       run: |

.github/workflows/linux-groovy-5.14.yml

@@ -33,7 +33,7 @@ jobs:
         qml-module-qtquick2 qt5-default qt5-qmake qtchooser qtconnectivity5-dev \
         qtdeclarative5-dev qtdeclarative5-private-dev qtlocation5-dev \
         qtpositioning5-dev qtscript5-dev qttools5-dev qttools5-dev-tools \
-        qtquickcontrols2-5-dev xvfb libbluetooth-dev
+        qtquickcontrols2-5-dev xvfb libbluetooth-dev libmtp-dev
 
     - name: build Subsurface-mobile
       run: |

.github/workflows/linux-tumbleweed-qt-latest.yml

@@ -28,7 +28,7 @@ jobs:
         libqt5-qtscript-devel libqt5-qtdeclarative-devel \
         libqt5-qtconnectivity-devel libqt5-qtlocation-devel libcurl-devel \
         libQt5QuickControls2-devel bluez-devel \
-        which libgit2-devel libssh2-devel
+        which libgit2-devel libssh2-devel libmtp-devel
 
 # if we want to run the tests below, add    xvfb-run
 

.github/workflows/mac.yml

@@ -14,7 +14,7 @@ jobs:
     - name: checkout sources
       uses: actions/checkout@v1
     - name: setup Homebrew
-      run: brew install autoconf automake libtool xz hidapi libusb libxml2 libxslt libzip openssl pkg-config libgit2 libssh2 libjpg libpng
+      run: brew install autoconf automake libtool xz hidapi libusb libxml2 libxslt libzip openssl pkg-config libgit2 libssh2 libjpg libpng libmtp
     - name: set our Qt build
       run: |
         mkdir -p Qt/5.13.0

.github/workflows/scripts/linux-in-container-build.sh

@@ -11,7 +11,7 @@ export CMAKE_PREFIX_PATH=$QT_ROOT/lib/cmake
 
 echo "--------------------------------------------------------------"
 echo "install missing packages"
-apt install -y libbluetooth-dev
+apt install -y libbluetooth-dev libmtp-dev
 
 # the container currently has things under / that need to be under /__w/subsurface/subsurface instead
 cp -a /appdir /__w/subsurface/

.github/workflows/scripts/windows-container-prep.sh

@@ -5,3 +5,4 @@
 echo "downloading sources for fresh build"
 bash subsurface/scripts/get-dep-lib.sh single . libzip
 bash subsurface/scripts/get-dep-lib.sh single . googlemaps
+bash subsurface/scripts/get-dep-lib.sh single . libmtp

CMakeLists.txt

@@ -156,6 +156,7 @@ if(NOT ANDROID)
 		pkg_config_library(BLUEZ bluez REQUIRED)
 	endif()
 	pkg_config_library(LIBUSB libusb-1.0 QUIET)
+	pkg_config_library(LIBMTP libmtp QUIET)
 endif()
 
 include_directories(.
@@ -284,7 +285,7 @@ if(NOT ANDROID)
 endif()
 
 #set up the subsurface_link_libraries variable
-set(SUBSURFACE_LINK_LIBRARIES ${SUBSURFACE_LINK_LIBRARIES} ${LIBDIVECOMPUTER_LIBRARIES} ${LIBGIT2_LIBRARIES} ${LIBUSB_LIBRARIES})
+set(SUBSURFACE_LINK_LIBRARIES ${SUBSURFACE_LINK_LIBRARIES} ${LIBDIVECOMPUTER_LIBRARIES} ${LIBGIT2_LIBRARIES} ${LIBUSB_LIBRARIES} ${LIBMTP_LIBRARIES})
 qt5_add_resources(SUBSURFACE_RESOURCES subsurface.qrc map-widget/qml/map-widget.qrc)
 
 # hack to build successfully on LGTM
@@ -473,7 +474,7 @@ elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
 elseif(CMAKE_SYSTEM_NAME STREQUAL "Windows")
 	if (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
 		if(NOT DEFINED OBJCOPY)
-			set(OBJCOPY i686-w64-mingw32.shared-objcopy)
+			set(OBJCOPY x86_64-w64-mingw32.shared-objcopy)
 		endif()
 		find_program(OBJCOPY_FOUND ${OBJCOPY})
 		if (OBJCOPY_FOUND)

INSTALL

@@ -144,7 +144,7 @@ sudo dnf install autoconf automake bluez-libs-devel cmake gcc-c++ git \
 	qt5-qtbase-devel qt5-qtconnectivity-devel qt5-qtdeclarative-devel \
 	qt5-qtlocation-devel qt5-qtscript-devel qt5-qtsvg-devel \
 	qt5-qttools-devel qt5-qtwebkit-devel redhat-rpm-config \
-	bluez-libs-devel libgit2-devel libzip-devel
+	bluez-libs-devel libgit2-devel libzip-devel libmtp-devel
 
 
 Package names are sadly different on OpenSUSE
@@ -155,7 +155,7 @@ sudo zypper install git gcc-c++ make autoconf automake libtool cmake libzip-deve
 	libqt5-qtbase-devel libQt5WebKit5-devel libqt5-qtsvg-devel \
 	libqt5-qtscript-devel libqt5-qtdeclarative-devel \
 	libqt5-qtconnectivity-devel libqt5-qtlocation-devel libcurl-devel \
-	bluez-devel libgit2-devel
+	bluez-devel libgit2-devel libmtp-devel
 
 On Debian Buster this seems to work
 
@@ -167,7 +167,7 @@ sudo apt install \
 	qml-module-qtlocation qml-module-qtpositioning qml-module-qtquick2 \
 	qt5-default qt5-qmake qtchooser qtconnectivity5-dev qtdeclarative5-dev \
 	qtdeclarative5-private-dev qtlocation5-dev qtpositioning5-dev \
-	qtscript5-dev qttools5-dev qttools5-dev-tools
+	qtscript5-dev qttools5-dev qttools5-dev-tools libmtp-dev
 
 In order to build and run mobile-on-desktop, you also need
 
@@ -187,7 +187,7 @@ sudo apt install \
 	qml-module-qtlocation qml-module-qtpositioning qml-module-qtquick2 \
 	qt5-default qt5-qmake qtchooser qtconnectivity5-dev qtdeclarative5-dev \
 	qtdeclarative5-private-dev qtlocation5-dev qtpositioning5-dev \
-	qtscript5-dev qttools5-dev qttools5-dev-tools
+	qtscript5-dev qttools5-dev qttools5-dev-tools libmtp-dev
 
 In order to build and run mobile-on-desktop, you also need
 
@@ -207,7 +207,7 @@ sudo apt install \
 	qml-module-qtlocation qml-module-qtpositioning qml-module-qtquick2 \
 	qt5-default qt5-qmake qtchooser qtconnectivity5-dev qtdeclarative5-dev \
 	qtdeclarative5-private-dev qtlocation5-dev qtpositioning5-dev \
-	qtscript5-dev qttools5-dev qttools5-dev-tools
+	qtscript5-dev qttools5-dev qttools5-dev-tools libmtp-dev
 
 In order to build and run mobile-on-desktop, you also need
 
@@ -281,7 +281,7 @@ what else you have installed on your mac.
 
 homebrew option:
 
-1) brew install libzip cmake libusb pkg-config automake libtool hidapi curl libssh2
+1) brew install autoconf automake libtool hidapi libusb libxml2 libxslt libzip openssl pkg-config libgit2 libssh2 libjpg libpng libmtp
 
 2) cd <repo>/..; bash <repo>/scripts/build.sh
 

libdivecomputer

@@ -1 +1 @@
-Subproject commit 0ebd5d3879854206cd1cf0272e6c357004b33727
+Subproject commit c9537fb0fae8e26acb868583a104dd370483e1e6

packaging/windows/create-win-installer.sh

@@ -12,12 +12,12 @@ bash subsurface/.github/workflows/scripts/windows-container-prep.sh
 rm mdbtools/include/mdbver.h
 
 # build the installer
-rm -rf win32
+rm -rf win64
-mkdir win32
+mkdir win64
-cd win32
+cd win64
 
 # build Subsurface and then smtk2ssrf
-export MXEBUILDTYPE=i686-w64-mingw32.shared
+export MXEBUILDTYPE=x86_64-w64-mingw32.shared
 bash -ex ../subsurface/packaging/windows/mxe-based-build.sh installer
 mv subsurface/subsurface-*.exe /__w
 

packaging/windows/mxe-based-build.sh

@@ -17,7 +17,7 @@
 # JOBS := 1
 #
 # # This variable controls the targets that will build.
-# MXE_TARGETS :=  i686-w64-mingw32.shared.posix.dw2
+# MXE_TARGETS :=  x86_64-w64-mingw32.shared
 #
 # # Uncomment the next line if you want to do debug builds later
 # # qtbase_CONFIGURE_OPTS=-debug-and-release
@@ -101,7 +101,7 @@ EXECDIR=`pwd`
 BASEDIR=$(cd "$EXECDIR/.."; pwd)
 BUILDDIR=$(cd "$EXECDIR"; pwd)
 MXEDIR=${MXEDIR:-mxe}
-MXEBUILDTYPE=${MXEBUILDTYPE:-i686-w64-mingw32.shared.posix.dw2}
+MXEBUILDTYPE=${MXEBUILDTYPE:-x86_64-w64-mingw32.shared}
 
 echo $BUILDDIR
 
@@ -153,6 +153,28 @@ if [ ! -f libdivecomputer/configure ] ; then
 	autoreconf --install
 fi
 
+# build libmtp as that isn't available via MXE
+cd "$BUILDDIR"
+if [[ ! -d libmtp || -f build.libmtp ]] ; then
+	rm -f build.libmtp
+	cd "$BASEDIR/libmtp"
+	export NOCONFIGURE=1
+	# crudely disable the interactive part of autogen.sh
+	sed --in-place=.bak 's/read IN/IN="N"/;s/echo "Auto/#echo "Auto/' autogen.sh
+	bash autogen.sh
+	mv autogen.sh.bak autogen.sh
+	cd "$BUILDDIR"
+	mkdir -p libmtp
+	cd libmtp
+	"$BASEDIR"/libmtp/configure \
+		CC="$MXEBUILDTYPE"-gcc \
+		--host="$MXEBUILDTYPE" \
+		--enable-shared \
+		--prefix="$BASEDIR"/"$MXEDIR"/usr/"$MXEBUILDTYPE"
+	make $JOBS
+	make install
+fi
+
 cd "$BUILDDIR"
 CURRENT_SHA=$(cd "$BASEDIR"/subsurface/libdivecomputer ; git describe)
 PREVIOUS_SHA=$(cat "libdivecomputer.SHA" 2>/dev/null || echo)

packaging/windows/subsurface.nsi.in

@@ -28,7 +28,7 @@
     !define MUI_ICON "subsurface.ico"
 
     # Default installation folder
-    InstallDir "$PROGRAMFILES\Subsurface"
+    InstallDir "$PROGRAMFILES64\Subsurface"
 
     # Get installation folder from registry if available
     InstallDirRegKey HKCU "Software\Subsurface" ""

scripts/docker/mxe-build-container/Dockerfile

@@ -3,13 +3,15 @@
 #
 
 # Start from Ubuntu
-From ubuntu:18.04
+From ubuntu:20.04
+
+# very often master is broken, so we pass in a known good SHA
 ARG mxe_sha=master
 ENV _ver=${mxe_sha}
 
 # update and set up the packages we need for this cross build
 RUN apt-get update  &&  apt-get upgrade -y && \
-apt-get install -y \
+DEBIAN_FRONTEND=noninteractive apt-get install --no-install-recommends -y \
     autoconf \
     automake \
     autopoint \
@@ -47,18 +49,24 @@ apt-get install -y \
     lzip \
     scons
 
-# copy over the partial MXE settings (simply missing QtWebKit so we don't exceed the 6h limit)
+# checkout MXE at the right version
 RUN mkdir -p /win
-ADD settings-stage1.mk /win/settings.mk
 RUN cd /win ; git clone git://github.com/mxe/mxe ; \
     cd mxe ; \
     git checkout ${_ver} ;
+
 # Patch the qtconnectivity build to explicilty enable native-win32-bluetooth and ensure another
 # backend is not picked
-ADD qtconnectivity-1.patch /win/qtconnectivity-1.patch
+ADD qtconnectivity-1.patch /win/mxe/src/qtconnectivity-1.patch
-RUN mv /win/qtconnectivity-1.patch /win/mxe/src
+
-RUN mv /win/settings.mk /win/mxe
+# Move the settings into place to build everything that we need
+# separate download from build so that we can redo the build
+ADD settings.mk /win/mxe/settings.mk
 RUN cd /win/mxe ; \
-    make -j 6 2>&1 | tee build.log ;
+    make -j 6 download 2>&1 | tee mxe-build.log
 RUN cd /win/mxe ; \
-    make MXE_TARGETS=i686-w64-mingw32.static glib -j 6 2>&1 | tee -a build.log ;
+    make -j 6 2>&1 | tee -a mxe-build.log ;
+
+# for some reason smtk2ssrf needs a static build of mdbtools
+RUN cd /win/mxe ; \
+    make MXE_TARGETS=x86_64-w64-mingw32.static glib mdbtools -j 6 2>&1 | tee -a mxe-build.log ;

scripts/docker/mxe-build-container/build-container.sh

@@ -2,10 +2,13 @@
 set -x
 set -e
 
+# known good MXE sha
+MXE_SHA="8966a64"
 SCRIPTPATH=$(dirname $0)
 
-export VERSION=1.1
+# version of the docker image
+VERSION=2.0
+
 pushd $SCRIPTPATH
-docker build -t subsurface/mxe-build-container:$VERSION --build-arg=mxe_sha=1ee37f8 -f Dockerfile-stage1 .
+docker build -t subsurface/mxe-build-container:$VERSION --build-arg=mxe_sha=$MXE_SHA -f Dockerfile .
-docker build -t subsurface/mxe-build-container:$VERSION --build-arg=VERSION=$VERSION -f Dockerfile-stage2 .
 popd

scripts/docker/mxe-build-container/settings.mk

@@ -0,0 +1,37 @@
+# This is a template of configuration file for MXE. See
+# index.html for more extensive documentations.
+
+# This variable controls the number of compilation processes
+# within one package ("intra-package parallelism").
+JOBS := 8
+
+# This variable controls the targets that will build.
+MXE_TARGETS :=  x86_64-w64-mingw32.shared
+
+# The three lines below makes `make` build these "local packages" instead of all packages.
+LOCAL_PKG_LIST := nsis \
+                  curl \
+                  libxml2 \
+                  libxslt \
+                  libzip \
+                  libusb1 \
+                  hidapi \
+                  libgit2 \
+                  libftdi1 \
+                  mdbtools \
+                  qtbase \
+                  qtconnectivity \
+                  qtdeclarative \
+                  qtimageformats \
+                  qtlocation \
+                  qtmultimedia \
+                  qtwebkit \
+                  qtquickcontrols \
+                  qtquickcontrols2 \
+                  qtcharts \
+                  qtsvg \
+                  qttools \
+                  qttranslations \
+                  zstd
+.DEFAULT local-pkg-list:
+local-pkg-list: $(LOCAL_PKG_LIST)

scripts/get-dep-lib.sh

@@ -17,6 +17,7 @@ CURRENT_KIRIGAMI="v5.62.0"
 CURRENT_BREEZE_ICONS="4daac191fb33c8c03bba8356db9767816cb8ee02"
 CURRENT_MDBTOOLS="master"
 CURRENT_QT_ANDROID_CMAKE="master"
+CURRENT_LIBMTP="master"
 
 # Checkout library from git
 # Ensure specified version is checked out,
@@ -155,6 +156,9 @@ for package in "${PACKAGES[@]}" ; do
 		libusb)
 			git_checkout_library libusb $CURRENT_LIBUSB https://github.com/libusb/libusb.git
 			;;
+		libmtp)
+			git_checkout_library libmtp $CURRENT_LIBMTP https://github.com/libmtp/libmtp.git
+			;;
 		libxml2)
 			git_checkout_library libxml2 $CURRENT_LIBXML2 https://github.com/GNOME/libxml2.git
 			;;

scripts/windows-container/in-container-build.sh

@@ -10,11 +10,11 @@
 set -x
 set -e
 
-mkdir -p win32
+mkdir -p win64
-cd win32
+cd win64
 
 # right now the container still has an older version of MXE installed
-export MXEBUILDTYPE=i686-w64-mingw32.shared
+export MXEBUILDTYPE=x86_64-w64-mingw32.shared
 
 bash -ex ../subsurface/packaging/windows/mxe-based-build.sh installer