24.04 (#1904) (#1942)

* Changes to support building on Ubuntu 24.04 and windows-2022 Github runner.
* Update Python to 3.12
    * Install dependencies in virtual environment
    * Run python scripts from the virtual environment)
* Update dependencies
    * Ubuntu dependencies in snap/snapcraft24.yaml
    * Python dependencies in requirements.txt
    * Windows dependencies built in OpenDroneMap/windows-deps repo
* Update CUDA
    * 12.8.1 on Windows
    * 13.0.0 on Ubuntu)
* Run tests as part of docker build
* Use exact commits to specify dependencies that are built from source, instead of branch names
* Use upstream versions of Libraries:
    * PDAL
    * PDAL-Python
    * untwine
    * ExifRead
    * draco
* Build windows builds with -j2
* Use Micasense's latest version of dls.py
* Update failing unit tests to match current behavior

Co-authored-by: Stephen Mather <[email protected]>

Author: spwoodcock

.dockerignore

@@ -1,5 +1,4 @@
 Dockerfile
-tests/test_data
 SuperBuild/build
 SuperBuild/download
 SuperBuild/install
@@ -18,8 +17,6 @@ images_resize
 /hooks
 /img
 /license
-/tests
-tests.sh
 settings.yml
 code_of_conduct.md
 configure_18_04.sh

.github/workflows/build_vcpkg_env.yaml

@@ -0,0 +1,28 @@
+name: Build Windows Dependencies
+
+on:
+  workflow_dispatch:
+
+jobs:
+  windows:
+    runs-on: windows-2022
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up vcpkg
+      run: |
+        git clone https://github.com/microsoft/vcpkg.git
+        .\vcpkg\bootstrap-vcpkg.bat
+
+    - name: Install vcpkg dependencies
+      run: .\vcpkg\vcpkg.exe install --triplet x64-windows
+
+    - name: Export vcpkg dependencies
+      run: .\vcpkg\vcpkg.exe export --zip --output=vcpkg-export --output-dir=.
+
+    - name: Upload artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: vcpkg-env-x64-windows
+        path: vcpkg-export.zip

.github/workflows/publish-windows.yml

@@ -16,12 +16,12 @@ jobs:
     - name: Setup Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.8.1'
+        python-version: '3.12.10'
         architecture: 'x64'
-    - uses: Jimver/[email protected].4
+    - uses: Jimver/[email protected].24
       id: cuda-toolkit
       with:
-        cuda: '11.4.0'
+        cuda: '12.8.1'
     - name: Setup cmake
       uses: jwlawson/[email protected]
       with:

.github/workflows/test-build-prs.yaml

@@ -49,19 +49,19 @@ jobs:
   #       isClassic: 'false'
 
   windows:
-    runs-on: windows-2019
+    runs-on: windows-2022
     steps:
     - name: Checkout
       uses: actions/checkout@v2
     - name: Setup Python
       uses: actions/setup-python@v2
       with:
-        python-version: '3.8.1'
+        python-version: '3.12.10'
         architecture: 'x64'
-    - uses: Jimver/[email protected].4
+    - uses: Jimver/[email protected].24
       id: cuda-toolkit
       with:
-        cuda: '11.4.0'
+        cuda: '12.8.1'
     - name: Setup cmake
       uses: jwlawson/[email protected]
       with:

.gitignore

@@ -32,7 +32,7 @@ storage/
 
 vcpkg/
 venv/
-python38/
+python312/
 dist/
 innosetup/
 .DS_Store

Dockerfile

@@ -1,8 +1,8 @@
-FROM ubuntu:21.04 AS builder
+FROM ubuntu:24.04 AS builder
 
 # Env variables
 ENV DEBIAN_FRONTEND=noninteractive \
-    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.9/dist-packages:/code/SuperBuild/install/lib/python3.8/dist-packages:/code/SuperBuild/install/bin/opensfm" \
+    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
     LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib"
 
 # Prepare directories
@@ -11,24 +11,25 @@ WORKDIR /code
 # Copy everything
 COPY . ./
 
-# Use old-releases for 21.04
-RUN printf "deb http://old-releases.ubuntu.com/ubuntu/ hirsute main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-backports main restricted universe multiverse" > /etc/apt/sources.list
-
 # Run the build
 RUN bash configure.sh install
 
+# Run the tests
+ENV PATH="/code/venv/bin:$PATH"
+RUN bash test.sh
+
 # Clean Superbuild
 RUN bash configure.sh clean
 
 ### END Builder
 
 ### Use a second image for the final asset to reduce the number and
 # size of the layers.
-FROM ubuntu:21.04
+FROM ubuntu:24.04
 
 # Env variables
 ENV DEBIAN_FRONTEND=noninteractive \
-    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.9:/code/SuperBuild/install/lib/python3.8/dist-packages:/code/SuperBuild/install/bin/opensfm" \
+    PYTHONPATH="$PYTHONPATH:/code/SuperBuild/install/lib/python3.12/dist-packages:/code/SuperBuild/install/bin/opensfm" \
     LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/code/SuperBuild/install/lib" \
     PDAL_DRIVER_PATH="/code/SuperBuild/install/bin"
 
@@ -37,11 +38,7 @@ WORKDIR /code
 # Copy everything we built from the builder
 COPY --from=builder /code /code
 
-# Copy the Python libraries installed via pip from the builder
-COPY --from=builder /usr/local /usr/local
-
-# Use old-releases for 21.04
-RUN printf "deb http://old-releases.ubuntu.com/ubuntu/ hirsute main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates main restricted\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates universe\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-updates multiverse\ndeb http://old-releases.ubuntu.com/ubuntu/ hirsute-backports main restricted universe multiverse" > /etc/apt/sources.list
+ENV PATH="/code/venv/bin:$PATH"
 
 # Install shared libraries that we depend on via APT, but *not*
 # the -dev packages to save space!
@@ -51,5 +48,6 @@ RUN bash configure.sh installruntimedepsonly \
   && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
   && bash run.sh --help \
   && bash -c "eval $(python3 /code/opendm/context.py) && python3 -c 'from opensfm import io, pymap'"
+
 # Entry point
 ENTRYPOINT ["python3", "/code/run.py"]

README.md

@@ -130,9 +130,9 @@ You're in good shape!
 
 See https://github.com/NVIDIA/nvidia-docker and https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker for information on docker/NVIDIA setup.
 
-## Native Install (Ubuntu 21.04)
+## Native Install (Ubuntu 24.04)
 
-You can run ODM natively on Ubuntu 21.04 (although we don't recommend it):  
+You can run ODM natively on Ubuntu 24.04 (although we don't recommend it):  
 
 ```bash
 git clone https://github.com/OpenDroneMap/ODM
@@ -144,6 +144,8 @@ You can then process datasets with `./run.sh /datasets/odm_data_aukerman`
 
 ## Native Install (MacOS)
 
+> **Warning:** Installation on Mac is currently unmaintained, and may not work out-of-the-box. See this [issue](https://community.opendronemap.org/t/odm-install-on-a-mac-os-14-6-1/25007/3).
+
 You can run ODM natively on Intel/ARM MacOS.
 
 First install:

SuperBuild/CMakeLists.txt

@@ -96,14 +96,9 @@ if (WIN32)
 
   # Use the GDAL version that comes with pip
   set(GDAL_ROOT "${PYTHON_HOME}/Lib/site-packages/osgeo")
-  set(GDAL_LIBRARY "${GDAL_ROOT}/lib/gdal_i.lib")
+  set(GDAL_LIBRARY "${GDAL_ROOT}/gdal.lib")
   set(GDAL_INCLUDE_DIR "${GDAL_ROOT}/include/gdal")
 
-  # Also download missing headers :/
-  if (NOT EXISTS "${GDAL_INCLUDE_DIR}/ogrsf_frmts.h")
-    file(DOWNLOAD "https://raw.githubusercontent.com/OSGeo/gdal/release/3.2/gdal/ogr/ogrsf_frmts/ogrsf_frmts.h" "${GDAL_INCLUDE_DIR}/ogrsf_frmts.h")
-  endif()
-
   message("Copying VCPKG DLLs...")
   file(GLOB COPY_DLLS "${VCPKG_ROOT}/installed/x64-windows/bin/*.dll")
   file(COPY ${COPY_DLLS} DESTINATION "${SB_INSTALL_DIR}/bin")
@@ -117,7 +112,8 @@ elseif(APPLE)
     set(PYTHON_HOME "${SB_ROOT_DIR}/../venv")
     set(PYTHON_EXE_PATH "${PYTHON_HOME}/bin/python")
 else()
-  set(PYTHON_EXE_PATH "/usr/bin/python3")
+  set(PYTHON_HOME "${SB_ROOT_DIR}/../venv")
+  set(PYTHON_EXE_PATH "${PYTHON_HOME}/bin/python3")
 endif()
 
 # Path to additional CMake modules
@@ -133,7 +129,7 @@ include(ExternalProject-Setup)
 # ---------------------------------------------------------------------------------------------
 # Open Source Computer Vision (OpenCV)
 #
-set(ODM_OpenCV_Version 2.4.11)
+set(ODM_OpenCV_Version 4.12.0)
 option(ODM_BUILD_OpenCV "Force to build OpenCV library" OFF)
 
 SETUP_EXTERNAL_PROJECT(OpenCV ${ODM_OpenCV_Version} ${ODM_BUILD_OpenCV})
@@ -160,7 +156,7 @@ SETUP_EXTERNAL_PROJECT(Ceres ${ODM_Ceres_Version} ${ODM_BUILD_Ceres})
 # ---------------------------------------------------------------------------------------------
 # Hexer
 #
-SETUP_EXTERNAL_PROJECT(Hexer 1.4 ON)
+SETUP_EXTERNAL_PROJECT(Hexer 1.5 ON)
 
 # ---------------------------------------------------------------------------------------------
 # Open Structure from Motion (OpenSfM)
@@ -269,9 +265,9 @@ externalproject_add(lastools
 )
 
 externalproject_add(draco
-    GIT_REPOSITORY    https://github.com/OpenDroneMap/draco
-    GIT_SHALLOW       ON
-    GIT_TAG           304
+    GIT_REPOSITORY    https://github.com/google/draco
+    GIT_SHALLOW       OFF
+    GIT_TAG           e801aebf9665f6dcf3b6b0a604055a300ff85aa4
     PREFIX            ${SB_BINARY_DIR}/draco
     SOURCE_DIR        ${SB_SOURCE_DIR}/draco
     CMAKE_ARGS        -DDRACO_TRANSCODER_SUPPORTED=ON 

SuperBuild/cmake/External-Ceres.cmake

@@ -11,6 +11,7 @@ ExternalProject_Add(${_proj_name}
   URL               http://ceres-solver.org/ceres-solver-2.0.0.tar.gz
   #--Update/Patch step----------
   UPDATE_COMMAND    ""
+  PATCH_COMMAND    git apply ${CMAKE_MODULE_PATH}/ceres.patch
   #--Configure step-------------
   SOURCE_DIR        ${SB_SOURCE_DIR}/${_proj_name}
   CMAKE_ARGS

SuperBuild/cmake/External-Entwine.cmake

@@ -13,7 +13,7 @@ ExternalProject_Add(${_proj_name}
   #--Download step--------------
   DOWNLOAD_DIR      ${SB_DOWNLOAD_DIR}
   GIT_REPOSITORY    https://github.com/OpenDroneMap/entwine/
-  GIT_TAG           290
+  GIT_TAG           0cf957432f291e841ff1385085dadad933dcba8d
   #--Update/Patch step----------
   UPDATE_COMMAND    ""
   #--Configure step-------------
@@ -23,6 +23,8 @@ ExternalProject_Add(${_proj_name}
     -DADDITIONAL_LINK_DIRECTORIES_PATHS=${SB_INSTALL_DIR}/lib
     -DWITH_TESTS=OFF
     -DWITH_ZSTD=OFF
+    -DWITH_CURL=OFF
+    -DWITH_OPENSSL=OFF
     -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
     -DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
   #--Build step-----------------