Merge pull request #2619 from mzakharo/scipy_r21e

lapack/scipy: support NDK r21e, x86/64 archs

Author: AndreMiras

ci/makefiles/android.mk

@@ -2,7 +2,7 @@
 
 # Those android NDK/SDK variables can be override when running the file
 ANDROID_NDK_VERSION ?= 23b
-ANDROID_NDK_VERSION_LEGACY ?= 19c
+ANDROID_NDK_VERSION_LEGACY ?= 21e
 ANDROID_SDK_TOOLS_VERSION ?= 6514223
 ANDROID_SDK_BUILD_TOOLS_VERSION ?= 29.0.3
 ANDROID_HOME ?= $(HOME)/.android
@@ -93,12 +93,14 @@ extract_android_ndk_legacy:
 	&& rm -f $(ANDROID_NDK_ARCHIVE_LEGACY)
 
 extract_android_ndk_gfortran:
-	rm -rf $(ANDROID_NDK_HOME_LEGACY)/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/
-	mkdir  $(ANDROID_NDK_HOME_LEGACY)/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/
-	tar -xvf $(ANDROID_NDK_GFORTRAN_ARCHIVE_ARM64) -C $(ANDROID_NDK_HOME_LEGACY)/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/ --strip-components 1
-	rm -rf $(ANDROID_NDK_HOME_LEGACY)/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/
-	mkdir  $(ANDROID_NDK_HOME_LEGACY)/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/
-	tar -xvf $(ANDROID_NDK_GFORTRAN_ARCHIVE_ARM) -C $(ANDROID_NDK_HOME_LEGACY)/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/ --strip-components 1
+	rm -rf $(ANDROID_NDK_HOME_LEGACY)/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/ \
+	&& mkdir  $(ANDROID_NDK_HOME_LEGACY)/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/ \
+	&& tar -xf $(ANDROID_NDK_GFORTRAN_ARCHIVE_ARM64) -C $(ANDROID_NDK_HOME_LEGACY)/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/ --strip-components 1 \
+	&& rm -f $(ANDROID_NDK_GFORTRAN_ARCHIVE_ARM64) \
+	&& rm -rf $(ANDROID_NDK_HOME_LEGACY)/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/ \
+	&& mkdir  $(ANDROID_NDK_HOME_LEGACY)/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/ \
+	&& tar -xf $(ANDROID_NDK_GFORTRAN_ARCHIVE_ARM) -C $(ANDROID_NDK_HOME_LEGACY)/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/ --strip-components 1 \
+	&& rm -f $(ANDROID_NDK_GFORTRAN_ARCHIVE_ARM)
 
 
 

pythonforandroid/recipes/lapack/__init__.py

@@ -1,5 +1,5 @@
 '''
-known to build with cmake version 3.19.2 and NDK r19c.
+known to build with cmake version 3.23.2 and NDK r21e.
 See https://gitlab.kitware.com/cmake/cmake/-/issues/18739
 '''
 
@@ -15,10 +15,16 @@
 arch_to_sysroot = {'armeabi': 'arm', 'armeabi-v7a': 'arm', 'arm64-v8a': 'arm64'}
 
 
+def arch_to_toolchain(arch):
+    if 'arm' in arch.arch:
+        return arch.command_prefix
+    return arch.arch
+
+
 class LapackRecipe(Recipe):
 
     name = 'lapack'
-    version = 'v3.9.0'
+    version = 'v3.10.1'
     url = 'https://github.com/Reference-LAPACK/lapack/archive/{version}.tar.gz'
     libdir = 'build/install/lib'
     built_libraries = {'libblas.so': libdir, 'liblapack.so': libdir, 'libcblas.so': libdir}
@@ -28,14 +34,14 @@ def get_recipe_env(self, arch):
 
         ndk_dir = environ.get("LEGACY_NDK")
         if ndk_dir is None:
-            raise BuildInterruptingException("Please set the environment variable 'LEGACY_NDK' to point to a NDK location with gcc/gfortran support (last tested NDK version was 'r19c')")
+            raise BuildInterruptingException("Please set the environment variable 'LEGACY_NDK' to point to a NDK location with gcc/gfortran support (supported NDK version: 'r21e')")
 
         GCC_VER = '4.9'
         HOST = build_platform
 
         sysroot_suffix = arch_to_sysroot.get(arch.arch, arch.arch)
         sysroot = f"{ndk_dir}/platforms/{env['NDK_API']}/arch-{sysroot_suffix}"
-        FC = f"{ndk_dir}/toolchains/{arch.command_prefix}-{GCC_VER}/prebuilt/{HOST}/bin/{arch.command_prefix}-gfortran"
+        FC = f"{ndk_dir}/toolchains/{arch_to_toolchain(arch)}-{GCC_VER}/prebuilt/{HOST}/bin/{arch.command_prefix}-gfortran"
         env['FC'] = f'{FC} --sysroot={sysroot}'
         if sh.which(FC) is None:
             raise BuildInterruptingException(f"{FC} not found. See https://github.com/mzakharo/android-gfortran")
@@ -51,19 +57,20 @@ def build_arch(self, arch):
             env = self.get_recipe_env(arch)
             ndk_dir = environ["LEGACY_NDK"]
             shprint(sh.rm, '-rf', 'CMakeFiles/', 'CMakeCache.txt', _env=env)
-            shprint(sh.cmake, source_dir,
+            opts = [
                     '-DCMAKE_SYSTEM_NAME=Android',
                     '-DCMAKE_POSITION_INDEPENDENT_CODE=1',
                     '-DCMAKE_ANDROID_ARCH_ABI={arch}'.format(arch=arch.arch),
                     '-DCMAKE_ANDROID_NDK=' + ndk_dir,
+                    '-DCMAKE_ANDROID_API={api}'.format(api=self.ctx.ndk_api),
                     '-DCMAKE_BUILD_TYPE=Release',
                     '-DCMAKE_INSTALL_PREFIX={}'.format(install_target),
-                    '-DANDROID_ABI={arch}'.format(arch=arch.arch),
-                    '-DANDROID_ARM_NEON=ON',
-                    '-DENABLE_NEON=ON',
                     '-DCBLAS=ON',
                     '-DBUILD_SHARED_LIBS=ON',
-                    _env=env)
+                    ]
+            if arch.arch == 'armeabi-v7a':
+                opts.append('-DCMAKE_ANDROID_ARM_NEON=ON')
+            shprint(sh.cmake, source_dir, *opts, _env=env)
             shprint(sh.make, '-j' + str(cpu_count()), _env=env)
             shprint(sh.make, 'install', _env=env)
 

pythonforandroid/recipes/numpy/__init__.py

@@ -19,7 +19,6 @@ class NumpyRecipe(CompiledComponentsPythonRecipe):
     patches = [
         join("patches", "remove-default-paths.patch"),
         join("patches", "add_libm_explicitly_to_build.patch"),
-        join("patches", "fix-missing-threads.h.patch"),
     ]
 
     def get_recipe_env(self, arch=None, with_flags_in_cc=True):

pythonforandroid/recipes/numpy/patches/fix-missing-threads.h.patch

@@ -4,7 +4,11 @@
 from os import environ
 from pythonforandroid.util import build_platform
 
-arch_to_sysroot = {'armeabi': 'arm', 'armeabi-v7a': 'arm', 'arm64-v8a': 'arm64'}
+
+def arch_to_toolchain(arch):
+    if 'arm' in arch.arch:
+        return arch.command_prefix
+    return arch.arch
 
 
 class ScipyRecipe(CompiledComponentsPythonRecipe):
@@ -35,16 +39,12 @@ def get_recipe_env(self, arch):
         suffix = '64' if '64' in arch.arch else ''
 
         prefix = arch.command_prefix
-        sysroot_suffix = arch_to_sysroot.get(arch.arch, arch.arch)
-        sysroot = f"{ndk_dir}/platforms/{env['NDK_API']}/arch-{sysroot_suffix}"
-        sysroot_include = f'{ndk_dir}/toolchains/llvm/prebuilt/{HOST}/sysroot/usr/include'
         CLANG_BIN = f'{ndk_dir}/toolchains/llvm/prebuilt/{HOST}/bin/'
-        GCC = f'{ndk_dir}/toolchains/{prefix}-{GCC_VER}/prebuilt/{HOST}'
+        GCC = f'{ndk_dir}/toolchains/{arch_to_toolchain(arch)}-{GCC_VER}/prebuilt/{HOST}'
         libgfortran = f'{GCC}/{prefix}/lib{suffix}'
-
         numpylib = self.ctx.get_python_install_dir(arch.arch) + '/numpy'
-        LDSHARED_opts = env['LDSHARED'].split('clang')[1]
         arch_cflags = ' '.join(arch.arch_cflags)
+        LDSHARED_opts = f'-target {arch.target} {arch_cflags} ' + ' '.join(arch.common_ldshared)
 
         # TODO: add pythran support
         env['SCIPY_USE_PYTHRAN'] = '0'
@@ -58,13 +58,16 @@ def get_recipe_env(self, arch):
         env['F90'] = f'{GCC}/bin/{prefix}-gfortran'
         env['CC'] = f'{CLANG_BIN}/clang -target {arch.target} {arch_cflags}'
         env['CXX'] = f'{CLANG_BIN}/clang++ -target {arch.target} {arch_cflags}'
+
+        # scipy expects ldshared to be a single executable without options
         env['LDSHARED'] = f'{CLANG_BIN}/clang'
 
-        # flags
-        env['CPPFLAGS'] = f'-DANDROID -I{sysroot_include}/{prefix} --sysroot={sysroot} -I{sysroot_include}/c++/v1 -I{sysroot_include}'
-        env['LDFLAGS'] += f' {LDSHARED_opts} --sysroot={sysroot} -L{libgfortran} -L{numpylib}/core/lib -L{numpylib}/random/lib'
-        env['LDFLAGS'] += f' -l{self.stl_lib_name} '
-        env['LDFLAGS'] += f' -L{ndk_dir}/sources/cxx-stl/llvm-libc++/libs/{arch.arch}/'  # for arm32 - unwind
+        # erase the default NDK C++ include options
+        env['CPPFLAGS'] = '-DANDROID'
+
+        # configure linker
+        env['LDFLAGS'] += f' {LDSHARED_opts} -L{libgfortran} -L{numpylib}/core/lib -L{numpylib}/random/lib'
+        env['LDFLAGS'] += f' -l{self.stl_lib_name}'
         return env