Build XNNPACK as an ExternalProject

swolchok · swolchok · commit ff134fcc651b · 2025-07-18T01:13:22.000-07:00
I believe this is necessary to support EXPORT. (We probably should not land this unless we have a PR stacked on top of this that successfully converts the build to EXPORT; sending this out so I can get CI going) ghstack-source-id: d179497 ghstack-comment-id: 3067161235 Pull-Request: #12425
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -731,7 +731,10 @@ if(EXECUTORCH_BUILD_EXECUTOR_RUNNER)
     endif()
 
     set(CMAKE_EXECUTABLE_SUFFIX ".html")
-    target_link_options(executor_runner PUBLIC -sALLOW_MEMORY_GROWTH --embed-file "${WASM_MODEL_DIR}@/")
+    target_link_options(
+      executor_runner PUBLIC -sALLOW_MEMORY_GROWTH --embed-file
+      "${WASM_MODEL_DIR}@/"
+    )
   endif()
 endif()
 
diff --git a/backends/xnnpack/CMakeLists.txt b/backends/xnnpack/CMakeLists.txt
@@ -92,9 +92,8 @@ target_include_directories(
                            ${EXECUTORCH_ROOT}/third-party/flatbuffers/include
 )
 
-set(xnnpack_third_party pthreadpool extension_threadpool cpuinfo)
-
 include(cmake/Dependencies.cmake)
+set(xnnpack_third_party XNNPACK pthreadpool extension_threadpool cpuinfo)
 
 list(TRANSFORM _xnnpack_backend__srcs PREPEND "${EXECUTORCH_ROOT}/")
 add_library(xnnpack_backend ${_xnnpack_backend__srcs})
diff --git a/backends/xnnpack/cmake/Dependencies.cmake b/backends/xnnpack/cmake/Dependencies.cmake
@@ -10,73 +10,88 @@ set(THIRD_PARTY_ROOT "${CMAKE_CURRENT_SOURCE_DIR}/third-party")
 
 # --- XNNPACK
 
-# Setting this global PIC flag for all XNNPACK targets. This is needed for
-# Object libraries within XNNPACK which must be PIC to successfully link this
-# static libXNNPACK
-set(ORIGINAL_CMAKE_POSITION_INDEPENDENT_CODE_FLAG
-    ${CMAKE_POSITION_INDEPENDENT_CODE}
-)
-set(CMAKE_POSITION_INDEPENDENT_CODE ON)
-
 set(XNNPACK_SOURCE_DIR "${THIRD_PARTY_ROOT}/XNNPACK")
 set(XNNPACK_INCLUDE_DIR "${XNNPACK_SOURCE_DIR}/include")
-set(XNNPACK_LIBRARY_TYPE
-    "static"
-    CACHE STRING ""
+
+include(ExternalProject)
+include(GNUInstallDirs) # For CMAKE_INSTALL_LIBDIR
+set(XNNPACK_INSTALL_DIR "${CMAKE_CURRENT_BINARY_DIR}/XNNPACK/install")
+set(XNNPACK_STATIC_LIB
+    "${XNNPACK_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libXNNPACK.a"
 )
-set(XNNPACK_BUILD_BENCHMARKS
-    OFF
-    CACHE BOOL ""
+set(XNNPACK_MICROKERNELS_STATIC_LIB
+    "${XNNPACK_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libxnnpack-microkernels-prod.a"
 )
-set(XNNPACK_BUILD_TESTS
-    OFF
-    CACHE BOOL ""
+set(KLEIDIAI_STATIC_LIB
+    "${XNNPACK_INSTALL_DIR}/${CMAKE_INSTALL_LIBDIR}/libkleidiai.a"
+)
+get_extra_cmake_args_for_external_project(XNNPACK_EXTRA_CMAKE_ARGS)
+ExternalProject_Add(
+  XNNPACKExternalProject
+  SOURCE_DIR ${XNNPACK_SOURCE_DIR}
+  # Not 100% clear on these locations
+  BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/XNNPACK
+  INSTALL_DIR ${XNNPACK_INSTALL_DIR}
+  INSTALL_BYPRODUCTS ${XNNPACK_STATIC_LIB} ${XNNPACK_MICROKERNELS_STATIC_LIB}
+  ${KLEIDIAI_STATIC_LIB}
+  CMAKE_ARGS
+    ${XNNPACK_EXTRA_CMAKE_ARGS}
+    -D
+    XNNPACK_LIBRARY_TYPE=static
+    -D
+    XNNPACK_BUILD_BENCHMARKS=OFF
+    -D
+    XNNPACK_BUILD_TESTS=OFF
+    -D
+    XNNPACK_ENABLE_AVXVNNI=OFF
+    # Work around observed failure:
+    # https://github.com/pytorch/executorch/pull/10362#issuecomment-2906391232
+    -D
+    XNNPACK_ENABLE_AVX512VNNIGFNI=OFF
+    -D
+    ENABLE_XNNPACK_WEIGHTS_CACHE=${EXECUTORCH_XNNPACK_ENABLE_WEIGHT_CACHE}
+    -D
+    ENABLE_XNNPACK_SHARED_WORKSPACE=${EXECUTORCH_XNNPACK_SHARED_WORKSPACE}
+    -D
+    XNNPACK_ENABLE_KLEIDIAI=${EXECUTORCH_XNNPACK_ENABLE_KLEIDI}
+    -D
+    CMAKE_INSTALL_PREFIX=<INSTALL_DIR>
+    -D
+    XNNPACK_BUILD_ALL_MICROKERNELS=OFF
+    -D
+    CMAKE_POSITION_INDEPENDENT_CODE=ON
 )
-set(XNNPACK_ENABLE_AVXVNNI
-    OFF
-    CACHE BOOL ""
-  )
-# Work around observed failure: https://github.com/pytorch/executorch/pull/10362#issuecomment-2906391232
-set(XNNPACK_ENABLE_AVX512VNNIGFNI
-  OFF
-  CACHE BOOL "")
 
-if(EXECUTORCH_XNNPACK_ENABLE_KLEIDI)
-    set(XNNPACK_ENABLE_KLEIDIAI
-        ON
-        CACHE BOOL ""
-    )
-else()
-    set(XNNPACK_ENABLE_KLEIDIAI
-        OFF
-        CACHE BOOL ""
-    )
-endif()
+add_library(XNNPACK STATIC IMPORTED GLOBAL)
+# TODO: this probably doesn't work on Windows.
+set_property(TARGET XNNPACK PROPERTY IMPORTED_LOCATION ${XNNPACK_STATIC_LIB})
 
+add_dependencies(XNNPACK XNNPACKExternalProject)
 
-set(XNNPACK_BUILD_ALL_MICROKERNELS
-    OFF
-    CACHE BOOL ""
+add_library(xnnpack-microkernels-prod STATIC IMPORTED GLOBAL)
+set_property(
+  TARGET xnnpack-microkernels-prod PROPERTY IMPORTED_LOCATION
+                                            ${XNNPACK_MICROKERNELS_STATIC_LIB}
 )
-add_subdirectory("${XNNPACK_SOURCE_DIR}")
-include_directories(SYSTEM ${XNNPACK_INCLUDE_DIR})
-list(APPEND xnnpack_third_party XNNPACK)
-install(TARGETS xnnpack-microkernels-prod
-    LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-    PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+add_dependencies(xnnpack-microkernels-prod XNNPACKExternalProject)
 
+set_target_properties(
+  XNNPACK PROPERTIES INTERFACE_LINK_LIBRARIES xnnpack-microkernels-prod
+)
+
+install(DIRECTORY ${XNNPACK_INSTALL_DIR}/ DESTINATION ${CMAKE_INSTALL_PREFIX})
 
 if(EXECUTORCH_XNNPACK_ENABLE_KLEIDI)
-    if(TARGET kleidiai)
-        install(TARGETS kleidiai
-            LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
-            ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
-            PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
-    endif()
+  add_library(kleidiai SHARED IMPORTED)
+  install(FILES ${KLEIDIAI_STATIC_LIB} PUBLIC_HEADER
+          DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
+  )
+  set_property(
+    TARGET kleidiai PROPERTY IMPORTED_LOCATION ${KLEIDIAI_STATIC_LIB}
+  )
+  set_property(
+    TARGET XNNPACK
+    APPEND
+    PROPERTY INTERFACE_LINK_LIBRARIES kleidiai
+  )
 endif()
-
-# Revert PIC Flag to what it originally was
-set(CMAKE_POSITION_INDEPENDENT_CODE
-    ${ORIGINAL_CMAKE_POSITION_INDEPENDENT_CODE_FLAG}
-)
diff --git a/tools/cmake/Utils.cmake b/tools/cmake/Utils.cmake
@@ -278,3 +278,32 @@ macro(find_package_torch)
     find_package(Torch CONFIG REQUIRED)
   endif()
 endmacro()
+
+# In order to support cross compiling, we need to propagate a bunch of CMake
+# variables to ExternalProject. Call this to get the current values of all
+# relevant variables, which should then be passed to CMAKE_ARGS.
+function(get_extra_cmake_args_for_external_project outVar)
+  set(VARIABLES_TO_PROPAGATE
+      ANDROID_ABI
+      ANDROID_PLATFORM
+      CMAKE_ARCHIVE_OUTPUT_DIRECTORY
+      CMAKE_BUILD_TYPE
+      CMAKE_C_COMPILER_LAUNCHER
+      CMAKE_CXX_COMPILER_LAUNCHER
+      CMAKE_FIND_ROOT_PATH
+      CMAKE_OSX_DEPLOYMENT_TARGET
+      CMAKE_TOOLCHAIN_FILE
+      DEPLOYMENT_TARGET
+      PLATFORM
+  )
+  set(${outVar} "")
+  foreach(var ${VARIABLES_TO_PROPAGATE})
+    if(DEFINED ${var})
+      list(APPEND ${outVar} -D "${var}=${${var}}")
+    endif()
+  endforeach()
+  set(${outVar}
+      ${${outVar}}
+      PARENT_SCOPE
+  )
+endfunction()
diff --git a/tools/cmake/executorch-config.cmake b/tools/cmake/executorch-config.cmake
@@ -150,7 +150,9 @@ if(TARGET coremldelegate)
 endif()
 
 if(TARGET etdump)
-  set_target_properties(etdump PROPERTIES INTERFACE_LINK_LIBRARIES "flatccrt;executorch")
+  set_target_properties(
+    etdump PROPERTIES INTERFACE_LINK_LIBRARIES "flatccrt;executorch"
+  )
 endif()
 
 if(TARGET optimized_native_cpu_ops_lib)
@@ -174,13 +176,11 @@ if(TARGET extension_threadpool)
 endif()
 
 set(shared_lib_list
-  # executorch -- size tests fail due to regression if we include this and I'm not sure it's needed.
-  optimized_native_cpu_ops_lib
-  portable_ops_lib
-  quantized_ops_lib
-  xnnpack_backend
-  vulkan_backend
-  quantized_ops_aot_lib)
+    # executorch -- size tests fail due to regression if we include this and I'm
+    # not sure it's needed.
+    optimized_native_cpu_ops_lib portable_ops_lib quantized_ops_lib
+    xnnpack_backend vulkan_backend quantized_ops_aot_lib
+)
 foreach(lib ${shared_lib_list})
   if(TARGET ${lib})
     target_link_options_shared_lib(${lib})

Original file line number	Diff line number	Diff line change
`@@ -92,9 +92,8 @@ target_include_directories(`
`92`	`92`	`${EXECUTORCH_ROOT}/third-party/flatbuffers/include`
`93`	`93`	`)`
`94`	`94`
`95`		`-set(xnnpack_third_party pthreadpool extension_threadpool cpuinfo)`
`96`		`-`
`97`	`95`	`include(cmake/Dependencies.cmake)`
	`96`	`+set(xnnpack_third_party XNNPACK pthreadpool extension_threadpool cpuinfo)`
`98`	`97`
`99`	`98`	`list(TRANSFORM _xnnpack_backend__srcs PREPEND "${EXECUTORCH_ROOT}/")`
`100`	`99`	`add_library(xnnpack_backend ${_xnnpack_backend__srcs})`