From 00084c8ea13187847aacae1a9ffcf11dd73a3c3e Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 24 Aug 2025 14:18:56 +0200 Subject: [PATCH 1/2] ci: build with clang instead of gcc --- README.rst | 2 +- pyproject.toml | 8 +- scripts/install-static-clang.sh | 101 ++++++++++++++++++ .../manylinux-build-and-install-openssl.sh | 6 ++ 4 files changed, 113 insertions(+), 4 deletions(-) create mode 100755 scripts/install-static-clang.sh diff --git a/README.rst b/README.rst index 06aa3987..ea0d3f57 100644 --- a/README.rst +++ b/README.rst @@ -68,7 +68,7 @@ The following platforms are supported with binary wheels: | Linux IBM Z | | manylinux2014+ s390x | | | | musllinux_1_2+ s390x | +---------------+---------------------------+ - | Linux RISC-V | | manylinux_2_35+ riscv64 | + | Linux RISC-V | | manylinux_2_31+ riscv64 | | | | musllinux_1_2+ riscv64 | +---------------+---------------------------+ | macOS 10.10+ | Intel | diff --git a/pyproject.toml b/pyproject.toml index 49329c0b..b9bb5510 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -105,7 +105,7 @@ manylinux-aarch64-image = "manylinux2014" manylinux-ppc64le-image = "manylinux2014" manylinux-s390x-image = "manylinux2014" manylinux-armv7l-image = "manylinux_2_31" -manylinux-riscv64-image = "ghcr.io/mayeut/manylinux_2_35:2025.08.02-2" +manylinux-riscv64-image = "ghcr.io/mayeut/manylinux_2_31:2025.08.23-1" musllinux-x86_64-image = "musllinux_1_2" musllinux-i686-image = "musllinux_1_2" musllinux-aarch64-image = "musllinux_1_2" @@ -121,9 +121,11 @@ environment = { MACOSX_DEPLOYMENT_TARGET = "10.10" } [[tool.cibuildwheel.overrides]] select = "*-*linux*" -before-all = "./scripts/manylinux-build-and-install-openssl.sh" +before-all = "./scripts/install-static-clang.sh && ./scripts/manylinux-build-and-install-openssl.sh" inherit.environment = "prepend" -environment = { PKG_CONFIG_PATH = "/usr/local/ssl/lib/pkgconfig" } +environment = { CC = "/opt/clang/bin/clang", CXX = "/opt/clang/bin/clang++", LDFLAGS = "-fuse-ld=lld -Wl,--build-id", PKG_CONFIG_PATH = "/usr/local/ssl/lib/pkgconfig" } +inherit.environment-pass = "prepend" +environment-pass = ["RUNNER_ARCH"] inherit.config-settings = "prepend" config-settings."cmake.define.OPENSSL_ROOT_DIR" = "/usr/local/ssl" config-settings."cmake.define.OPENSSL_USE_STATIC_LIBS" = "ON" diff --git a/scripts/install-static-clang.sh b/scripts/install-static-clang.sh new file mode 100755 index 00000000..e253fd44 --- /dev/null +++ b/scripts/install-static-clang.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +# Stop at any error, show all commands +set -exuo pipefail + +TOOLCHAIN_PATH=/opt/clang + +# Download static-clang +DEFAULT_ARCH="$(uname -m)" +if [ "${STATIC_CLANG_ARCH:-}" == "" ]; then + STATIC_CLANG_ARCH="${RUNNER_ARCH:-${DEFAULT_ARCH}}" +fi +case "${STATIC_CLANG_ARCH}" in + ARM64|aarch64|arm64|arm64/*) GO_ARCH=arm64;; + ARM|armv7l|armv8l|arm|arm/v7) GO_ARCH=arm;; # assume arm/v7 for arm + X64|x86_64|amd64|amd64/*) GO_ARCH=amd64;; + X86|i686|386) GO_ARCH=386;; + ppc64le) GO_ARCH=ppc64le;; + riscv64) GO_ARCH=riscv64;; + s390x) GO_ARCH=s390x;; + *) echo "No static-clang toolchain for ${CLANG_ARCH}">2; exit 1;; +esac +STATIC_CLANG_VERSION=20.1.8.0 +STATIC_CLANG_FILENAME="static-clang-linux-${GO_ARCH}.tar.xz" +STATIC_CLANG_URL="https://github.com/mayeut/static-clang-images/releases/download/v${STATIC_CLANG_VERSION}/${STATIC_CLANG_FILENAME}" +pushd /tmp +cat<<'EOF' | grep "${STATIC_CLANG_FILENAME}" > "${STATIC_CLANG_FILENAME}.sha256" +5ef070163055340d5bbf11263e60396b4f221c7dab90e167424e1b9b68a6048e static-clang-linux-386.tar.xz +3a25cb4eae724cad96d7e504ce7669a824f61d57d27716fd47d0a9f1fe2d8fdf static-clang-linux-amd64.tar.xz +250e5cc18fb7b6f7a4a0d8fe63ddf8a885f7351c4be7e4a164f4c04dfbee5a7f static-clang-linux-arm.tar.xz +da56c2b13a0c3e676e010d61a87753a975de18ae9eaa65247ca6a6d22bb95ab4 static-clang-linux-arm64.tar.xz +466af6ca1be0cd4f2ae6704ce5ae0f86a1648e999756def1680b639bc63d2916 static-clang-linux-ppc64le.tar.xz +8078365b22f1d5290db7e60501daa80c91da8d530720d3fc974ab3f81e56bae5 static-clang-linux-riscv64.tar.xz +97e933c45a35c827530888e1c2f6b6eee0140ccb2089fdfdf9f99454d4c470e0 static-clang-linux-s390x.tar.xz +EOF +curl -fsSLO "${STATIC_CLANG_URL}" +sha256sum -c "${STATIC_CLANG_FILENAME}.sha256" +tar -C /opt -xf "${STATIC_CLANG_FILENAME}" +popd + +# configure target triple +case "${AUDITWHEEL_POLICY}-${AUDITWHEEL_ARCH}" in + manylinux*-armv7l) TARGET_TRIPLE=armv7-unknown-linux-gnueabihf;; + musllinux*-armv7l) TARGET_TRIPLE=armv7-alpine-linux-musleabihf;; + manylinux*-ppc64le) TARGET_TRIPLE=powerpc64le-unknown-linux-gnu;; + musllinux*-ppc64le) TARGET_TRIPLE=powerpc64le-alpine-linux-musl;; + manylinux*-*) TARGET_TRIPLE=${AUDITWHEEL_ARCH}-unknown-linux-gnu;; + musllinux*-*) TARGET_TRIPLE=${AUDITWHEEL_ARCH}-alpine-linux-musl;; +esac +case "${AUDITWHEEL_ARCH}" in + riscv64) M_ARCH="-march=rv64gc";; + x86_64) M_ARCH="-march=x86-64";; + armv7l) M_ARCH="-march=armv7a";; + i686) M_ARCH="-march=i686";; +esac +GCC_TRIPLE=$(gcc -dumpmachine) + +if [ "${AUDITWHEEL_ARCH}" == "riscv64" ]; then + # the LDFLAGS from pyproject.toml seems not to be taken into account when building CMake (no problem with OpenSSL) + # FAILED: [code=1] Source/kwsys/cmsysTestProcess + # : && /opt/clang/bin/clang -D_POSIX_C_SOURCE=199506L -D_POSIX_SOURCE=1 -D_SVID_SOURCE=1 -D_BSD_SOURCE=1 -O3 -DNDEBUG -lstdc++ -lgcc -lrt Source/kwsys/CMakeFiles/cmsysTestProcess.dir/testProcess.c.o -o Source/kwsys/cmsysTestProcess Source/kwsys/libcmsys_c.a && : + # /usr/bin/riscv64-linux-gnu-ld: -march=rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0_zifencei2p0_zmmul1p0_zaamo1p0_zalrsc1p0: unsupported ISA subset `z' + # /usr/bin/riscv64-linux-gnu-ld: failed to merge target specific data of file Source/kwsys/libcmsys_c.a(ProcessUNIX.c.o) + # /usr/bin/riscv64-linux-gnu-ld: -march=rv64i2p1_m2p0_a2p1_f2p2_d2p2_c2p0_zicsr2p0_zifencei2p0_zmmul1p0_zaamo1p0_zalrsc1p0: unsupported ISA subset `z' + # /usr/bin/riscv64-linux-gnu-ld: failed to merge target specific data of file Source/kwsys/libcmsys_c.a(System.c.o) + # clang: error: linker command failed with exit code 1 (use -v to see invocation) + IMPLICIT_LLD="-fuse-ld=lld" +fi + +cat<"${TOOLCHAIN_PATH}/bin/${AUDITWHEEL_PLAT}.cfg" + -target ${TARGET_TRIPLE} + ${M_ARCH:-} + --gcc-toolchain=${DEVTOOLSET_ROOTPATH:-}/usr + --gcc-triple=${GCC_TRIPLE} + ${IMPLICIT_LLD:-} +EOF + +cat<"${TOOLCHAIN_PATH}/bin/clang.cfg" + @${AUDITWHEEL_PLAT}.cfg +EOF + +cat<"${TOOLCHAIN_PATH}/bin/clang++.cfg" + @${AUDITWHEEL_PLAT}.cfg +EOF + +cat<"${TOOLCHAIN_PATH}/bin/clang-cpp.cfg" + @${AUDITWHEEL_PLAT}.cfg +EOF + +# override entrypoint to add the toolchain to PATH +mv /usr/local/bin/manylinux-entrypoint /usr/local/bin/manylinux-entrypoint-org +cat</usr/local/bin/manylinux-entrypoint +#!/bin/bash + +set -eu + +export PATH="${TOOLCHAIN_PATH}/bin:\${PATH}" +exec /usr/local/bin/manylinux-entrypoint-org "\$@" +EOF + +chmod +x /usr/local/bin/manylinux-entrypoint diff --git a/scripts/manylinux-build-and-install-openssl.sh b/scripts/manylinux-build-and-install-openssl.sh index 081703cf..f5dd13f8 100755 --- a/scripts/manylinux-build-and-install-openssl.sh +++ b/scripts/manylinux-build-and-install-openssl.sh @@ -46,6 +46,12 @@ check_sha256sum ${OPENSSL_ROOT}.tar.gz ${OPENSSL_HASH} tar -xzf ${OPENSSL_ROOT}.tar.gz rm -rf ${OPENSSL_ROOT}.tar.gz +LIBATOMIC= +if [ "${AUDITWHEEL_ARCH}" == "i686" ]; then + LIBATOMIC=-latomic +fi +export LDLIBS="${LIBATOMIC}" + # Configure pushd ${OPENSSL_ROOT} ./config no-shared no-tests -fPIC --prefix=/usr/local/ssl --openssldir=/usr/local/ssl > /dev/null From b2cbc1a883a4fc2dcd38d61df355cd234cc2f02f Mon Sep 17 00:00:00 2001 From: mayeut Date: Sun, 24 Aug 2025 14:28:05 +0200 Subject: [PATCH 2/2] ci: remove QEMU conditions --- .github/workflows/build.yml | 32 +------------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 2f3051d1..bcf1965f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,11 +2,6 @@ name: CI on: workflow_dispatch: - inputs: - use_qemu: - description: 'Use qemu to build linux ppc64le, s390x & riscv64' - required: true - default: true schedule: - cron: '0 18 * * 5' # "At 18:00 on Friday." pull_request: @@ -21,9 +16,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -env: - USE_QEMU: ${{ fromJSON(github.event.inputs.use_qemu || 'false') || (github.event_name == 'schedule') || startsWith(github.ref, 'refs/tags/') }} - jobs: lint: name: Lint @@ -47,100 +39,78 @@ jobs: - os: ubuntu-latest arch: "x86_64" build: "manylinux_" - use_qemu: false - os: ubuntu-latest arch: "x86_64" build: "musllinux_" - use_qemu: false - os: ubuntu-latest arch: "i686" build: "manylinux_" - use_qemu: false - os: ubuntu-latest arch: "i686" build: "musllinux_" - use_qemu: false - os: ubuntu-24.04-arm arch: "aarch64" build: "manylinux_" - use_qemu: false - os: ubuntu-24.04-arm arch: "aarch64" build: "musllinux_" - use_qemu: false - os: ubuntu-latest arch: "ppc64le" build: "manylinux_" - use_qemu: true - os: ubuntu-latest arch: "ppc64le" build: "musllinux_" - use_qemu: true - os: ubuntu-latest arch: "s390x" build: "manylinux_" - use_qemu: true - os: ubuntu-latest arch: "s390x" build: "musllinux_" - use_qemu: true - os: ubuntu-latest arch: "riscv64" build: "manylinux_" - use_qemu: true - os: ubuntu-latest arch: "riscv64" build: "musllinux_" - use_qemu: true - os: ubuntu-24.04-arm arch: "armv7l" build: "manylinux_" - use_qemu: false - os: ubuntu-24.04-arm arch: "armv7l" build: "musllinux_" - use_qemu: false - os: windows-2022 arch: "AMD64" build: "" - use_qemu: false - os: windows-11-arm arch: "ARM64" build: "" - use_qemu: false - os: windows-2022 arch: "x86" build: "" - use_qemu: false - os: macos-14 arch: "universal2" build: "" - use_qemu: false steps: - uses: actions/checkout@v5 - if: (!matrix.use_qemu) || fromJSON(env.USE_QEMU) with: fetch-depth: 0 # required for versioneer to find tags - uses: astral-sh/setup-uv@v6 - if: (!matrix.use_qemu) || fromJSON(env.USE_QEMU) with: enable-cache: false - name: Set up QEMU + if: matrix.arch == 'ppc64le' || matrix.arch == 'riscv64' || matrix.arch == 's390x' uses: docker/setup-qemu-action@v3.6.0 - if: matrix.use_qemu && fromJSON(env.USE_QEMU) - name: Build wheels uses: pypa/cibuildwheel@v3.1 - if: (!matrix.use_qemu) || fromJSON(env.USE_QEMU) env: CIBW_ARCHS: "${{ matrix.arch }}" CIBW_BUILD: "cp310-${{ matrix.build }}*" - uses: actions/upload-artifact@v4 - if: (!matrix.use_qemu) || fromJSON(env.USE_QEMU) with: name: cibw-${{ runner.os }}-${{ matrix.build }}${{ matrix.arch }} path: ./wheelhouse/*.whl