From 20eacbc2c40731cb4c2c250c493eeb9a1e204a01 Mon Sep 17 00:00:00 2001 From: Geoffrey Thomas Date: Sun, 10 Aug 2025 20:09:59 -0400 Subject: [PATCH 1/2] Give _tkinter $ORIGIN-relative dependencies on glibc and an rpath on musl Partially addresses #742 and makes it consistent with what we're doing on macOS. There's an argument in the comments above that we should not set an rpath on libpython (except on musl where it's needed), and I need to see if I still believe that. In the meantime I'm following that pattern and setting $ORIGIN-relative NEEDED on glibc and rpath on musl only. This also adds a specific ldd regression test but not the additional tests listed in #742. --- cpython-unix/build-cpython.sh | 19 ++++++ src/validation.rs | 125 ++++++++++++++++++++++++++++------ 2 files changed, 125 insertions(+), 19 deletions(-) diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index 658395a9..ba9e62a6 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -821,6 +821,25 @@ if [ "${PYBUILD_SHARED}" = "1" ]; then ${ROOT}/out/python/install/lib/libpython3.so fi fi + + # PyInstaller would like to see `ldd` work on modules. + # https://github.com/pyinstaller/pyinstaller/issues/9204#issuecomment-3171583553 + # Also this probably helps programs linking libpython avoid having to set an rpath. + patchelf_args=() + if [ "${CC}" == "musl-clang" ]; then + patchelf_args+=(--set-rpath '${ORIGIN}/../..') + else + for lib in ${ROOT}/out/python/install/lib/*; do + basename=${lib##*/} + patchelf_args+=(--replace-needed "$basename" '${ORIGIN}/../../'"$basename") + done + fi + # At the moment, python3 and libpython don't have shared-library + # dependencies, but at some point we will want to run this for + # them too. + for module in ${ROOT}/out/python/install/lib/python*/lib-dynload/*.so; do + patchelf "${patchelf_args[@]}" "$module" + done fi fi diff --git a/src/validation.rs b/src/validation.rs index 3b1ef5c8..5578fbd2 100644 --- a/src/validation.rs +++ b/src/validation.rs @@ -265,24 +265,32 @@ static ELF_ALLOWED_LIBRARIES_BY_TRIPLE: Lazy>> = - Lazy::new(|| { - [ - ( - // libcrypt is provided by the system, but only on older distros. - "_crypt", - vec!["libcrypt.so.1"], - ), - ( - // libtcl and libtk are shipped in our distribution. - "_tkinter", - vec!["libtcl8.6.so", "libtk8.6.so"], - ), - ] - .iter() - .cloned() - .collect() - }); +#[derive(Copy, Clone, PartialEq)] +enum DepSource { + SystemRequired, + SystemOptional, + Vendored, +} +use DepSource::*; + +static ELF_ALLOWED_LIBRARIES_BY_MODULE: Lazy< + HashMap<&'static str, Vec<(&'static str, DepSource)>>, +> = Lazy::new(|| { + [ + ( + // libcrypt is provided by the system, but only on older distros. + "_crypt", + vec![("libcrypt.so.1", SystemOptional)], + ), + ( + "_tkinter", + vec![("libtcl8.6.so", Vendored), ("libtk8.6.so", Vendored)], + ), + ] + .iter() + .cloned() + .collect() +}); static DARWIN_ALLOWED_DYLIBS: Lazy> = Lazy::new(|| { [ @@ -1022,7 +1030,7 @@ fn validate_elf>( if let Some(filename) = path.file_name() { if let Some((module, _)) = filename.to_string_lossy().split_once(".cpython-") { if let Some(extra) = ELF_ALLOWED_LIBRARIES_BY_MODULE.get(module) { - allowed_libraries.extend(extra.iter().map(|x| x.to_string())); + allowed_libraries.extend(extra.iter().map(|x| x.0.to_string())); } } } @@ -2186,6 +2194,85 @@ fn verify_distribution_behavior(dist_path: &Path) -> Result> { errors.push("errors running interpreter tests".to_string()); } + // Explicitly test ldd directly on the extension modules, which PyInstaller + // relies on. This is not strictly needed for a working distribution (e.g. + // you can set an rpath on just python+libpython), so we test here for + // compatibility with tools that run ldd. + // that fails this check (e.g. by setting an rpath on just python+libpython). + // https://github.com/pyinstaller/pyinstaller/issues/9204#issuecomment-3171050891 + // TODO(geofft): musl doesn't do lazy binding for the argument to + // ldd, so we will get complaints about missing Py_* symbols. Need + // to handle this somehow, skip testing for now. + if cfg!(target_os = "linux") && !python_json.target_triple.contains("-musl") { + // musl's ldd is packaged in the "musl-tools" Debian package. + let ldd = if python_json.target_triple.contains("-musl") && cfg!(not(target_env = "musl")) { + "musl-ldd" + } else { + "ldd" + }; + for (name, variants) in python_json.build_info.extensions.iter() { + for ext in variants { + let Some(shared_lib) = &ext.shared_lib else { + continue; + }; + let shared_lib_path = temp_dir.path().join("python").join(shared_lib); + let output = duct::cmd(ldd, [shared_lib_path]) + .unchecked() + .stdout_capture() + .run() + .context(format!("Failed to run `{ldd} {shared_lib}`"))?; + let stdout = String::from_utf8_lossy(&output.stdout); + // Format of ldd output, for both glibc and musl: + // - Everything starts with a tab. + // - Most things are "libxyz.so.1 => /usr/lib/libxyz.so.1 (0xabcde000)". + // - The ELF interpreter is displayed as just "/lib/ld.so (0xabcde000)". + // - glibc, but not musl, shows the vDSO as "linux-vdso.so.1 (0xfffff000)". + // - If a library is listed in DT_NEEDED with an absolute path, or (currently only + // supported on glibc) with an $ORIGIN-relative path, it displays as just + // "/path/to/libxyz.so (0xabcde000)". + // - On glibc, if a library cannot be found ldd returns zero and shows "=> not + // found" as the resolution (even if it wouldn't use the => form if found). + // - On musl, if a library cannot be found, ldd returns nonzero and shows "Error + // loading shared library ...:" on stderr. + if !output.status.success() { + // TODO: If we ever have any optional dependencies besides libcrypt (which is + // glibc-only), we will need to capture musl ldd's stderr and parse it. + errors.push(format!( + "`{ldd} {shared_lib}` exited with {}:\n{stdout}", + output.status + )); + } else { + let mut ldd_errors = vec![]; + let deps = ELF_ALLOWED_LIBRARIES_BY_MODULE.get(&name[..]); + let temp_dir_lossy = temp_dir.path().to_string_lossy().into_owned(); + for line in stdout.lines() { + let Some((needed, resolution)) = line.trim().split_once(" => ") else { + continue; + }; + let dep_source = deps + .and_then(|deps| { + deps.iter().find(|dep| dep.0 == needed).map(|dep| dep.1) + }) + .unwrap_or(SystemRequired); + if resolution.starts_with("not found") && dep_source != SystemOptional { + ldd_errors.push(format!("{needed} was expected to be found")); + } else if !resolution.contains(&temp_dir_lossy) && dep_source == Vendored { + ldd_errors.push(format!( + "{needed} should not come from the OS (missing rpath/$ORIGIN?)" + )); + } + } + if !ldd_errors.is_empty() { + errors.push(format!( + "In `{ldd} {shared_lib}`:\n - {}\n{stdout}", + ldd_errors.join("\n - ") + )); + } + } + } + } + } + Ok(errors) } From 51a59c1b57f045ed6ebfef3452fb008e23d3fdf1 Mon Sep 17 00:00:00 2001 From: Geoffrey Thomas Date: Mon, 11 Aug 2025 14:19:07 -0400 Subject: [PATCH 2/2] Move copying of libtcl.so and libtk.so before patchelf This block was just added in #676 and isn't actually Tcl/Tk-specific. --- cpython-unix/build-cpython.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index ba9e62a6..f79aff69 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -683,6 +683,14 @@ fi # This ensures we can run the binary in any location without # LD_LIBRARY_PATH pointing to the directory containing libpython. if [ "${PYBUILD_SHARED}" = "1" ]; then + ( + shopt -s nullglob + dylibs=(${TOOLS_PATH}/deps/lib/lib*.dylib ${TOOLS_PATH}/deps/lib/lib*.so) + if [ "${#dylibs[@]}" -gt 0 ]; then + cp -av "${dylibs[@]}" ${ROOT}/out/python/install/lib/ + fi + ) + if [[ "${PYBUILD_PLATFORM}" = macos* ]]; then # There's only 1 dylib produced on macOS and it has the binary suffix. LIBPYTHON_SHARED_LIBRARY_BASENAME=libpython${PYTHON_MAJMIN_VERSION}${PYTHON_BINARY_SUFFIX}.dylib @@ -1266,14 +1274,6 @@ if [ -d "${TOOLS_PATH}/deps/lib/tcl8" ]; then for source in ${TOOLS_PATH}/deps/lib/{itcl4.2.4,tcl8,tcl8.6,thread2.8.9,tk8.6}; do cp -av $source ${ROOT}/out/python/install/lib/ done - - ( - shopt -s nullglob - dylibs=(${TOOLS_PATH}/deps/lib/lib*.dylib ${TOOLS_PATH}/deps/lib/lib*.so) - if [ "${#dylibs[@]}" -gt 0 ]; then - cp -av "${dylibs[@]}" ${ROOT}/out/python/install/lib/ - fi - ) fi # Copy the terminfo database if present.