Skip to content

Commit 4ed2de1

Browse files
authored
Support older libtool versions (#115)
Fixes #114. * toolchain: update the tool names * toolchain: add machinery to expose information about host tools to `cc_toolchain_config` `cc_toolchain_config` is not a repository rule and cannot poke at the host tools itself * toolchain: add logic to detect if a tool supports arg files * toolchain: generalize the `cc_wrapper` plumbing to support "wrapper files" * toolchain: move the `host_tools` utils to a struct * toolchain: add a libtool wrapper that flattens arg files * toolchain: use the libtool wrapper when needed this commit also changes `cc_toolchain_config.bzl` to check that `strip` and `ld` exist on the host machine when we use them we use tags here instead of the `/usr/bin/` paths so that we can easily transition to using `rctx.which` to find where the tools live instead if we want to do so in the future * misc: fmt
1 parent 03f380c commit 4ed2de1

File tree

11 files changed

+215
-32
lines changed

11 files changed

+215
-32
lines changed

WORKSPACE

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -152,8 +152,8 @@ http_archive(
152152
load("@rules_rust//rust:repositories.bzl", "rust_repositories")
153153

154154
rust_repositories(
155-
version = "1.55.0",
156155
edition = "2018",
156+
version = "1.55.0",
157157
)
158158

159159
# We're using `git2` as our Rust test because it links against C code
@@ -166,15 +166,11 @@ rust_repositories(
166166
# anyways (as of this writing nothing in `@rules_rust//tests` seems to test
167167
# this).
168168
GIT2_RS_VER = "0.13.22"
169+
169170
GIT2_RS_SHA = "9c1cbbfc9a1996c6af82c2b4caf828d2c653af4fcdbb0e5674cc966eee5a4197"
170171

171172
http_archive(
172173
name = "git2",
173-
sha256 = GIT2_RS_SHA,
174-
canonical_id = GIT2_RS_VER,
175-
url = "https://crates.io/api/v1/crates/git2/{ver}/download".format(ver = GIT2_RS_VER),
176-
type = "tar.gz",
177-
strip_prefix = "git2-{ver}".format(ver = GIT2_RS_VER),
178174
build_file_content = """
179175
package(default_visibility = ["//visibility:public"])
180176
@@ -207,16 +203,22 @@ rust_test(
207203
for t in glob(["tests/*.rs"])
208204
]
209205
""",
206+
canonical_id = GIT2_RS_VER,
207+
patch_args = ["-p1"],
210208
# We need to remove some `[target]` entries in `git2`'s `Cargo.toml` to
211209
# make `crate-universe` happy.
212210
#
213211
# See: https://github.com/bazelbuild/rules_rust/issues/783
214212
patches = ["//tests/rust:git2-rs-cargo-toml.patch"],
215-
patch_args = ["-p1"]
213+
sha256 = GIT2_RS_SHA,
214+
strip_prefix = "git2-{ver}".format(ver = GIT2_RS_VER),
215+
type = "tar.gz",
216+
url = "https://crates.io/api/v1/crates/git2/{ver}/download".format(ver = GIT2_RS_VER),
216217
)
217218

218219
# Snippets for `crate_universe`:
219220
RULES_RUST_CRATE_UNIVERSE_URL_TEMPLATE = "https://github.com/bazelbuild/rules_rust/releases/download/crate_universe-13/crate_universe_resolver-{host_triple}{extension}"
221+
220222
RULES_RUST_CRATE_UNIVERSE_SHA256_CHECKSUMS = {
221223
"aarch64-apple-darwin": "c6017cd8a4fee0f1796a8db184e9d64445dd340b7f48a65130d7ee61b97051b4",
222224
"aarch64-unknown-linux-gnu": "d0a310b03b8147e234e44f6a93e8478c260a7c330e5b35515336e7dd67150f35",

tests/BUILD.bazel

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
# limitations under the License.
1414

1515
load("@bazel_skylib//rules:build_test.bzl", "build_test")
16-
load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test", "cc_binary")
16+
load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test")
1717
load(":transitions.bzl", "dwp_file")
1818

1919
cc_library(
@@ -44,7 +44,7 @@ build_test(
4444
name = "stripped_binary_test",
4545
targets = [
4646
":stdlib_bin.stripped",
47-
]
47+
],
4848
)
4949

5050
# We want to test that `llvm-dwp` (used when assembling a `.dwp` file from

tests/transitions.bzl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ dwp_file = rule(
7878
values = ["dbg", "fastbuild", "opt"],
7979
),
8080
"_allowlist_function_transition": attr.label(
81-
default = "@bazel_tools//tools/allowlists/function_transition_allowlist"
81+
default = "@bazel_tools//tools/allowlists/function_transition_allowlist",
8282
),
8383
},
8484
incompatible_use_toolchain_transition = True,

toolchain/BUILD.toolchain.tpl

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,11 @@ filegroup(
2626
)
2727

2828
filegroup(
29-
name = "cc-wrapper",
30-
srcs = ["bin/cc_wrapper.sh"],
29+
name = "wrapper-files",
30+
srcs = [
31+
"bin/cc_wrapper.sh",
32+
"bin/host_libtool_wrapper.sh",
33+
],
3134
)
3235

3336
%{cc_toolchains}

toolchain/cc_toolchain_config.bzl

Lines changed: 25 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ load(
1919
load(
2020
"//toolchain/internal:common.bzl",
2121
_check_os_arch_keys = "check_os_arch_keys",
22+
_host_tool_features = "host_tool_features",
23+
_host_tools = "host_tools",
2224
_os_arch_pair = "os_arch_pair",
2325
)
2426

@@ -32,10 +34,11 @@ def cc_toolchain_config(
3234
target_os,
3335
toolchain_path_prefix,
3436
tools_path_prefix,
35-
cc_wrapper_prefix,
37+
wrapper_bin_prefix,
3638
sysroot_path,
3739
additional_include_dirs,
38-
llvm_version):
40+
llvm_version,
41+
host_tools_info = {}):
3942
host_os_arch_key = _os_arch_pair(host_os, host_arch)
4043
target_os_arch_key = _os_arch_pair(target_os, target_arch)
4144
_check_os_arch_keys([host_os_arch_key, target_os_arch_key])
@@ -233,21 +236,35 @@ def cc_toolchain_config(
233236
# `llvm-strip` was introduced in V7 (https://reviews.llvm.org/D46407):
234237
llvm_version = llvm_version.split(".")
235238
llvm_major_ver = int(llvm_version[0]) if len(llvm_version) else 0
236-
strip_binary = (tools_path_prefix + "bin/llvm-strip") if llvm_major_ver >= 7 else "/usr/bin/strip"
239+
strip_binary = (tools_path_prefix + "bin/llvm-strip") if llvm_major_ver >= 7 else _host_tools.get_and_assert(host_tools_info, "strip")
237240

241+
# TODO: The command line formed on darwin does not work with llvm-ar.
242+
ar_binary = tools_path_prefix + "bin/llvm-ar"
243+
if host_os == "darwin":
244+
# Bazel uses arg files for longer commands; some old macOS `libtool`
245+
# versions do not support this.
246+
#
247+
# In these cases we want to use `libtool_wrapper.sh` which translates
248+
# the arg file back into command line arguments.
249+
if not _host_tools.tool_supports(host_tools_info, "libtool", features = [_host_tool_features.SUPPORTS_ARG_FILE]):
250+
ar_binary = wrapper_bin_prefix + "bin/host_libtool_wrapper.sh"
251+
else:
252+
ar_binary = host_tools_info["libtool"]["path"]
253+
254+
# The tool names come from [here](https://github.com/bazelbuild/bazel/blob/c7e58e6ce0a78fdaff2d716b4864a5ace8917626/src/main/java/com/google/devtools/build/lib/rules/cpp/CppConfiguration.java#L76-L90):
238255
tool_paths = {
239-
# TODO: The command line formed on darwin does not work with llvm-ar.
240-
"ar": tools_path_prefix + "bin/llvm-ar" if host_os != "darwin" else "/usr/bin/libtool",
256+
"ar": ar_binary,
241257
"cpp": tools_path_prefix + "bin/clang-cpp",
242-
"dwp": tools_path_prefix + "bin/llvm-dwp",
243-
"gcc": cc_wrapper_prefix + "bin/cc_wrapper.sh",
258+
"gcc": wrapper_bin_prefix + "bin/cc_wrapper.sh",
244259
"gcov": tools_path_prefix + "bin/llvm-profdata",
245-
"ld": tools_path_prefix + "bin/ld.lld" if use_lld else "/usr/bin/ld",
260+
"ld": tools_path_prefix + "bin/ld.lld" if use_lld else _host_tools.get_and_assert(host_tools_info, "ld"),
246261
"llvm-cov": tools_path_prefix + "bin/llvm-cov",
247262
"nm": tools_path_prefix + "bin/llvm-nm",
248263
"objcopy": tools_path_prefix + "bin/llvm-objcopy",
249264
"objdump": tools_path_prefix + "bin/llvm-objdump",
250265
"strip": strip_binary,
266+
"dwp": tools_path_prefix + "bin/llvm-dwp",
267+
"llvm-profdata": tools_path_prefix + "bin/llvm-profdata",
251268
}
252269

253270
# Start-end group linker support:

toolchain/cc_wrapper.sh.tpl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@
2828
set -eu
2929

3030
# See note in toolchain/internal/configure.bzl where we define
31-
# `cc_wrapper_prefix` for why this wrapper is needed.
31+
# `wrapper_bin_prefix` for why this wrapper is needed.
3232

3333
# Call the C++ compiler.
3434
if [[ -f %{toolchain_path_prefix}bin/clang ]]; then
Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
#!/usr/bin/env bash
2+
3+
# Some older `libtool` versions (~macOS 10.12) don't support arg files.
4+
#
5+
# This script flattens arg files into regular command line arguments.
6+
7+
args=()
8+
for a in "${@}"; do
9+
if [[ ${a} =~ @.* ]]; then
10+
IFS=$'\n' read -d '' -r -a args_in_file < "${a:1}"
11+
for arg in "${args_in_file[@]}"; do
12+
args+=("${arg}")
13+
done
14+
else
15+
args+=("${a}")
16+
fi
17+
done
18+
19+
exec "%{libtool_path}" "${args[@]}"

toolchain/internal/common.bzl

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414

1515
SUPPORTED_TARGETS = [("linux", "x86_64"), ("linux", "aarch64"), ("darwin", "x86_64")]
1616

17+
host_tool_features = struct(
18+
SUPPORTS_ARG_FILE = "supports_arg_file",
19+
)
20+
1721
def python(rctx):
1822
# Get path of the python interpreter.
1923

@@ -53,6 +57,107 @@ def arch(rctx):
5357
fail("Failed to detect machine architecture: \n%s\n%s" % (exec_result.stdout, exec_result.stderr))
5458
return exec_result.stdout.strip()
5559

60+
# Tries to figure out if a tool supports newline separated arg files (i.e.
61+
# `@file`).
62+
def _tool_supports_arg_file(rctx, tool_path):
63+
# We assume nothing other than that `tool_path` is an executable.
64+
#
65+
# First we have to find out what command line flag gets the tool to just
66+
# print out some text and exit successfully.
67+
#
68+
# Most tools support `-v` or `--version` or (for `libtool`) `-V` but some
69+
# tools don't have such an option (BSD `ranlib` and `ar`, for example).
70+
#
71+
# We just try all the options we know of until one works and if none work
72+
# we return "None" indicating an indeterminate result.
73+
opts = (
74+
["-v", "--version", "-version", "-V"] +
75+
["-h", "--help", "-help", "-H"]
76+
)
77+
78+
no_op_opt = None
79+
for opt in opts:
80+
if rctx.execute([tool_path, opt], timeout = 2).return_code == 0:
81+
no_op_opt = opt
82+
break
83+
84+
if no_op_opt == None:
85+
return None
86+
87+
# Okay! Once we have an opt that we *know* does nothing but make the
88+
# executable exit successfully, we'll stick that opt in a file and try
89+
# again:
90+
tmp_file = "tmp-arg-file"
91+
rctx.file(tmp_file, content = "{}\n".format(no_op_opt), executable = False)
92+
93+
res = rctx.execute([tool_path, "@{}".format(tmp_file)]).return_code == 0
94+
rctx.delete(tmp_file)
95+
96+
return res
97+
98+
def _get_host_tool_info(rctx, tool_path, features_to_test = [], tool_key = None):
99+
if tool_key == None:
100+
tool_key = tool_path
101+
102+
if tool_path == None or not rctx.path(tool_path).exists:
103+
return {}
104+
105+
f = host_tool_features
106+
features = {}
107+
for feature in features_to_test:
108+
features[feature] = {
109+
f.SUPPORTS_ARG_FILE: _tool_supports_arg_file,
110+
}[feature](rctx, tool_path)
111+
112+
return {
113+
tool_key: struct(
114+
path = tool_path,
115+
features = features,
116+
),
117+
}
118+
119+
def _extract_tool_path_and_features(tool_info):
120+
# Have to support structs or dicts:
121+
tool_path = tool_info.path if type(tool_info) == "struct" else tool_info["path"]
122+
tool_features = tool_info.features if type(tool_info) == "struct" else tool_info["features"]
123+
124+
return (tool_path, tool_features)
125+
126+
def _check_host_tool_supports(host_tool_info, tool_key, features = []):
127+
if tool_key in host_tool_info:
128+
_, tool_features = _extract_tool_path_and_features(host_tool_info[tool_key])
129+
130+
for f in features:
131+
if not f in tool_features or not tool_features[f]:
132+
return False
133+
134+
return True
135+
else:
136+
return False
137+
138+
def _get_host_tool_and_assert_supports(host_tool_info, tool_key, features = []):
139+
if tool_key in host_tool_info:
140+
tool_path, tool_features = _extract_tool_path_and_features(host_tool_info[tool_key])
141+
142+
missing = [f for f in features if not f in tool_features or not tool_features[f]]
143+
144+
if missing:
145+
fail("Host tool `{key}` (`{path}`) is missing these features: `{missing}`.".format(
146+
key = tool_key,
147+
path = tool_path,
148+
missing = missing,
149+
))
150+
151+
return tool_path
152+
else:
153+
return False
154+
155+
host_tools = struct(
156+
get_tool_info = _get_host_tool_info,
157+
tool_supports = _check_host_tool_supports,
158+
get_and_assert = _get_host_tool_and_assert_supports,
159+
)
160+
56161
def os_arch_pair(os, arch):
57162
return "{}-{}".format(os, arch)
58163

0 commit comments

Comments
 (0)