Add CROSS_BOOTSTRAP_JULIA option for building the sysimg with another Julia (#59033)

Adds makefile variables called `CROSS_BOOTSTRAP_JULIA` and
`CROSS_BOOTSTRAP_SYSBASE` that control the Julia executable and `sysbase.so`
used to build `sys.so`. This is really useful for building a
ThreadSanitizer-enabled version of Julia in a reasonable amount of time.  It is
used like this:

```
make O=bootstrap julia-src-release julia-sysbase-release
make CROSS_BOOTSTRAP_JULIA=bootstrap/usr/bin/julia CROSS_BOOTSTRAP_SYSBASE=bootstrap/usr/lib/julia/sysbase.so julia-src-release julia-sysimg-release
```

This also adds a faster way to create a debug build of Julia (documented in the
devdocs).

Author: xal-0

Make.inc

@@ -798,6 +798,8 @@ ifneq ($(OS), Darwin)
 endif
 endif
 
+bootstrap_julia_flags :=
+
 ifeq ($(SANITIZE),1)
 SANITIZE_OPTS :=
 SANITIZE_LDFLAGS :=
@@ -815,6 +817,9 @@ endif
 ifeq ($(SANITIZE_THREAD),1)
 SANITIZE_OPTS += -fsanitize=thread
 SANITIZE_LDFLAGS += -fsanitize=thread
+ifneq ($(CROSS_BOOTSTRAP_JULIA),)
+bootstrap_julia_flags += --target-sanitize=thread
+endif
 endif
 ifeq ($(SANITIZE_OPTS),)
 $(error SANITIZE=1, but no sanitizer selected, set either SANITIZE_MEMORY, SANITIZE_THREAD, or SANITIZE_ADDRESS)
@@ -1761,9 +1766,15 @@ JULIA_BUILD_MODE := debug
 endif
 endif
 
+ifneq ($(CROSS_BOOTSTRAP_JULIA),)
+JULIA_EXECUTABLE_debug := $(CROSS_BOOTSTRAP_JULIA)
+JULIA_EXECUTABLE_release := $(CROSS_BOOTSTRAP_JULIA)
+JULIA_EXECUTABLE := $(CROSS_BOOTSTRAP_JULIA)
+else
 JULIA_EXECUTABLE_debug := $(build_bindir)/julia-debug$(EXE)
 JULIA_EXECUTABLE_release := $(build_bindir)/julia$(EXE)
 JULIA_EXECUTABLE := $(JULIA_EXECUTABLE_$(JULIA_BUILD_MODE))
+endif
 
 JULIA_SYSIMG_debug := $(build_private_libdir)/sys-debug.$(SHLIB_EXT)
 JULIA_SYSIMG_release := $(build_private_libdir)/sys.$(SHLIB_EXT)

Makefile

@@ -113,6 +113,10 @@ julia-cli-release julia-cli-debug: julia-cli-% : julia-deps
 julia-sysimg-release julia-sysimg-debug : julia-sysimg-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir)
 	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysimg-$*
 
+# Useful for cross-bootstrapping
+julia-sysbase-release julia-sysbase-debug : julia-sysbase-% : julia-src-% $(TOP_LEVEL_PKG_LINK_TARGETS) julia-stdlib julia-base julia-cli-% | $(build_private_libdir)
+	@$(MAKE) $(QUIET_MAKE) -C $(BUILDROOT) -f sysimage.mk sysbase-$*
+
 julia-debug julia-release : julia-% : julia-sysimg-% julia-src-% julia-symlink julia-libccalltest \
                                       julia-libccalllazyfoo julia-libccalllazybar julia-libllvmcalltest julia-base-cache
 

contrib/tsan/Make.user.tsan

@@ -10,3 +10,5 @@ USE_BINARYBUILDER_LLVM=1
 
 override SANITIZE=1
 override SANITIZE_THREAD=1
+override CROSS_BOOTSTRAP_JULIA=$(BUILDROOT)/../bootstrap/usr/bin/julia
+override CROSS_BOOTSTRAP_SYSBASE=$(BUILDROOT)/../bootstrap/usr/lib/julia/sysbase.$(SHLIB_EXT)

contrib/tsan/build.sh

@@ -3,7 +3,7 @@
 
 #
 # Usage:
-#     contrib/tsan/build.sh <path> [<make_targets>...]
+#     contrib/tsan/build.sh [-j<N>] <path> [<make_targets>...]
 #
 # Build TSAN-enabled julia.  Given a workspace directory <path>, build
 # TSAN-enabled julia in <path>/tsan.  Required toolss are install under
@@ -13,6 +13,16 @@
 # make target is `debug`.
 
 set -ue
+set -x
+
+JOBS=1
+while getopts j: opt
+do
+    case $opt in
+        j) JOBS="$OPTARG";;
+    esac
+done
+shift $((OPTIND-1))
 
 # `$WORKSPACE` is a directory in which we create `toolchain` and `tsan`
 # sub-directories.
@@ -44,6 +54,12 @@ fi
 
 make -C "$TOOLCHAIN/deps" install-clang install-llvm-tools
 
+echo
+echo "Building bootstrap Julia..."
+BUILD="$WORKSPACE/bootstrap"
+
+make -j "$JOBS" O="$BUILD" julia-src-release julia-sysbase-release
+
 echo
 echo "Building Julia..."
 
@@ -54,4 +70,6 @@ if [ ! -d "$BUILD" ]; then
 fi
 
 cd "$BUILD"  # so that we can pass `-C src` to `make`
-make "$@"
+# Reporting tsan warnings will interfere with bootstrapping.
+export TSAN_OPTIONS="report_bugs=0 exitcode=0"
+make -j "$JOBS" "$@"

doc/src/devdocs/build/build.md

@@ -304,6 +304,20 @@ LLVM_ASSERTIONS=1
 
 Please note that assert builds of Julia will be slower than regular (non-assert) builds.
 
+## Building a debug build of Julia
+
+A full debug build of Julia can be built with `make debug`.  This builds a debug
+version of `libjulia` and uses it to bootstrap the compiler, before creating a
+system image with debug symbols enabled.  This can take more than 15 minutes.
+
+Although it may result in some differences, a debug build can be built much
+quicker by bootstrapping from a release build:
+
+```sh
+$ make julia-src-release julia-sysbase-release
+$ make julia-sysimg-debug CROSS_BOOTSTRAP_JULIA=$PWD/usr/bin/julia CROSS_BOOTSTRAP_SYSBASE=$PWD/usr/lib/julia/sysbase.so
+```
+
 ## Building 32-bit Julia on a 64-bit machine
 
 Occasionally, bugs specific to 32-bit architectures may arise, and when this happens it is useful to be able to debug the problem on your local machine. Since most modern 64-bit systems support running programs built for 32-bit ones, if you don't have to recompile Julia from source (e.g. you mainly need to inspect the behavior of a 32-bit Julia without having to touch the C code), you can likely use a 32-bit build of Julia for your system that you can obtain from the [official downloads page](https://julialang.org/downloads/).

sysimage.mk

@@ -11,6 +11,8 @@ sysimg-ji: $(build_private_libdir)/sysbase.ji
 sysimg-bc: $(build_private_libdir)/sys-bc.a
 sysimg-release: $(build_private_libdir)/sys.$(SHLIB_EXT)
 sysimg-debug: $(build_private_libdir)/sys-debug.$(SHLIB_EXT)
+sysbase-release: $(build_private_libdir)/sysbase.$(SHLIB_EXT)
+sysbase-debug: $(build_private_libdir)/sysbase-debug.$(SHLIB_EXT)
 
 VERSDIR := v$(shell cut -d. -f1-2 < $(JULIAHOME)/VERSION)
 
@@ -124,7 +126,8 @@ $$(build_private_libdir)/sysbase$1-o.a $$(build_private_libdir)/sysbase$1-bc.a :
 		false; \
 	fi )
 	@mv $$@.tmp $$@
-$$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(build_private_libdir)/sys$1-%.a : $$(build_private_libdir)/sysbase$1.$$(SHLIB_EXT) $$(JULIAHOME)/contrib/generate_precompile.jl
+build_sysbase_$1 := $$(or $$(CROSS_BOOTSTRAP_SYSBASE),$$(build_private_libdir)/sysbase$1.$$(SHLIB_EXT))
+$$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(build_private_libdir)/sys$1-%.a : $$(build_sysbase_$1) $$(JULIAHOME)/contrib/generate_precompile.jl
 	@$$(call PRINT_JULIA, cd $$(JULIAHOME)/base && \
 	if ! JULIA_BINDIR=$$(call cygpath_w,$(build_bindir)) \
 		 WINEPATH="$$(call cygpath_w,$$(build_bindir));$$$$WINEPATH" \
@@ -133,6 +136,7 @@ $$(build_private_libdir)/sys$1-o.a $$(build_private_libdir)/sys$1-bc.a : $$(buil
 		 JULIA_DEPOT_PATH=':' \
 		 JULIA_NUM_THREADS=1 \
 			$$(call spawn, $3) $2 -C "$$(JULIA_CPU_TARGET)" $$(HEAPLIM) --output-$$* $$(call cygpath_w,$$@).tmp $$(JULIA_SYSIMG_BUILD_FLAGS) \
+			$(bootstrap_julia_flags) \
 			--startup-file=no --warn-overwrite=yes --depwarn=error --sysimage $$(call cygpath_w,$$<) $$(call cygpath_w,$$(JULIAHOME)/contrib/generate_precompile.jl) $(JULIA_PRECOMPILE); then \
 		echo '*** This error is usually fixed by running `make clean`. If the error persists$$(COMMA) try `make cleanall`. ***'; \
 		false; \