add flag to split sourcemap into a secondary rmeta file

Author: yaahc

compiler/rustc_metadata/src/lib.rs

@@ -33,6 +33,8 @@ pub use native_libs::{
     NativeLibSearchFallback, find_native_static_library, try_find_native_dynamic_library,
     try_find_native_static_library, walk_native_lib_search_dirs,
 };
-pub use rmeta::{EncodedMetadata, METADATA_HEADER, encode_metadata, rendered_const};
+pub use rmeta::{
+    EncodedMetadata, METADATA_EXTRA_HEADER, METADATA_HEADER, encode_metadata, rendered_const,
+};
 
 rustc_fluent_macro::fluent_messages! { "../messages.ftl" }

compiler/rustc_metadata/src/rmeta/encoder.rs

@@ -39,6 +39,7 @@ use crate::rmeta::*;
 
 pub(super) struct EncodeContext<'a, 'tcx> {
     opaque: opaque::FileEncoder,
+    opaque_extra: Option<opaque::FileEncoder>,
     tcx: TyCtxt<'tcx>,
     feat: &'tcx rustc_feature::Features,
     tables: TableBuilders,
@@ -600,7 +601,18 @@ impl<'a, 'tcx> EncodeContext<'a, 'tcx> {
             adapted.set_some(on_disk_index, self.lazy(adapted_source_file));
         }
 
-        adapted.encode(&mut self.opaque)
+        if let Some(opaque_extra) = &mut self.opaque_extra {
+            let table = adapted.encode(opaque_extra);
+            // leave a hole full of zeros in the original location in rmeta to avoid messing with offsets of subsequent items
+            for _ in 0..table.size() {
+                const WIDTH: usize =
+                    std::mem::size_of::<Option<LazyValue<rustc_span::SourceFile>>>();
+                self.opaque.emit_raw_bytes(&[0; WIDTH]);
+            }
+            table
+        } else {
+            adapted.encode(&mut self.opaque)
+        }
     }
 
     fn encode_crate_root(&mut self) -> LazyValue<CrateRoot> {
@@ -2432,6 +2444,16 @@ fn with_encode_metadata_header(
     // Will be filled with the root position after encoding everything.
     encoder.emit_raw_bytes(&0u64.to_le_bytes());
 
+    let encoder_extra = if tcx.sess.is_split_rmeta_enabled() {
+        let path = path.with_extension("rmeta-extra");
+        let mut encoder = opaque::FileEncoder::new(path)
+            .unwrap_or_else(|err| tcx.dcx().emit_fatal(FailCreateFileEncoder { err }));
+        encoder.emit_raw_bytes(METADATA_EXTRA_HEADER);
+        Some(encoder)
+    } else {
+        None
+    };
+
     let source_map_files = tcx.sess.source_map().files();
     let source_file_cache = (Arc::clone(&source_map_files[0]), 0);
     let required_source_files = Some(FxIndexSet::default());
@@ -2441,6 +2463,7 @@ fn with_encode_metadata_header(
 
     let mut ecx = EncodeContext {
         opaque: encoder,
+        opaque_extra: encoder_extra,
         tcx,
         feat: tcx.features(),
         tables: Default::default(),
@@ -2468,6 +2491,12 @@ fn with_encode_metadata_header(
         tcx.dcx().emit_fatal(FailWriteFile { path: &path, err });
     }
 
+    if let Some(opaque_extra) = &mut ecx.opaque_extra {
+        if let Err((path, err)) = opaque_extra.finish() {
+            tcx.dcx().emit_fatal(FailWriteFile { path: &path, err });
+        }
+    }
+
     let file = ecx.opaque.file();
     if let Err(err) = encode_root_position(file, root_position) {
         tcx.dcx().emit_fatal(FailWriteFile { path: ecx.opaque.path(), err });

compiler/rustc_metadata/src/rmeta/mod.rs

@@ -65,6 +65,11 @@ const METADATA_VERSION: u8 = 10;
 /// unsigned integer, and further followed by the rustc version string.
 pub const METADATA_HEADER: &[u8] = &[b'r', b'u', b's', b't', 0, 0, 0, METADATA_VERSION];
 
+/// Metadata extras header which includes `METADATA_VERSION`.
+///
+/// This header is followed by the length of the compressed data
+pub const METADATA_EXTRA_HEADER: &[u8] = &[b'R', b'U', b'S', b'T', 0, 0, 0, METADATA_VERSION];
+
 /// A value of type T referred to by its absolute position
 /// in the metadata, and which can be decoded lazily.
 ///

compiler/rustc_session/src/options.rs

@@ -2537,6 +2537,8 @@ written to standard error output)"),
                   by the linker"),
     split_lto_unit: Option<bool> = (None, parse_opt_bool, [TRACKED],
         "enable LTO unit splitting (default: no)"),
+    split_rmeta: Option<bool> = (None, parse_opt_bool, [TRACKED],
+        "split extra rmeta data that isn't relevant rlib ABI into a separate file"),
     src_hash_algorithm: Option<SourceFileHashAlgorithm> = (None, parse_src_file_hash, [TRACKED],
         "hash algorithm of source files in debug info (`md5`, `sha1`, or `sha256`)"),
     #[rustc_lint_opt_deny_field_access("use `Session::stack_protector` instead of this field")]

compiler/rustc_session/src/session.rs

@@ -400,6 +400,10 @@ impl Session {
         self.opts.unstable_opts.split_lto_unit == Some(true)
     }
 
+    pub fn is_split_rmeta_enabled(&self) -> bool {
+        self.opts.unstable_opts.split_rmeta == Some(true)
+    }
+
     /// Check whether this compile session and crate type use static crt.
     pub fn crt_static(&self, crate_type: Option<CrateType>) -> bool {
         if !self.target.crt_static_respected {

tests/run-make/rdr-ignores-comments/rmake.rs

@@ -1,4 +1,4 @@
-// tests that changing the content of a comment will cause a change in the rmeta of a file
+// tests that changing the content of a comment will not cause a change in the rmeta of a library if -Zsplit-rmeta is enabled
 use run_make_support::{rfs, rustc};
 use std::hash::{Hash, Hasher};
 use std::path::Path;
@@ -7,15 +7,15 @@ fn main() {
     let before = check_and_hash("before.rs");
     let after = check_and_hash("after.rs");
     dbg!(before, after);
-    assert_neq!(before, after);
+    assert_eq!(before, after);
 }
 
 fn check_and_hash<P>(filename: P) -> u64
 where
     P: AsRef<Path>,
 {
     rfs::rename(filename, "foo.rs");
-    rustc().input("foo.rs").emit("metadata").run();
+    rustc().input("foo.rs").emit("metadata").arg("-Zsplit-rmeta").run();
     // hash the output
     let bytes = rfs::read("libfoo.rmeta");
     let mut hasher = std::hash::DefaultHasher::new();