Add relnotes-api-list in-tree tool

Cargo.lock

@@ -2188,6 +2188,7 @@ dependencies = [
 name = "linkchecker"
 version = "0.1.0"
 dependencies = [
+ "clap",
  "html5ever",
  "regex",
 ]
@@ -3196,6 +3197,17 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c"
 
+[[package]]
+name = "relnotes-api-list"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "rustdoc-json-types",
+ "serde",
+ "serde_json",
+ "tempfile",
+]
+
 [[package]]
 name = "remote-test-client"
 version = "0.1.0"

Cargo.toml

@@ -30,6 +30,7 @@ members = [
   "src/tools/miri/cargo-miri",
   "src/tools/miropt-test-tools",
   "src/tools/opt-dist",
+  "src/tools/relnotes-api-list",
   "src/tools/remote-test-client",
   "src/tools/remote-test-server",
   "src/tools/replace-version-placeholder",

src/bootstrap/src/core/build_steps/check.rs

@@ -616,3 +616,9 @@ tool_check_step!(CoverageDump {
     mode: Mode::ToolBootstrap,
     default: false
 });
+
+tool_check_step!(Linkchecker {
+    path: "src/tools/linkchecker",
+    mode: Mode::ToolBootstrap,
+    default: false
+});

src/bootstrap/src/core/build_steps/dist.rs

@@ -2636,3 +2636,62 @@
         tarball.generate()
     }
 }
+
+#[derive(Clone, Debug, Eq, PartialEq, Hash)]
+pub struct RelnotesApiList {
+    pub host: TargetSelection,
+}
+
+impl Step for RelnotesApiList {
+    type Output = ();
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        let default = run.builder.config.docs;
+        run.alias("relnotes-api-list").default_condition(default)
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(RelnotesApiList { host: run.target });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Self::Output {
+        let host = self.host;
+        let dest = builder.out.join("dist").join(format!("relnotes-api-list-{host}.json"));
+        builder.create_dir(dest.parent().unwrap());
+
+        // The HTML documentation for the standard library is needed to check all links generated by
+        // the tool are not broken.
+        builder.ensure(crate::core::build_steps::doc::Std::new(
+            builder.top_stage,
+            host,
+            DocumentationFormat::Html,
+        ));
+
+        if std::env::var_os("EMILY_SKIP_DOC").is_none() {
+            // TODO: remove the condition
+            builder.ensure(
+                crate::core::build_steps::doc::Std::new(
+                    builder.top_stage,
+                    host,
+                    DocumentationFormat::Json,
+                )
+                // Crates containing symbols exported by any std crate:
+                .add_extra_crate("rustc-literal-escaper")
+                .add_extra_crate("std_detect"),
+            );
+        }
+
+        let linkchecker = builder.tool_exe(Tool::Linkchecker);
+
+        builder.info("Generating the API list for the release notes");
+        builder
+            .tool_cmd(Tool::RelnotesApiList)
+            .arg(builder.json_doc_out(host))
+            .arg(&dest)
+            .env("LINKCHECKER_PATH", linkchecker)
+            .env("STD_HTML_DOCS", builder.doc_out(self.host))
+            .run(builder);
+        builder.info(&format!("API list for the release notes available at {}", dest.display()));
+    }
+}

src/bootstrap/src/core/build_steps/doc.rs

@@ -560,11 +560,17 @@ pub struct Std {
     pub target: TargetSelection,
     pub format: DocumentationFormat,
     crates: Vec<String>,
+    extra_crates: Vec<String>,
 }
 
 impl Std {
     pub(crate) fn new(stage: u32, target: TargetSelection, format: DocumentationFormat) -> Self {
-        Std { stage, target, format, crates: vec![] }
+        Std { stage, target, format, crates: vec![], extra_crates: vec![] }
+    }
+
+    pub(crate) fn add_extra_crate(mut self, krate: &str) -> Self {
+        self.extra_crates.push(krate.to_string());
+        self
     }
 }
 
@@ -592,6 +598,7 @@ impl Step for Std {
                 DocumentationFormat::Html
             },
             crates,
+            extra_crates: vec![],
         });
     }
 
@@ -602,7 +609,7 @@ impl Step for Std {
     fn run(self, builder: &Builder<'_>) {
         let stage = self.stage;
         let target = self.target;
-        let crates = if self.crates.is_empty() {
+        let mut crates = if self.crates.is_empty() {
             builder
                 .in_tree_crates("sysroot", Some(target))
                 .iter()
@@ -611,6 +618,7 @@ impl Step for Std {
         } else {
             self.crates
         };
+        crates.extend(self.extra_crates.iter().cloned());
 
         let out = match self.format {
             DocumentationFormat::Html => builder.doc_out(target),

src/bootstrap/src/core/build_steps/tool.rs

@@ -526,6 +526,7 @@ bootstrap_tool!(
     FeaturesStatusDump, "src/tools/features-status-dump", "features-status-dump";
     OptimizedDist, "src/tools/opt-dist", "opt-dist", submodules = &["src/tools/rustc-perf"];
     RunMakeSupport, "src/tools/run-make-support", "run_make_support", artifact_kind = ToolArtifactKind::Library;
+    RelnotesApiList, "src/tools/relnotes-api-list", "relnotes-api-list";
 );
 
 /// These are the submodules that are required for rustbook to work due to

src/bootstrap/src/core/builder/mod.rs

@@ -987,6 +987,7 @@ impl<'a> Builder<'a> {
                 tool::CoverageDump,
                 tool::LlvmBitcodeLinker,
                 tool::RustcPerf,
+                tool::RelnotesApiList,
             ),
             Kind::Clippy => describe!(
                 clippy::Std,
@@ -1033,6 +1034,7 @@ impl<'a> Builder<'a> {
                 check::Compiletest,
                 check::FeaturesStatusDump,
                 check::CoverageDump,
+                check::Linkchecker,
                 // This has special staging logic, it may run on stage 1 while others run on stage 0.
                 // It takes quite some time to build stage 1, so put this at the end.
                 //
@@ -1163,7 +1165,8 @@ impl<'a> Builder<'a> {
                 dist::PlainSourceTarball,
                 dist::BuildManifest,
                 dist::ReproducibleArtifacts,
-                dist::Gcc
+                dist::Gcc,
+                dist::RelnotesApiList,
             ),
             Kind::Install => describe!(
                 install::Docs,

src/tools/linkchecker/Cargo.toml

@@ -1,7 +1,7 @@
 [package]
 name = "linkchecker"
 version = "0.1.0"
-edition = "2021"
+edition = "2024"
 
 [[bin]]
 name = "linkchecker"
@@ -10,3 +10,4 @@ path = "main.rs"
 [dependencies]
 regex = "1"
 html5ever = "0.29.0"
+clap = { version = "4.5.40", features = ["derive"] }

src/tools/linkchecker/main.rs

@@ -17,13 +17,16 @@
 //! should catch the majority of "broken link" cases.
 
 use std::cell::{Cell, RefCell};
+use std::collections::hash_map::Entry;
 use std::collections::{HashMap, HashSet};
+use std::fs;
 use std::io::ErrorKind;
+use std::iter::once;
 use std::path::{Component, Path, PathBuf};
 use std::rc::Rc;
 use std::time::Instant;
-use std::{env, fs};
 
+use clap::Parser;
 use html5ever::tendril::ByteTendril;
 use html5ever::tokenizer::{
     BufferQueue, TagToken, Token, TokenSink, TokenSinkResult, Tokenizer, TokenizerOpts,
@@ -110,10 +113,22 @@ macro_rules! t {
     };
 }
 
+#[derive(Parser)]
+struct Cli {
+    docs: PathBuf,
+    #[clap(long)]
+    link_targets_dir: Vec<PathBuf>,
+}
+
 fn main() {
-    let docs = env::args_os().nth(1).expect("doc path should be first argument");
-    let docs = env::current_dir().unwrap().join(docs);
-    let mut checker = Checker { root: docs.clone(), cache: HashMap::new() };
+    let mut cli = Cli::parse();
+    cli.docs = cli.docs.canonicalize().unwrap();
+
+    let mut checker = Checker {
+        root: cli.docs.clone(),
+        link_targets_dirs: cli.link_targets_dir,
+        cache: HashMap::new(),
+    };
     let mut report = Report {
         errors: 0,
         start: Instant::now(),
@@ -125,7 +140,7 @@ fn main() {
         intra_doc_exceptions: 0,
         has_broken_urls: false,
     };
-    checker.walk(&docs, &mut report);
+    checker.walk(&cli.docs, &mut report);
     report.report();
     if report.errors != 0 {
         println!("found some broken links");
@@ -135,6 +150,7 @@ fn main() {
 
 struct Checker {
     root: PathBuf,
+    link_targets_dirs: Vec<PathBuf>,
     cache: Cache,
 }
 
@@ -427,15 +443,23 @@ impl Checker {
         let pretty_path =
             file.strip_prefix(&self.root).unwrap_or(file).to_str().unwrap().to_string();
 
-        let entry =
-            self.cache.entry(pretty_path.clone()).or_insert_with(|| match fs::metadata(file) {
+        for base in once(&self.root).chain(self.link_targets_dirs.iter()) {
+            let entry = self.cache.entry(pretty_path.clone());
+            if let Entry::Occupied(e) = &entry
+                && !matches!(e.get(), FileEntry::Missing)
+            {
+                break;
+            }
+
+            let file = base.join(&pretty_path);
+            entry.insert_entry(match fs::metadata(&file) {
                 Ok(metadata) if metadata.is_dir() => FileEntry::Dir,
                 Ok(_) => {
                     if file.extension().and_then(|s| s.to_str()) != Some("html") {
                         FileEntry::OtherFile
                     } else {
                         report.html_files += 1;
-                        load_html_file(file, report)
+                        load_html_file(&file, report)
                     }
                 }
                 Err(e) if e.kind() == ErrorKind::NotFound => FileEntry::Missing,
@@ -451,6 +475,9 @@ impl Checker {
                     panic!("unexpected read error for {}: {}", file.display(), e);
                 }
             });
+        }
+
+        let entry = self.cache.get(&pretty_path).unwrap();
         (pretty_path, entry)
     }
 }

src/tools/relnotes-api-list/Cargo.toml

@@ -0,0 +1,11 @@
+[package]
+name = "relnotes-api-list"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+anyhow = "1"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+rustdoc-json-types = { path = "../../rustdoc-json-types" }
+tempfile = "3.20.0"

src/tools/relnotes-api-list/README.md

@@ -0,0 +1,17 @@
+# API list generator for the release notes
+
+Rust's [release notes] include a "Stabilized APIs" section for each
+release, listing all APIs that became stable each release. This tool supports
+the creation of that section by generating a JSON file containing a concise
+representation of the standard library API. The [relnotes tool] will then
+compare the JSON files of two releases to generate the section.
+
+The tool is executed by CI and produces the `relnotes-api-list-$target.json`
+dist artifact. You can also run the tool locally with:
+
+```
+./x dist relnotes-api-list
+```
+
+[release notes]: https://doc.rust-lang.org/stable/releases.html
+[relnotes tool]: https://github.com/rust-lang/relnotes

src/tools/relnotes-api-list/src/check_urls.rs

@@ -0,0 +1,65 @@
+//! We generate a lot of URLs in the resulting JSON, and we need all those URLs to be correct. While
+//! some URLs are fairly trivial to generate, others are quite tricky (especially `impl` blocks).
+//!
+//! To ensure we always generate good URLs, we prepare a temporary HTML file containing `<a>` tags for
+//! every itme we collected, and we run it through linkchecker. If this fails, it means the code
+//! generating URLs has a bug.
+
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::Command;
+
+use anyhow::{Error, anyhow, bail};
+use tempfile::tempdir;
+
+use crate::schema::{Schema, SchemaItem};
+
+pub(crate) fn check_urls(schema: &Schema) -> Result<(), Error> {
+    let mut urls = Vec::new();
+    collect_urls(&mut urls, &schema.items);
+
+    let html_dir = tempdir()?;
+
+    let mut file = File::create(html_dir.path().join("urls.html"))?;
+    file.write_all(render_html(&urls).as_bytes())?;
+
+    eprintln!("checking that all generated URLs are valid...");
+    let result = Command::new(require_env("LINKCHECKER_PATH")?)
+        .arg(html_dir.path())
+        .arg("--link-targets-dir")
+        .arg(require_env("STD_HTML_DOCS")?)
+        .status()?;
+
+    if !result.success() {
+        bail!("some URLs are broken, the relnotes-api-list tool has a bug");
+    }
+
+    dbg!(require_env("STD_HTML_DOCS")?);
+
+    Ok(())
+}
+
+fn collect_urls<'a>(result: &mut Vec<&'a str>, items: &'a [SchemaItem]) {
+    for item in items {
+        if let Some(url) = &item.url {
+            result.push(url);
+        }
+        collect_urls(result, &item.children);
+    }
+}
+
+fn render_html(urls: &[&str]) -> String {
+    let mut content = "<!DOCTYPE html>\n".to_string();
+    for url in urls {
+        content.push_str(&format!("<a href=\"{url}\"></a>\n"));
+    }
+    content
+}
+
+fn require_env(name: &str) -> Result<PathBuf, Error> {
+    match std::env::var_os(name) {
+        Some(value) => Ok(value.into()),
+        None => Err(anyhow!("missing environment variable {name}")),
+    }
+}