Turbopack: make stats.json useable

Author: mischnic

crates/next-api/src/app.rs

@@ -1368,8 +1368,12 @@ impl AppEndpoint {
                 .should_create_webpack_stats()
                 .await?
             {
-                let webpack_stats =
-                    generate_webpack_stats(app_entry.original_name.clone(), &client_assets).await?;
+                let webpack_stats = generate_webpack_stats(
+                    *module_graphs.base,
+                    app_entry.original_name.clone(),
+                    client_assets.iter().copied(),
+                )
+                .await?;
                 let stats_output = VirtualOutputAsset::new(
                     node_root.join(&format!(
                         "server/app{manifest_path_prefix}/webpack-stats.json",

crates/next-api/src/pages.rs

@@ -1392,8 +1392,12 @@ impl PageEndpoint {
             .should_create_webpack_stats()
             .await?
         {
-            let webpack_stats =
-                generate_webpack_stats(original_name.to_owned(), &client_assets.await?).await?;
+            let webpack_stats = generate_webpack_stats(
+                self.client_module_graph(),
+                original_name.to_owned(),
+                client_assets.await?.iter().copied(),
+            )
+            .await?;
             let stats_output = VirtualOutputAsset::new(
                 node_root.join(&format!(
                     "server/pages{manifest_path_prefix}/webpack-stats.json",

crates/next-api/src/webpack_stats.rs

@@ -1,79 +1,178 @@
 use anyhow::Result;
+use rustc_hash::FxHashSet;
 use serde::Serialize;
+use tracing::{Level, instrument};
 use turbo_rcstr::RcStr;
-use turbo_tasks::{FxIndexMap, FxIndexSet, ResolvedVc, Vc};
+use turbo_tasks::{
+    FxIndexMap, FxIndexSet, ResolvedVc, TryJoinIterExt, ValueToString, Vc, fxindexmap,
+};
 use turbopack_browser::ecmascript::EcmascriptBrowserChunk;
 use turbopack_core::{
-    chunk::{Chunk, ChunkItem},
+    chunk::{Chunk, ChunkItem, ChunkItemExt, ModuleId},
+    module::Module,
+    module_graph::ModuleGraph,
     output::OutputAsset,
 };
 
-pub async fn generate_webpack_stats<'a, I>(
+#[instrument(level = Level::INFO, skip_all)]
+pub async fn generate_webpack_stats<I>(
+    module_graph: Vc<ModuleGraph>,
     entry_name: RcStr,
     entry_assets: I,
 ) -> Result<WebpackStats>
 where
-    I: IntoIterator<Item = &'a ResolvedVc<Box<dyn OutputAsset>>>,
+    I: IntoIterator<Item = ResolvedVc<Box<dyn OutputAsset>>>,
 {
     let mut assets = vec![];
     let mut chunks = vec![];
     let mut chunk_items: FxIndexMap<Vc<Box<dyn ChunkItem>>, FxIndexSet<RcStr>> =
         FxIndexMap::default();
-    let mut modules = vec![];
+
+    let entry_assets = entry_assets.into_iter().collect::<Vec<_>>();
+
+    let (asset_parents, asset_children) = {
+        let mut asset_children =
+            FxIndexMap::with_capacity_and_hasher(entry_assets.len(), Default::default());
+        let mut visited =
+            FxHashSet::with_capacity_and_hasher(entry_assets.len(), Default::default());
+        let mut queue = entry_assets.clone();
+        while let Some(asset) = queue.pop() {
+            if visited.insert(asset) {
+                let references = asset.references().await?;
+                asset_children.insert(asset, references.clone());
+                queue.extend(references);
+            }
+        }
+
+        let mut asset_parents: FxIndexMap<_, Vec<_>> =
+            FxIndexMap::with_capacity_and_hasher(entry_assets.len(), Default::default());
+        for (&parent, children) in &asset_children {
+            for child in children {
+                asset_parents.entry(*child).or_default().push(parent);
+            }
+        }
+
+        (asset_parents, asset_children)
+    };
+
+    let asset_reasons = {
+        let module_graph = module_graph.await?;
+        let mut edges = vec![];
+        module_graph
+            .traverse_all_edges_unordered(|(parent_node, r), current| {
+                edges.push((
+                    parent_node.module,
+                    RcStr::from(format!("{}: {}", r.chunking_type, r.export)),
+                    current.module,
+                ));
+                Ok(())
+            })
+            .await?;
+
+        let edges = edges
+            .into_iter()
+            .map(async |(parent, ty, child)| {
+                let parent_path = parent.ident().path().await?.path.clone();
+                Ok((
+                    child,
+                    WebpackStatsReason {
+                        module: parent_path.clone(),
+                        module_identifier: parent.ident().to_string().owned().await?,
+                        module_name: parent_path,
+                        ty,
+                    },
+                ))
+            })
+            .try_join()
+            .await?;
+
+        let mut asset_reasons: FxIndexMap<_, Vec<_>> = FxIndexMap::default();
+        for (child, reason) in edges {
+            asset_reasons.entry(child).or_default().push(reason);
+        }
+        asset_reasons
+    };
+
     for asset in entry_assets {
-        let path = normalize_client_path(&asset.path().await?.path);
+        let path = RcStr::from(normalize_client_path(&asset.path().await?.path));
 
         let Some(asset_len) = *asset.size_bytes().await? else {
             continue;
         };
 
-        if let Some(chunk) = ResolvedVc::try_downcast_type::<EcmascriptBrowserChunk>(*asset) {
-            let chunk_ident = normalize_client_path(&chunk.path().await?.path);
+        if let Some(chunk) = ResolvedVc::try_downcast_type::<EcmascriptBrowserChunk>(asset) {
             chunks.push(WebpackStatsChunk {
                 size: asset_len,
-                files: vec![chunk_ident.clone().into()],
-                id: chunk_ident.clone().into(),
+                files: vec![path.clone()],
+                id: path.clone(),
+                parents: if let Some(parents) = asset_parents.get(&asset) {
+                    parents
+                        .iter()
+                        .map(async |c| Ok(normalize_client_path(&c.path().await?.path).into()))
+                        .try_join()
+                        .await?
+                } else {
+                    vec![]
+                },
+                children: if let Some(children) = asset_children.get(&asset) {
+                    children
+                        .iter()
+                        .map(async |c| Ok(normalize_client_path(&c.path().await?.path).into()))
+                        .try_join()
+                        .await?
+                } else {
+                    vec![]
+                },
                 ..Default::default()
             });
 
             for item in chunk.chunk().chunk_items().await? {
-                // let name =
-                chunk_items
-                    .entry(**item)
-                    .or_default()
-                    .insert(chunk_ident.clone().into());
+                chunk_items.entry(**item).or_default().insert(path.clone());
             }
         }
 
         assets.push(WebpackStatsAsset {
             ty: "asset".into(),
-            name: path.clone().into(),
-            chunks: vec![path.into()],
+            name: path.clone(),
+            chunk_names: vec![path],
             size: asset_len,
             ..Default::default()
         });
     }
 
-    for (chunk_item, chunks) in chunk_items {
-        let size = *chunk_item
-            .content_ident()
-            .path()
-            .await?
-            .read()
-            .len()
-            .await?;
-        let path = chunk_item.asset_ident().path().await?.path.clone();
-        modules.push(WebpackStatsModule {
-            name: path.clone(),
-            id: path.clone(),
-            chunks: chunks.into_iter().collect(),
-            size,
-        });
-    }
+    // TODO map the chunk items back to the module
+
+    // TODO try downcast modules to `EcmascriptMergedModule` to get the socpe hoisted modules as
+    // well
+
+    let modules = chunk_items
+        .into_iter()
+        .map(async |(chunk_item, chunks)| {
+            let size = *chunk_item
+                .content_ident()
+                .path()
+                .await?
+                .read()
+                .len()
+                .await?;
+            Ok(WebpackStatsModule {
+                name: chunk_item.asset_ident().path().await?.path.clone(),
+                id: chunk_item.id().owned().await?,
+                identifier: chunk_item.asset_ident().to_string().owned().await?,
+                chunks: chunks.into_iter().collect(),
+                size,
+                // TODO Find all incoming edges to this module
+                reasons: asset_reasons
+                    .get(&chunk_item.module().to_resolved().await?)
+                    .cloned()
+                    .unwrap_or_default(),
+            })
+        })
+        .try_join()
+        .await?;
 
-    let mut entrypoints = FxIndexMap::default();
-    entrypoints.insert(
-        entry_name.clone(),
+    let entrypoints: FxIndexMap<_, _> = fxindexmap!(
+        entry_name.clone() =>
         WebpackStatsEntrypoint {
             name: entry_name.clone(),
             chunks: chunks.iter().map(|c| c.id.clone()).collect(),
@@ -83,7 +182,7 @@ where
                     name: a.name.clone(),
                 })
                 .collect(),
-        },
+        }
     );
 
     Ok(WebpackStats {
@@ -108,35 +207,81 @@ pub struct WebpackStatsAssetInfo {}
 pub struct WebpackStatsAsset {
     #[serde(rename = "type")]
     pub ty: RcStr,
+    /// The `output` filename
     pub name: RcStr,
     pub info: WebpackStatsAssetInfo,
+    /// The size of the file in bytes
     pub size: u64,
+    /// Indicates whether or not the asset made it to the `output` directory
     pub emitted: bool,
+    /// Indicates whether or not the asset was compared with the same file on the output file
+    /// system
     pub compared_for_emit: bool,
     pub cached: bool,
+    /// The chunks this asset contains
+    pub chunk_names: Vec<RcStr>,
+    /// The chunk IDs this asset contains
     pub chunks: Vec<RcStr>,
 }
 
 #[derive(Serialize, Debug, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct WebpackStatsChunk {
+    /// Indicates whether or not the chunk went through Code Generation
     pub rendered: bool,
+    /// Indicates whether this chunk is loaded on initial page load or [on
+    /// demand](/guides/lazy-loading)
     pub initial: bool,
+    /// Indicates whether or not the chunk contains the webpack runtime
     pub entry: bool,
     pub recorded: bool,
+    /// The ID of this chunk
     pub id: RcStr,
+    /// Chunk size in bytes
     pub size: u64,
     pub hash: RcStr,
+    /// An array of filename strings that contain this chunk
     pub files: Vec<RcStr>,
+    /// An list of chunk names contained within this chunk
+    pub names: Vec<RcStr>,
+    /// Parent chunk IDs
+    pub parents: Vec<RcStr>,
+    /// Child chunk IDs
+    pub children: Vec<RcStr>,
 }
 
 #[derive(Serialize, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct WebpackStatsModule {
+    /// Path to the actual file
     pub name: RcStr,
-    pub id: RcStr,
+    /// The ID of the module
+    pub id: ModuleId,
+    /// A unique ID used internally
+    pub identifier: RcStr,
     pub chunks: Vec<RcStr>,
     pub size: Option<u64>,
+    pub reasons: Vec<WebpackStatsReason>,
+}
+
+#[derive(Clone, Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct WebpackStatsReason {
+    /// The [WebpackStatsModule::name]
+    pub module: RcStr,
+    // /// The [WebpackStatsModule::id]
+    // pub module_id: ModuleId,
+    /// The [WebpackStatsModule::identifier]
+    pub module_identifier: RcStr,
+    /// A more readable name for the module (used for "pretty-printing")
+    pub module_name: RcStr,
+    /// The [type of request](/api/module-methods) used
+    #[serde(rename = "type")]
+    pub ty: RcStr,
+    // /// Raw string used for the `import` or `require` request
+    // pub user_request: RcStr,
+    // /// Lines of code that caused the module to be included
+    // pub loc: RcStr
 }
 
 #[derive(Serialize, Debug)]

packages/next/src/shared/lib/turbopack/manifest-loader.ts

@@ -339,7 +339,7 @@ export class TurbopackManifestLoader {
   private mergeWebpackStats(statsFiles: Iterable<WebpackStats>): WebpackStats {
     const entrypoints: Record<string, StatsChunkGroup> = {}
     const assets: Map<string, StatsAsset> = new Map()
-    const chunks: Map<string, StatsChunk> = new Map()
+    const chunks: Map<string | number, StatsChunk> = new Map()
     const modules: Map<string | number, StatsModule> = new Map()
 
     for (const statsFile of statsFiles) {
@@ -361,8 +361,8 @@ export class TurbopackManifestLoader {
 
       if (statsFile.chunks) {
         for (const chunk of statsFile.chunks) {
-          if (!chunks.has(chunk.name)) {
-            chunks.set(chunk.name, chunk)
+          if (!chunks.has(chunk.id!)) {
+            chunks.set(chunk.id!, chunk)
           }
         }
       }
@@ -388,6 +388,7 @@ export class TurbopackManifestLoader {
     }
 
     return {
+      version: 'Turbopack',
       entrypoints,
       assets: [...assets.values()],
       chunks: [...chunks.values()],