Cache bundle analysis per output file

@@ -1,30 +1,27 @@ mod groups; mod membership;+mod output_file; -use std::{borrow::Cow, io::Write};+use std::io::Write;  use anyhow::Result; use bincode::{Decode, Encode}; use byteorder::{BE, WriteBytesExt};-use either::Either;-use membership::output_chunk_modules; use next_core::app_structure::FileSystemPathVec;+use output_file::{AnalyzedChunkPart, analyze_output_file, analyze_traced_file}; use rustc_hash::{FxHashMap, FxHashSet}; use serde::Serialize; use turbo_rcstr::RcStr; use turbo_tasks::{-    FxIndexSet, JoinIterExt, NonLocalValue, ResolvedVc, TryFlatJoinIterExt, ValueToString,-    ValueToStringRef, Vc,+    FxIndexSet, JoinIterExt, NonLocalValue, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt,+    ValueToString, ValueToStringRef, Vc, }; use turbo_tasks_fs::{     File, FileContent, FileSystemPath,     rope::{Rope, RopeBuilder}, }; use turbo_tasks_hash::hash_xxh3_hash64;-use turbopack_analyze::split_chunk::{split_output_asset_into_parts, split_traced_file_into_parts};-use turbopack_browser::ecmascript::EcmascriptBrowserChunk; use turbopack_core::{-    SOURCE_URL_PROTOCOL,     asset::{Asset, AssetContent},     chunk::{ChunkingType, TracedMode},     module::Module,@@ -254,11 +251,12 @@ struct ModulesDataHeader {     pub traced_module_dependencies: EdgesDataReference, } -#[derive(Clone, Copy, Serialize)]+#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, NonLocalValue, Encode, Decode, Serialize)] #[serde(rename_all = "snake_case")] enum AnalyzeOutputFileCoverage {     Exact,     Unsupported,+    #[default]     NotAChunk, } @@ -687,7 +685,8 @@ pub async fn analyze_output_assets(         },     };     let route_entries = route_entries.await?.iter().cloned().collect();-    let module_index = analyze_module_index(module_graph).await?;+    let module_index_vc = analyze_module_index(module_graph);+    let module_index = module_index_vc.await?;      let mut builder =         AnalyzeDataBuilder::new(route_entries, module_index.module_index_hash.clone());@@ -696,78 +695,63 @@ pub async fn analyze_output_assets(     let mut candidates = Vec::new();     let mut browser_chunks = FxHashSet::default(); -    let prefix = format!("{SOURCE_URL_PROTOCOL}///");--    // Process the output assets and extract chunk parts.-    // Also creates sources for the chunk parts.-    for asset in output_assets-        .await?+    let output_assets = output_assets.await?;+    let traced_files = traced_files.await?;+    // One call per file: per-route fan-out over shared assets dominates analysis+    // through task-graph bookkeeping, not the work behind each call.+    let files = output_assets         .iter()-        .copied()-        .map(Either::Left)-        .chain(traced_files.await?.iter().cloned().map(Either::Right))-    {-        let file_system_path = match &asset {-            Either::Left(asset) => Either::Left(asset.path().await?),-            Either::Right(path) => Either::Right(path),-        };-        let path = match &file_system_path {-            Either::Left(path) => &path.path,-            Either::Right(path) => &path.path,-        };-        if path.ends_with(".map") || path.ends_with(".nft.json") {-            // Skip source maps.-            continue;-        }--        let filename = match &file_system_path {-            Either::Left(path) => path.to_string_ref().await?,-            Either::Right(path) => path.to_string_ref().await?,-        };+        .map(|&asset| analyze_output_file(*asset, module_index_vc))+        .chain(+            traced_files+                .iter()+                .map(|path| analyze_traced_file(path.clone())),+        )+        .try_join()+        .await?; -        let is_client = filename.starts_with("[client-fs]/");+    for file in files.iter().filter_map(|file| (**file).as_ref()) {+        let is_client = file.filename.starts_with("[client-fs]/");         let output_file_index = builder.add_output_file(AnalyzeOutputFile {-            filename: filename.clone(),+            filename: file.filename.clone(),         });-        if let Either::Left(asset) = &asset {+        if let Some(emitted) = &file.emitted {+            let membership = &emitted.membership;             asset_indices-                .entry(*asset)+                .entry(emitted.asset)                 .or_default()                 .push(output_file_index);-            if ResolvedVc::try_downcast_type::<EcmascriptBrowserChunk>(*asset).is_some() {+            if emitted.is_browser_chunk {                 browser_chunks.insert(output_file_index);             }-            let (indices, coverage, loads, async_loaders) =-                output_chunk_modules(*asset, &filename, &module_index, output_file_index).await?;-            candidates.extend(loads);-            let file = &mut builder.output_files[output_file_index as usize];-            file.module_indices.extend(indices);-            file.module_indices.sort_unstable();-            file.module_indices.dedup();-            file.async_loader_indices.extend(async_loaders);-            file.async_loader_indices.sort_unstable();-            file.async_loader_indices.dedup();-            file.module_coverage = coverage;+            candidates.extend(+                membership+                    .load_targets+                    .iter()+                    .map(|load| ChunkLoadCandidate {+                        source: output_file_index,+                        target: load.target,+                        trigger_module_index: load.trigger_module_index,+                    }),+            );+            let output = &mut builder.output_files[output_file_index as usize];+            output.module_indices.extend(&membership.module_indices);+            output.module_indices.sort_unstable();+            output.module_indices.dedup();+            output+                .async_loader_indices+                .extend(&membership.async_loader_indices);+            output.async_loader_indices.sort_unstable();+            output.async_loader_indices.dedup();+            output.module_coverage = membership.coverage;         }-        let chunk_parts = match asset {-            Either::Left(asset) => split_output_asset_into_parts(*asset).await?,-            Either::Right(path) => split_traced_file_into_parts(path).await?,-        };-        for chunk_part in &chunk_parts {-            let decoded_source = urlencoding::decode(&chunk_part.source)?;-            let source = if let Some(stripped) = decoded_source.strip_prefix(&prefix) {-                Cow::Borrowed(stripped)-            } else if decoded_source.starts_with('[') && decoded_source.contains("]/") {-                decoded_source-            } else {-                Cow::Owned(format!(-                    "[project]/{}",-                    decoded_source.trim_start_matches("../")-                ))-            };-            let source_index = builder.ensure_source(&source).1;-            let size = chunk_part.real_size + chunk_part.unaccounted_size;-            let compressed_size = chunk_part.get_compressed_size().await?.unwrap_or(size);+        for &AnalyzedChunkPart {+            ref source,+            size,+            compressed_size,+        } in &file.chunk_parts+        {+            let source_index = builder.ensure_source(source).1;             summary.size += u64::from(size);             summary.compressed_size += u64::from(compressed_size);             if is_client {

@@ -2,9 +2,10 @@ //! snapshot's ordered module index. Unknown wrappers remain unsupported.  use anyhow::{Context, Result, anyhow};+use bincode::{Decode, Encode}; use rustc_hash::FxHashMap; use turbo_rcstr::RcStr;-use turbo_tasks::{FxIndexSet, ResolvedVc, ValueToString, Vc};+use turbo_tasks::{FxIndexSet, NonLocalValue, ResolvedVc, ValueToString, Vc}; use turbopack_browser::ecmascript::{     EcmascriptBrowserChunk, EcmascriptBrowserRuntimeChunk, EcmascriptBrowserSingleEntryChunk,     EcmascriptBrowserWorkerEntrypoint,@@ -23,7 +24,22 @@ use turbopack_ecmascript::{ }; use turbopack_nodejs::{EcmascriptBuildNodeChunk, EcmascriptBuildNodeRuntimeChunk}; -use crate::analyze::{AnalyzeModuleIndex, AnalyzeOutputFileCoverage, ChunkLoadCandidate};+use crate::analyze::{AnalyzeModuleIndex, AnalyzeOutputFileCoverage};++#[derive(Clone, Debug, PartialEq, Eq, NonLocalValue, Encode, Decode)]+pub(super) struct ChunkLoadTarget {+    pub target: ResolvedVc<Box<dyn OutputAsset>>,+    /// Index of the dynamically imported target module in the analyzer's shared module index.+    pub trigger_module_index: u32,+}++#[derive(Clone, Debug, Default, PartialEq, Eq, NonLocalValue, Encode, Decode)]+pub(super) struct OutputChunkModules {+    pub module_indices: Vec<u32>,+    pub coverage: AnalyzeOutputFileCoverage,+    pub load_targets: Vec<ChunkLoadTarget>,+    pub async_loader_indices: Vec<u32>,+}  async fn insert_chunk_item(     module: Vc<Box<dyn Module>>,@@ -78,8 +94,7 @@ fn insert_chunk_item_ident( async fn chunk_item_load_candidates(     item: ResolvedVc<Box<dyn turbopack_ecmascript::chunk::EcmascriptChunkItem>>,     index: &AnalyzeModuleIndex,-    source: u32,-) -> Result<Vec<ChunkLoadCandidate>> {+) -> Result<Vec<ChunkLoadTarget>> {     let module = item.module().to_resolved().await?;     let trigger = if let Some(loader) = ResolvedVc::try_downcast_type::<AsyncLoaderModule>(module) {         loader.await?.inner@@ -101,8 +116,7 @@ async fn chunk_item_load_candidates(     Ok(assets         .iter()         .copied()-        .map(|target| ChunkLoadCandidate {-            source,+        .map(|target| ChunkLoadTarget {             target,             trigger_module_index,         })@@ -112,22 +126,16 @@ async fn chunk_item_load_candidates( /// Enumerate known browser/Node.js JS and CSS chunk items; unknown wrappers remain unsupported. pub(super) async fn output_chunk_modules(     asset: ResolvedVc<Box<dyn OutputAsset>>,-    filename: &str,+    filename: &RcStr,     module_index: &AnalyzeModuleIndex,-    output_file_index: u32,-) -> Result<(-    Vec<u32>,-    AnalyzeOutputFileCoverage,-    Vec<ChunkLoadCandidate>,-    Vec<u32>,-)> {+) -> Result<OutputChunkModules> {     if ResolvedVc::try_downcast_type::<EcmascriptBrowserRuntimeChunk>(asset).is_some()         || ResolvedVc::try_downcast_type::<EcmascriptBuildNodeRuntimeChunk>(asset).is_some()         || ResolvedVc::try_downcast_type::<SingleFileEcmascriptOutput>(asset).is_some()         || ResolvedVc::try_downcast_type::<EcmascriptBrowserSingleEntryChunk>(asset).is_some()         || ResolvedVc::try_downcast_type::<EcmascriptBrowserWorkerEntrypoint>(asset).is_some()     {-        return Ok((vec![], AnalyzeOutputFileCoverage::NotAChunk, vec![], vec![]));+        return Ok(OutputChunkModules::default());     }     let mut indices = FxIndexSet::default();     let mut async_loaders = FxIndexSet::default();@@ -157,14 +165,8 @@ pub(super) async fn output_chunk_modules(                     )                     .await                     .with_context(|| format!("Analyzing output {filename}"))?;-                    candidates.extend(-                        chunk_item_load_candidates(-                            item.chunk_item,-                            module_index,-                            output_file_index,-                        )-                        .await?,-                    );+                    candidates+                        .extend(chunk_item_load_candidates(item.chunk_item, module_index).await?);                 }                 EcmascriptChunkItemOrBatchWithAsyncInfo::Batch(batch) => {                     for item in &batch.await?.chunk_items {@@ -177,12 +179,7 @@ pub(super) async fn output_chunk_modules(                         .await                         .with_context(|| format!("Analyzing output {filename}"))?;                         candidates.extend(-                            chunk_item_load_candidates(-                                item.chunk_item,-                                module_index,-                                output_file_index,-                            )-                            .await?,+                            chunk_item_load_candidates(item.chunk_item, module_index).await?,                         );                     }                 }@@ -204,19 +201,19 @@ pub(super) async fn output_chunk_modules(         // Other emitted JS/CSS wrappers cannot enumerate their module members.         enumerated = false;     } else {-        return Ok((vec![], AnalyzeOutputFileCoverage::NotAChunk, vec![], vec![]));+        return Ok(OutputChunkModules::default());     }     let coverage = if enumerated {         AnalyzeOutputFileCoverage::Exact     } else {         AnalyzeOutputFileCoverage::Unsupported     };-    Ok((-        indices.into_iter().collect(),+    Ok(OutputChunkModules {+        module_indices: indices.into_iter().collect(),         coverage,-        candidates,-        async_loaders.into_iter().collect(),-    ))+        load_targets: candidates,+        async_loader_indices: async_loaders.into_iter().collect(),+    }) }  #[cfg(test)]

@@ -0,0 +1,116 @@+//! One emitted or traced file's analysis, cached per file and shared by every+//! route that includes it.++use anyhow::Result;+use bincode::{Decode, Encode};+use turbo_rcstr::RcStr;+use turbo_tasks::{NonLocalValue, ResolvedVc, ValueToStringRef, Vc};+use turbo_tasks_fs::FileSystemPath;+use turbopack_analyze::split_chunk::{+    ChunkPart, split_output_asset_into_parts, split_traced_file_into_parts,+};+use turbopack_browser::ecmascript::EcmascriptBrowserChunk;+use turbopack_core::{SOURCE_URL_PROTOCOL, output::OutputAsset};++use crate::analyze::{+    AnalyzeModuleIndex,+    membership::{OutputChunkModules, output_chunk_modules},+};++#[derive(Clone, Debug, PartialEq, Eq, NonLocalValue, Encode, Decode)]+pub(super) struct AnalyzedChunkPart {+    /// Normalized source path, ready for the route's source tree.+    pub source: RcStr,+    pub size: u32,+    pub compressed_size: u32,+}++#[derive(Clone, Debug, PartialEq, Eq, NonLocalValue, Encode, Decode)]+pub(super) struct EmittedChunk {+    pub asset: ResolvedVc<Box<dyn OutputAsset>>,+    pub is_browser_chunk: bool,+    pub membership: OutputChunkModules,+}++#[derive(Clone, Debug, PartialEq, Eq, NonLocalValue, Encode, Decode)]+pub(super) struct AnalyzedOutputFile {+    pub filename: RcStr,+    pub chunk_parts: Vec<AnalyzedChunkPart>,+    /// `None` for traced files.+    pub emitted: Option<EmittedChunk>,+}++/// `None` for source maps and trace manifests, which are not reported.+#[turbo_tasks::value(transparent)]+pub(super) struct OptionAnalyzedOutputFile(Option<AnalyzedOutputFile>);++#[turbo_tasks::function]+pub(super) async fn analyze_output_file(+    asset: ResolvedVc<Box<dyn OutputAsset>>,+    module_index: Vc<AnalyzeModuleIndex>,+) -> Result<Vc<OptionAnalyzedOutputFile>> {+    let path = asset.path().await?;+    if !is_reported_file(&path) {+        return Ok(Vc::cell(None));+    }+    let filename = path.to_string_ref().await?;+    let membership = output_chunk_modules(asset, &filename, &*module_index.await?).await?;+    let chunk_parts = analyzed_chunk_parts(&split_output_asset_into_parts(*asset).await?).await?;+    Ok(Vc::cell(Some(AnalyzedOutputFile {+        filename,+        chunk_parts,+        emitted: Some(EmittedChunk {+            asset,+            is_browser_chunk: ResolvedVc::try_downcast_type::<EcmascriptBrowserChunk>(asset)+                .is_some(),+            membership,+        }),+    })))+}++#[turbo_tasks::function]+pub(super) async fn analyze_traced_file(+    path: FileSystemPath,+) -> Result<Vc<OptionAnalyzedOutputFile>> {+    if !is_reported_file(&path) {+        return Ok(Vc::cell(None));+    }+    Ok(Vc::cell(Some(AnalyzedOutputFile {+        filename: path.to_string_ref().await?,+        chunk_parts: analyzed_chunk_parts(&split_traced_file_into_parts(path).await?).await?,+        emitted: None,+    })))+}++fn is_reported_file(path: &FileSystemPath) -> bool {+    !(path.path.ends_with(".map") || path.path.ends_with(".nft.json"))+}++async fn analyzed_chunk_parts(chunk_parts: &[ChunkPart]) -> Result<Vec<AnalyzedChunkPart>> {+    let mut analyzed = Vec::with_capacity(chunk_parts.len());+    for part in chunk_parts {+        let size = part.real_size + part.unaccounted_size;+        analyzed.push(AnalyzedChunkPart {+            source: normalize_source(&part.source)?,+            size,+            compressed_size: part.get_compressed_size().await?.unwrap_or(size),+        });+    }+    Ok(analyzed)+}++fn normalize_source(raw: &str) -> Result<RcStr> {+    let decoded = urlencoding::decode(raw)?;+    Ok(+        if let Some(path) = decoded+            .strip_prefix(SOURCE_URL_PROTOCOL.as_str())+            .and_then(|rest| rest.strip_prefix("///"))+        {+            path.into()+        } else if decoded.starts_with('[') && decoded.contains("]/") {+            decoded.as_ref().into()+        } else {+            format!("[project]/{}", decoded.trim_start_matches("../")).into()+        },+    )+}

@@ -60,36 +60,30 @@ impl ChunkPart {     } } -#[turbo_tasks::value(transparent)]-#[derive(Debug)]-pub struct ChunkParts(Vec<ChunkPart>);--#[turbo_tasks::function]-pub async fn split_traced_file_into_parts(path: FileSystemPath) -> Result<Vc<ChunkParts>> {+pub async fn split_traced_file_into_parts(path: FileSystemPath) -> Result<Vec<ChunkPart>> {     let source = FileSource::new(path.clone());     let content = source.content().await?;     let AssetContent::File(file_content) = &*content else {-        return Ok(Vc::cell(vec![]));+        return Ok(vec![]);     };     let FileContent::Content(content) = &*file_content.await? else {-        return Ok(Vc::cell(vec![]));+        return Ok(vec![]);     };     let content = content.content();     let lines_vc = file_content.lines().to_resolved().await?;      self_mapped(path.to_string_ref().await?, content, lines_vc).await } -#[turbo_tasks::function] pub async fn split_output_asset_into_parts(     asset: Vc<Box<dyn OutputAsset>>,-) -> Result<Vc<ChunkParts>> {+) -> Result<Vec<ChunkPart>> {     let content = asset.content().await?;     let AssetContent::File(file_content) = &*content else {-        return Ok(Vc::cell(vec![]));+        return Ok(vec![]);     };     let FileContent::Content(content) = &*file_content.await? else {-        return Ok(Vc::cell(vec![]));+        return Ok(vec![]);     };     let content = content.content();     let lines_vc = file_content.lines().to_resolved().await?;@@ -397,37 +391,37 @@ pub async fn split_output_asset_into_parts(         }     } -    Ok(Vc::cell(chunk_parts.into_values().collect()))+    Ok(chunk_parts.into_values().collect()) }  pub async fn self_mapped(     path: RcStr,     content: &Rope,     lines: ResolvedVc<FileLinesContent>,-) -> Result<Vc<ChunkParts>> {+) -> Result<Vec<ChunkPart>> {     let len = content.len().try_into().unwrap_or(u32::MAX);-    Ok(Vc::cell(vec![ChunkPart {+    Ok(vec![ChunkPart {         source: path,         real_size: len,         unaccounted_size: 0,         ranges: vec![],         lines,-    }]))+    }]) }  async fn unaccounted(     asset: Vc<Box<dyn OutputAsset>>,     content: &Rope,     lines: ResolvedVc<FileLinesContent>,-) -> Result<Vc<ChunkParts>> {+) -> Result<Vec<ChunkPart>> {     let len = content.len().try_into().unwrap_or(u32::MAX);-    Ok(Vc::cell(vec![ChunkPart {+    Ok(vec![ChunkPart {         source: asset.path().to_string().owned().await?,         real_size: 0,         unaccounted_size: len,         ranges: vec![],         lines,-    }]))+    }]) }  fn append_content_between(

@@ -5,12 +5,12 @@ use anyhow::Result; use serde_json::json; use turbo_rcstr::rcstr;-use turbo_tasks::{OperationVc, ResolvedVc, Vc};+use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks_fs::{     File, FileContent, FileSystem, FileSystemPath, VirtualFileSystem, rope::Rope, }; use turbo_tasks_testing::{Registration, register, run_once};-use turbopack_analyze::split_chunk::{ChunkPartRange, ChunkParts, split_output_asset_into_parts};+use turbopack_analyze::split_chunk::{ChunkPart, ChunkPartRange, split_output_asset_into_parts}; use turbopack_core::{     asset::{Asset, AssetContent},     code_builder::{Code, CodeBuilder},@@ -55,13 +55,10 @@ async fn split_chunk() {         }         .resolved_cell(); -        #[turbo_tasks::function(operation, root)]-        fn split_parts_operation(asset: ResolvedVc<TestAsset>) -> Vc<ChunkParts> {-            split_output_asset_into_parts(Vc::upcast(*asset))-        }--        let parts_op = split_parts_operation(asset);-        let parts = parts_op.read_strongly_consistent().await?;+        let result = split_parts_operation(asset)+            .read_strongly_consistent()+            .await?;+        let parts = &result.parts;          assert_eq!(parts.len(), 2); @@ -96,34 +93,34 @@ async fn split_chunk() {             }]         ); -        #[turbo_tasks::function(operation, root)]-        async fn compressed_size_operation(-            parts: OperationVc<ChunkParts>,-            index: usize,-        ) -> Result<Vc<u32>> {-            Ok(Vc::cell(-                parts.connect().await?[index]-                    .get_compressed_size()-                    .await?-                    .unwrap(),-            ))-        }--        let compressed_size_0 = compressed_size_operation(parts_op, 0)-            .read_strongly_consistent()-            .await?;-        let compressed_size_1 = compressed_size_operation(parts_op, 1)-            .read_strongly_consistent()-            .await?;-        assert_eq!(*compressed_size_0, 43);-        assert_eq!(*compressed_size_1, 28);+        assert_eq!(result.compressed_sizes, [Some(43), Some(28)]);          anyhow::Ok(())     })     .await     .unwrap() } +#[turbo_tasks::value]+struct SplitResult {+    parts: Vec<ChunkPart>,+    compressed_sizes: Vec<Option<u32>>,+}++#[turbo_tasks::function(operation, root)]+async fn split_parts_operation(asset: ResolvedVc<TestAsset>) -> Result<Vc<SplitResult>> {+    let parts = split_output_asset_into_parts(Vc::upcast(*asset)).await?;+    let mut compressed_sizes = Vec::with_capacity(parts.len());+    for part in &parts {+        compressed_sizes.push(part.get_compressed_size().await?);+    }+    Ok(SplitResult {+        parts,+        compressed_sizes,+    }+    .cell())+}+ #[turbo_tasks::value] struct TestAsset {     code: ResolvedVc<Code>,