vercel/next.js · #94842
Turbopack: resolve per-module data once during module graph construction
crates/next-api/src/app.rs10 + / 14 −
@@ -892,6 +892,12 @@ impl AppProject { let next_mode_ref = next_mode.await?; let should_trace = *self.project.should_write_nft_manifests().await?; let should_read_binding_usage = next_mode_ref.is_production();+ let remove_unused_imports = *self+ .project+ .next_config()+ .turbopack_remove_unused_imports(next_mode)+ .await?;+ let graph_options = self.project.module_graph_options().await?; // Implements layout segment optimization to compute a graph "chain" for each layout // segment@@ -937,8 +943,7 @@ impl AppProject { ), ]), visited_modules,- should_trace,- should_read_binding_usage,+ graph_options, ); graphs.push(graph); visited_modules = VisitedModules::concatenate(visited_modules, graph);@@ -956,8 +961,7 @@ impl AppProject { ResolvedVc::upcast(*module), )]), visited_modules,- should_trace,- should_read_binding_usage,+ graph_options, ); graphs.push(graph); let is_layout = module.server_path().await?.file_stem() == Some("layout");@@ -979,8 +983,7 @@ impl AppProject { let graph = SingleModuleGraph::new_with_entries_visited_intern( GraphEntries::from_chunk_groups(vec![rsc_entry_chunk_group]), visited_modules,- should_trace,- should_read_binding_usage,+ graph_options, ); graphs.push(graph); visited_modules = VisitedModules::concatenate(visited_modules, graph);@@ -990,8 +993,7 @@ impl AppProject { let additional_module_graph = SingleModuleGraph::new_with_entries_visited_intern( additional_entries.owned().await?, visited_modules,- should_trace,- should_read_binding_usage,+ graph_options, ); graphs.push(additional_module_graph); @@ -1003,12 +1005,6 @@ impl AppProject { span.record("modules", module_count); } - let remove_unused_imports = *self- .project- .next_config()- .turbopack_remove_unused_imports(next_mode)- .await?;- let (full, binding_usage_info) = if remove_unused_imports { let full_with_unused_references = ModuleGraph::from_graphs(graphs.clone(), None);crates/next-api/src/next_server_nft.rs8 + / 3 −
@@ -17,7 +17,7 @@ use turbopack_core::{ context::AssetContext, file_source::FileSource, module::{Module, Modules},- module_graph::{GraphEntries, ModuleGraph, SingleModuleGraph},+ module_graph::{GraphEntries, ModuleGraph, ModuleGraphOptions, SingleModuleGraph}, output::{OutputAsset, OutputAssets, OutputAssetsReference}, reference_type::CommonJsReferenceSubType, resolve::{ResolveErrorMode, origin::PlainResolveOrigin, parse::Request},@@ -191,8 +191,13 @@ impl Asset for ServerNftJsonAsset { let module_graph = ModuleGraph::from_graphs( vec![SingleModuleGraph::new_with_entries( GraphEntries::new(vec![], self.entries().owned().await?).resolved_cell(),- true,- false,+ ModuleGraphOptions {+ include_ident_strings: false,+ include_side_effects: false,+ include_mergeable: false,+ include_traced: true,+ include_binding_usage: false,+ }, )], None, )crates/next-api/src/nft.rs1 + / 1 −
@@ -455,7 +455,7 @@ async fn traced_module_idents_for_graph( Ok(Vc::cell( traced_modules .into_iter()- .map(async |module| Ok((module, module.ident().await?)))+ .map(async |module| Ok((module, module_graph.module_ident(module)?.await?))) .try_join() .await? .into_iter()crates/next-api/src/pages.rs10 + / 18 −
@@ -828,8 +828,7 @@ impl PageEndpoint { // availability, so both entry groups have to live in the same graph. Build a graph // "chain" for app, page -- the same layout segment optimization `ssr_module_graph` uses // for document, app, page.- let should_trace = *project.should_write_nft_manifests().await?;- let should_read_binding_usage = project.next_mode().await?.is_production();+ let graph_options = project.module_graph_options().await?; let mut graphs = vec![]; let mut visited_modules = VisitedModules::empty();@@ -846,8 +845,7 @@ impl PageEndpoint { heuristics: EntryHeuristics::default(), }]), visited_modules,- should_trace,- should_read_binding_usage,+ graph_options, ); graphs.push(graph); visited_modules = VisitedModules::concatenate(visited_modules, graph);@@ -859,8 +857,7 @@ impl PageEndpoint { heuristics: EntryHeuristics::default(), }]), visited_modules,- should_trace,- should_read_binding_usage,+ graph_options, ); graphs.push(graph); @@ -874,9 +871,11 @@ impl PageEndpoint { if *project.per_page_module_graph().await? { let next_mode = project.next_mode();- let next_mode_ref = next_mode.await?;- let should_trace = *project.should_write_nft_manifests().await?;- let should_read_binding_usage = next_mode_ref.is_production();+ let remove_unused_imports = *project+ .next_config()+ .turbopack_remove_unused_imports(next_mode)+ .await?;+ let graph_options = project.module_graph_options().await?; let ssr_chunk_module = self.internal_ssr_chunk_module().await?; // Implements layout segment optimization to compute a graph "chain" for document, app,@@ -894,8 +893,7 @@ impl PageEndpoint { let graph = SingleModuleGraph::new_with_entries_visited_intern( GraphEntries::from_chunk_groups(vec![layout_chunk_group_entry(module)]), visited_modules,- should_trace,- should_read_binding_usage,+ graph_options, ); graphs.push(graph); visited_modules = VisitedModules::concatenate(visited_modules, graph);@@ -907,16 +905,10 @@ impl PageEndpoint { heuristics: EntryHeuristics::default(), }]), visited_modules,- should_trace,- should_read_binding_usage,+ graph_options, ); graphs.push(graph); - let remove_unused_imports = *project- .next_config()- .turbopack_remove_unused_imports(next_mode)- .await?;- let graph = if remove_unused_imports { let graph = ModuleGraph::from_graphs(graphs.clone(), None); // This is a per-route graph: a module that is unused here might still be codecrates/next-api/src/project.rs77 + / 43 −
@@ -76,7 +76,7 @@ use turbopack_core::{ }, module::{Module, Modules}, module_graph::{- GraphEntries, ModuleGraph, SingleModuleGraph, VisitedModules,+ GraphEntries, ModuleGraph, ModuleGraphOptions, SingleModuleGraph, VisitedModules, binding_usage_info::{ BindingUsageInfo, OptionBindingUsageInfo, compute_binding_usage_info, },@@ -1173,6 +1173,43 @@ impl Issue for ConflictIssue { } } +impl Project {+ /// Whether the project uses the deterministic module-id strategy, i.e. whether+ /// `get_global_module_id_strategy` runs (and therefore whether the module graph needs each+ /// module's stringified ident). `Named` (the dev default, and an opt-in build config) does not.+ ///+ /// The strategy only ever runs on the whole-app graph, which is only built when *not* using+ /// per-page graphs — so this also requires `!per_page_module_graph`. Encoding that here means+ /// every graph (per-page or whole-app) can derive `include_ident_strings` from this single+ /// query without the caller having to know which graph it's building.+ pub(super) async fn uses_deterministic_module_ids(self: Vc<Self>) -> Result<bool> {+ Ok(!*self.per_page_module_graph().await?+ && matches!(+ *self.next_config().module_ids(self.next_mode()).await?,+ ModuleIdStrategyConfig::Deterministic+ ))+ }++ /// Builds the [`ModuleGraphOptions`] shared by all project-config-derived module graphs, so the+ /// per-page and whole-app graphs can't drift. Every field is derived from project config;+ /// `include_ident_strings` follows [`Self::uses_deterministic_module_ids`] (the only consumer+ /// of the stringified idents is `get_global_module_id_strategy`, which runs only on the+ /// whole-app graph).+ pub(super) async fn module_graph_options(self: Vc<Self>) -> Result<ModuleGraphOptions> {+ let next_mode = self.next_mode();+ Ok(ModuleGraphOptions {+ include_ident_strings: self.uses_deterministic_module_ids().await?,+ include_side_effects: *self+ .next_config()+ .turbopack_remove_unused_imports(next_mode)+ .await?,+ include_mergeable: *self.next_config().turbo_scope_hoisting(next_mode).await?,+ include_traced: *self.should_write_nft_manifests().await?,+ include_binding_usage: self.next_mode().await?.is_production(),+ })+ }+}+ #[turbo_tasks::value_impl] impl Project { #[turbo_tasks::function]@@ -1595,8 +1632,7 @@ impl Project { modules: vec![entry], heuristics: EntryHeuristics::default(), },- /* include_traced */ *self.should_write_nft_manifests().await?,- /* include_binding_usage */ self.next_mode().await?.is_production(),+ self.module_graph_options().await?, )], None, )@@ -1625,8 +1661,7 @@ impl Project { heuristics: EntryHeuristics::default(), }]) .resolved_cell(),- /* include_traced */ *self.should_write_nft_manifests().await?,- /* include_binding_usage */ self.next_mode().await?.is_production(),+ self.module_graph_options().await?, )], None, )@@ -1953,31 +1988,34 @@ impl Project { // appears in the module graph, but the synthesized `target.css` module's path suffix does. // `ident.path.path` does not include the query string (that lives on `ident.query`), so // `ends_with` is the correct matcher here.- static FEATURE_MODULE_PATH_SUFFIXES: &[(&str, &str)] = &[- ("next/image", "/next/image.js"),- ("next/image", "/next/dist/api/image.js"),- ("next/image", "/next/dist/esm/api/image.js"),+ static FEATURE_MODULE_PATH_SUFFIXES: &[(RcStr, &str)] = &[+ (rcstr!("next/image"), "/next/image.js"),+ (rcstr!("next/image"), "/next/dist/api/image.js"),+ (rcstr!("next/image"), "/next/dist/esm/api/image.js"), // next/dist/api/image.js re-exports this module. When Turbopack can prove that // the re-export has no side effects, it may flatten the re-export // chain and leave this as the feature module in the graph.- ("next/image", "/next/dist/client/image-component.js"),- ("next/image", "/next/dist/esm/client/image-component.js"),+ (rcstr!("next/image"), "/next/dist/client/image-component.js"),+ (+ rcstr!("next/image"),+ "/next/dist/esm/client/image-component.js",+ ), (- "next/image",+ rcstr!("next/image"), "/next/dist/shared/lib/image-external-get-image-props.js", ), (- "next/image",+ rcstr!("next/image"), "/next/dist/esm/shared/lib/image-external-get-image-props.js", ),- ("next/future/image", "/next/future/image.js"),- ("next/legacy/image", "/next/legacy/image.js"),- ("next/script", "/next/script.js"),- ("next/dynamic", "/next/dynamic.js"),- ("next/font/google", "/next/font/google/target.css"),- ("next/font/local", "/next/font/local/target.css"),- ("@next/font/google", "/@next/font/google/target.css"),- ("@next/font/local", "/@next/font/local/target.css"),+ (rcstr!("next/future/image"), "/next/future/image.js"),+ (rcstr!("next/legacy/image"), "/next/legacy/image.js"),+ (rcstr!("next/script"), "/next/script.js"),+ (rcstr!("next/dynamic"), "/next/dynamic.js"),+ (rcstr!("next/font/google"), "/next/font/google/target.css"),+ (rcstr!("next/font/local"), "/next/font/local/target.css"),+ (rcstr!("@next/font/google"), "/@next/font/google/target.css"),+ (rcstr!("@next/font/local"), "/@next/font/local/target.css"), ]; // TODO: useSwcLoader is not being reported as it is not directly corresponds (it checks@@ -2044,12 +2082,12 @@ impl Project { // to that feature's unique-importer set. let module_graph = self.whole_app_module_graphs().await?.full.await?; - let matching: FxHashMap<ResolvedVc<Box<dyn Module>>, &'static str> = module_graph- .iter_nodes()- .map(async |node| {- let ident = node.ident().await?;+ let matching: FxHashMap<ResolvedVc<Box<dyn Module>>, &'static RcStr> = module_graph+ .iter_modules_with_idents()+ .map(async |(node, ident)| {+ let ident = ident.await?; let path = &ident.path.path;- for &(feature, suffix) in FEATURE_MODULE_PATH_SUFFIXES {+ for (feature, suffix) in FEATURE_MODULE_PATH_SUFFIXES { if path.ends_with(suffix) { return Ok(Some((node, feature))); }@@ -2068,16 +2106,16 @@ impl Project { // We could filter via `BindingUsageInfo` to only count edges that survive tree-shaking, // but staying parallel to webpack lets dashboards compare counts across the two bundlers // directly.- let mut pairs: FxHashSet<(&'static str, ResolvedVc<Box<dyn Module>>)> =- FxHashSet::default();+ let mut pairs: Vec<(&'static RcStr, ResolvedVc<Box<dyn Module>>)> =+ Vec::with_capacity(matching.len() * 2); module_graph.traverse_edges_unordered(|parent, node| { if let Some((parent_node, _)) = parent && let Some(&feature) = matching.get(&node) // A feature's public entry point may re-export another matched module. Only // count the external importer, not the internal re-export edge. && matching.get(&parent_node) != Some(&feature) {- pairs.insert((feature, parent_node));+ pairs.push((feature, parent_node)); } Ok(()) })?;@@ -2089,7 +2127,7 @@ impl Project { let parent_source_keys = pairs .into_iter() .map(async |(feature, parent)| {- let ident = parent.ident().await?;+ let ident = module_graph.module_ident(parent)?.await?; let key = ( ident.path.path.clone(), ident.query.clone(),@@ -2100,13 +2138,13 @@ impl Project { .try_join() .await?; - let mut importers: FxHashMap<&'static str, FxHashSet<(RcStr, RcStr, RcStr)>> =+ let mut importers: FxHashMap<&'static RcStr, FxHashSet<(RcStr, RcStr, RcStr)>> = FxHashMap::default(); for (feature, key) in parent_source_keys { importers.entry(feature).or_default().insert(key); } for (feature, unique_sources) in importers {- features.push((RcStr::from(feature), unique_sources.len() as u32));+ features.push((feature.clone(), unique_sources.len() as u32)); } features.sort_by(|a, b| a.0.cmp(&b.0));@@ -2946,22 +2984,19 @@ async fn whole_app_module_graph_operation( let span_clone = span.clone(); let result = async move { let next_mode = project.next_mode();- let should_trace = *project.should_write_nft_manifests().await?;- let should_read_binding_usage = next_mode.await?.is_production();+ let turbopack_remove_unused_imports = *project+ .next_config()+ .turbopack_remove_unused_imports(next_mode)+ .await?;+ let graph_options = project.module_graph_options().await?; let base_single_module_graph = SingleModuleGraph::new_with_entries( project.get_all_entries().to_resolved().await?,- should_trace,- should_read_binding_usage,+ graph_options, ); let base_visited_modules = VisitedModules::from_graph(base_single_module_graph); let base = ModuleGraph::from_graphs(vec![base_single_module_graph], None); - let turbopack_remove_unused_imports = *project- .next_config()- .turbopack_remove_unused_imports(next_mode)- .await?;- let base = if turbopack_remove_unused_imports { // TODO suboptimal that we do compute_binding_usage_info twice (once for the base // graph and later for the full graph)@@ -2982,8 +3017,7 @@ async fn whole_app_module_graph_operation( let additional_module_graph = SingleModuleGraph::new_with_entries_visited( additional_entries, base_visited_modules,- should_trace,- should_read_binding_usage,+ graph_options, ); if !span.is_disabled() {crates/next-api/src/server_actions.rs17 + / 9 −
@@ -48,8 +48,9 @@ use turbopack_core::{ ident::AssetIdent, module::{Module, Modules}, module_graph::{- GraphTraversalAction, ModuleGraph, ModuleGraphLayer, async_module_info::AsyncModulesInfo,- binding_usage_info::ModuleExportUsage, chunk_group_info::ChunkGroupKey,+ GraphTraversalAction, ModuleGraph, ModuleGraphLayer, SingleModuleGraphNode,+ async_module_info::AsyncModulesInfo, binding_usage_info::ModuleExportUsage,+ chunk_group_info::ChunkGroupKey, }, output::{OutputAsset, OutputAssets, OutputAssetsReference}, reference_type::{EcmaScriptModulesReferenceSubType, ReferenceType},@@ -1037,13 +1038,20 @@ pub struct AllModuleActions( pub async fn map_server_actions( graph: OperationVc<ModuleGraphLayer>, ) -> Result<Vc<AllModuleActions>> {- let graph = graph.connect();+ let graph = graph.connect().await?; let actions = graph- .await?- .iter_reachable_modules()?- .map(async |module| {+ .iter_reachable_nodes()?+ .map(async |(idx, node)| {+ let SingleModuleGraphNode::Module(module) = node else {+ // VisitedModule nodes were handled in their owning graph.+ return Ok(None);+ };+ let ident = graph+ .node_ident(idx)+ .context("Module nodes always have an ident")?; // TODO: compare module contexts instead?- let layer = match module.ident().await?.layer.as_ref() {+ let ident = ident.await?;+ let layer = match ident.layer.as_ref() { Some(layer) if layer.name() == "app-rsc" || layer.name() == "app-edge-rsc"@@ -1058,9 +1066,9 @@ pub async fn map_server_actions( }; // TODO the old implementation did parse_actions(to_rsc_context(module)) // is that really necessary?- Ok(parse_actions(*module)+ Ok(parse_actions(**module) .await?- .map(|action_map| (module, (layer, action_map))))+ .map(|action_map| (*module, (layer, action_map)))) }) .try_flat_join() .await?;crates/next-api/src/service_worker.rs11 + / 3 −
@@ -10,7 +10,7 @@ use turbopack_core::{ issue::{Issue, IssueExt, IssueSeverity, IssueStage, StyledString}, module::Module, module_graph::{- ModuleGraph, SingleModuleGraph,+ ModuleGraph, ModuleGraphOptions, SingleModuleGraph, chunk_group_info::{ChunkGroupEntry, ChunkGroupKey, EntryHeuristics}, }, output::{OutputAsset, OutputAssets},@@ -113,8 +113,16 @@ async fn service_worker_chunk( modules: vec![module], heuristics: EntryHeuristics::default(), },- /* include_traced */ *project.should_write_nft_manifests().await?,- /* include_binding_usage */ is_production,+ ModuleGraphOptions {+ // The module-id strategy isn't computed on this isolated service-worker graph.+ include_ident_strings: false,+ include_side_effects: is_production,+ // The service-worker chunking context doesn't enable module merging (scope+ // hoisting), so `compute_merged_modules` never runs on this graph.+ include_mergeable: false,+ include_traced: *project.should_write_nft_manifests().await?,+ include_binding_usage: is_production,+ }, )], /* binding_usage */ None, )crates/next-core/src/next_font/google/mod.rs2 + / 3 −
@@ -22,7 +22,7 @@ use turbopack_core::{ context::AssetContext, ident::Layer, issue::{IssueExt, IssueSeverity},- module_graph::{ModuleGraph, SingleModuleGraph},+ module_graph::{ModuleGraph, ModuleGraphOptions, SingleModuleGraph}, reference_type::{InnerAssets, ReferenceType}, resolve::{ ResolveResult,@@ -794,8 +794,7 @@ async fn get_mock_stylesheet( let module_graph = ModuleGraph::from_graphs( vec![SingleModuleGraph::new_with_entries( entries.graph_entries().to_resolved().await?,- false,- false,+ ModuleGraphOptions::default(), )], None, );crates/next-core/src/next_server_component/server_component_module.rs5 + / 3 −
@@ -1,6 +1,6 @@ use anyhow::Result; use indoc::formatdoc;-use turbo_rcstr::rcstr;+use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{ResolvedVc, Vc}; use turbo_tasks_fs::FileSystemPath; use turbopack_core::{@@ -60,11 +60,13 @@ impl NextServerComponentModule { } } +const NEXT_JS_SERVER_COMPONENT: RcStr = rcstr!("Next.js Server Component");+ impl NextServerComponentModule { fn module_reference(&self) -> Vc<Box<dyn ModuleReference>> { Vc::upcast(SingleChunkableModuleReference::new( Vc::upcast(*self.module),- rcstr!("Next.js Server Component"),+ NEXT_JS_SERVER_COMPONENT, ExportUsage::passthrough(true), )) }@@ -79,7 +81,7 @@ impl Module for NextServerComponentModule { .ident() .owned() .await?- .with_modifier(rcstr!("Next.js Server Component"))+ .with_modifier(NEXT_JS_SERVER_COMPONENT) .into_vc()) } turbopack/crates/turbopack-cli/src/build/mod.rs15 + / 3 −
@@ -34,7 +34,7 @@ use turbopack_core::{ issue::{IssueReporter, IssueSeverity, handle_issues}, module::Module, module_graph::{- GraphEntries, ModuleGraph, SingleModuleGraph,+ GraphEntries, ModuleGraph, ModuleGraphOptions, SingleModuleGraph, binding_usage_info::compute_binding_usage_info, chunk_group_info::{ChunkGroupEntry, ChunkGroupKey, EntryHeuristics}, },@@ -316,8 +316,20 @@ async fn build_internal( heuristics: EntryHeuristics::default(), }]) .resolved_cell(),- false,- true,+ ModuleGraphOptions {+ include_binding_usage: true,+ // `compute_binding_usage_info(_, true)` below runs+ // `compute_side_effect_free_module_info`, which reads `side_effects` from+ // the graph node.+ include_side_effects: true,+ // Module merging (driven by `scope_hoist` in the chunking context below) reads+ // `is_mergeable` from the graph node and bails if absent.+ include_mergeable: scope_hoist,+ // `get_global_module_id_strategy` below reads each module's `ident_string` from the+ // graph node; collect it so it doesn't fan out a `to_string()` per module.+ include_ident_strings: true,+ ..Default::default()+ }, ); let mut module_graph = ModuleGraph::from_graphs(vec![single_graph], None); let binding_usage = compute_binding_usage_info(turbopack/crates/turbopack-cli/src/dev/web_entry_source.rs2 + / 3 −
@@ -14,7 +14,7 @@ use turbopack_core::{ file_source::FileSource, module::Module, module_graph::{- GraphEntries, ModuleGraph, SingleModuleGraph,+ GraphEntries, ModuleGraph, ModuleGraphOptions, SingleModuleGraph, chunk_group_info::{ChunkGroupEntry, EntryHeuristics}, }, reference_type::{EntryReferenceSubType, ReferenceType},@@ -190,8 +190,7 @@ pub async fn create_web_entry_source( let module_graph = ModuleGraph::from_graphs( vec![SingleModuleGraph::new_with_entries( GraphEntries::from_chunk_groups(chunk_group_entries).resolved_cell(),- false,- false,+ ModuleGraphOptions::default(), )], None, );turbopack/crates/turbopack-core/src/chunk/chunking/production.rs23 + / 3 −
@@ -11,7 +11,7 @@ use turbo_tasks::{FxIndexMap, FxIndexSet, MappedReadRef, ReadRef, ResolvedVc, Tr use crate::{ chunk::{ ChunkItemBatchGroup, ChunkItemBatchWithAsyncModuleInfo, ChunkItemWithAsyncModuleInfo,- ChunkingConfig,+ ChunkingConfig, ChunkingContext, chunking::{ChunkItemOrBatchWithInfo, ComponentChunkItems, SplitContext, make_chunk}, }, module_graph::{ModuleGraph, chunk_group_info::ModuleToChunkGroups},@@ -43,7 +43,19 @@ pub async fn make_production_chunks( let module_chunk_groups = module_graph.chunk_group_info().module_chunk_groups(); let chunk_group_info = module_graph.chunk_group_info().await?; let heuristics = &chunk_group_info.chunking_heuristics;- let merged_modules = module_graph.merged_modules().await?;+ // Merged modules only exist when module merging is enabled; computing `merged_modules()`+ // otherwise would force the (potentially expensive) merge analysis — and the module graph+ // isn't even built to support it (it only collects `is_mergeable` when merging is on). So+ // only resolve it when merging is enabled; the lookup below is only reached for modules+ // that were merged away, which can't happen when merging is off.+ let merged_modules = if *(*split_context.chunking_context)+ .is_module_merging_enabled()+ .await?+ {+ Some(module_graph.merged_modules().await?)+ } else {+ None+ }; #[derive(Default)] struct GroupedChunkItems<'l> {@@ -77,8 +89,16 @@ pub async fn make_production_chunks( module_chunk_groups } else { // Merged modules don't have a chunk group in chunk_group_info, so- // lookup using the original module.+ // lookup using the original module. A module can only be missing here+ // if it was merged away, which only happens+ // when merging is enabled (so+ // `merged_modules` is `Some`). let original_module = merged_modules+ .as_ref()+ .context(+ "module has no chunk group and module merging is disabled, so \+ it cannot have been merged away",+ )? .get_original_module(module) .await? .context("every module should have a chunk group")?;turbopack/crates/turbopack-core/src/module_graph/async_module_info.rs6 + / 2 −
@@ -45,9 +45,13 @@ async fn compute_async_module_info_single( let graph = graph.read_strongly_consistent().await?; let self_async_modules = graph .iter_reachable_nodes()?- .map(async |node| {+ .map(async |(idx, node)| { Ok(match node {- SingleModuleGraphNode::Module(node) => node.is_self_async().await?.then_some(*node),+ SingleModuleGraphNode::Module(node) => graph+ .node_is_self_async(idx)+ .context("Module nodes always have is_self_async")?+ .await?+ .then_some(*node), SingleModuleGraphNode::VisitedModule { idx: _, module } => { // If a module is async in the parent then we need to mark reverse dependencies // async in this graph as well.turbopack/crates/turbopack-core/src/module_graph/binding_usage_info.rs5 + / 3 −
@@ -590,7 +590,7 @@ mod graph_tests { ident::AssetIdent, module::{Module, ModuleSideEffects}, module_graph::{- GraphEntries, ModuleGraph, SingleModuleGraph,+ GraphEntries, ModuleGraph, ModuleGraphOptions, SingleModuleGraph, chunk_group_info::{ChunkGroupEntry, EntryHeuristics}, }, reference::{ModuleReference, ModuleReferences, SingleChunkableModuleReference},@@ -733,8 +733,10 @@ mod graph_tests { }], vec![], )),- false,- true,+ ModuleGraphOptions {+ include_binding_usage: true,+ ..Default::default()+ }, ); let graph = ModuleGraph::from_graphs(vec![single_graph], None); let binding_usage = compute_binding_usage_info(turbopack/crates/turbopack-core/src/module_graph/merged_modules.rs20 + / 9 −
@@ -16,7 +16,7 @@ use crate::{ MergeableModulesExposed, }, module::Module,- module_graph::{GraphTraversalAction, ModuleGraph, RefData},+ module_graph::{GraphTraversalAction, ModuleGraph, RefData, SingleModuleGraphNode}, resolve::ExportUsage, }; @@ -227,15 +227,26 @@ pub async fn compute_merged_modules(module_graph: Vc<ModuleGraph>) -> Result<Vc< let inner_span = tracing::info_span!("collect mergeable modules"); let mergeable = module_graph- .iter_reachable_modules()?- .map(async |module| {- if let Some(mergeable) =- ResolvedVc::try_downcast::<Box<dyn MergeableModule>>(module)- && *mergeable.is_mergeable().await?- {- return Ok(Some(module));+ .iter_reachable_nodes()?+ .map(async |(idx, node)| {+ match node {+ SingleModuleGraphNode::Module(module) => {+ // Read the eagerly-resolved `is_mergeable` from the graph. The graph must+ // be built with `ModuleGraphOptions::include_mergeable` (we enable it+ // wherever the chunking context merges modules), so a missing value is a+ // misconfigured graph; `node_is_mergeable` bails rather than silently+ // treating everything as non-mergeable.+ // Tracked read: the producing task was resolved at graph construction, so+ // this is cheap while still depending on the value correctly.+ let is_mergeable = module_graph+ .node_is_mergeable(idx)?+ .context("Module nodes always have mergeability when collected")?;+ Ok(is_mergeable.await?.then_some(*module))+ }+ // VisitedModule nodes belong to a parent graph; their mergeability was already+ // accounted for there.+ SingleModuleGraphNode::VisitedModule { .. } => Ok(None), }- Ok(None) }) .try_flat_join() .instrument(inner_span)turbopack/crates/turbopack-core/src/module_graph/mod.rs499 + / 113 −
@@ -1,9 +1,10 @@ use std::{ collections::{BinaryHeap, VecDeque}, future::Future,+ hash::{Hash, Hasher}, iter::FusedIterator, ops::Deref,- sync::OnceLock,+ sync::{Mutex, OnceLock}, }; use anyhow::{Context, Result, bail};@@ -29,9 +30,10 @@ use turbo_tasks::{ use turbo_tasks_fs::FileSystemPath; use crate::{- chunk::{AsyncModuleInfo, ChunkingContext, ChunkingType, TracedMode},+ chunk::{AsyncModuleInfo, ChunkingContext, ChunkingType, MergeableModule, TracedMode},+ ident::AssetIdent, issue::{ImportTracer, ImportTraces, Issue, IssueExt, IssueSeverity, analyze::AnalyzeIssue},- module::Module,+ module::{Module, ModuleSideEffects}, module_graph::{ async_module_info::{AsyncModulesInfo, compute_async_module_info}, binding_usage_info::{BindingUsageInfo, UnusedReferenceEdges},@@ -269,6 +271,47 @@ impl GraphEntries { } } +/// Options controlling what data [`SingleModuleGraph`] collects while walking module references.+/// Passed by value to the graph constructors.+#[turbo_tasks::task_input]+#[derive(Debug, Copy, Clone, Default, PartialEq, Eq, Hash, Encode, Decode)]+pub struct ModuleGraphOptions {+ /// Eagerly resolve and store each module's `ident_string()` `Vc` in its graph node, so the+ /// module-id strategy reads it from the in-memory graph instead of fanning out a+ /// `module.ident_string()` (i.e. `ident().to_string()`) turbo-task launch per module. Only the+ /// whole-app/CLI-build graphs that run the global module-id strategy+ /// (`turbopack::global_module_ids::get_global_module_id_strategy`) need it; other graphs leave+ /// it off to avoid the per-module `to_string` resolve.+ ///+ /// (`ident()` itself is always collected — see [`ModuleGraphSnapshot::module_ident`].)+ pub include_ident_strings: bool,+ /// Eagerly resolve and store each module's `side_effects()` `Vc` in its graph node, so the+ /// side-effect-free aggregation reads it from the in-memory graph instead of fanning out a+ /// `module.side_effects()` turbo-task launch per module. Required by+ /// `compute_side_effect_free_module_info`: that pass `bail!`s on graphs built without this.+ pub include_side_effects: bool,+ /// Eagerly resolve and store each module's `MergeableModule::is_mergeable()` `Vc` in its graph+ /// node, so module merging (scope hoisting) reads it from the in-memory graph instead of+ /// fanning out a `module.is_mergeable()` turbo-task launch per module. Required by+ /// [`compute_merged_modules`]: that pass `bail!`s on graphs built without this. Should be+ /// enabled wherever the chunking context has module merging enabled+ /// ([`ChunkingContext::is_module_merging_enabled`]).+ ///+ /// When set, every `Module` node stores `Some`: the module's real `is_mergeable()` for types+ /// that implement [`MergeableModule`], or a resolved `false` for types that don't (so `None`+ /// unambiguously means "this graph wasn't built with `include_mergeable`").+ ///+ /// [`compute_merged_modules`]: crate::module_graph::merged_modules::compute_merged_modules+ /// [`MergeableModule`]: crate::chunk::MergeableModule+ /// [`ChunkingContext::is_module_merging_enabled`]:+ /// crate::chunk::ChunkingContext::is_module_merging_enabled+ pub include_mergeable: bool,+ /// Whether to walk `ChunkingType::Traced` references.+ pub include_traced: bool,+ /// Whether to read `ModuleReference::binding_usage()`.+ pub include_binding_usage: bool,+}+ #[turbo_tasks::value(cell = "new", eq = "manual")] #[derive(Default)] pub struct SingleModuleGraph {@@ -277,6 +320,25 @@ pub struct SingleModuleGraph { /// The number of modules in the graph (excluding VisitedModule nodes) pub number_of_modules: usize, + // Per-module data eagerly resolved at construction, stored as columns indexed by+ // `NodeIndex::index()`. `new_inner` adds every `Module` node before any `VisitedModule` node,+ // so the `Module` nodes are exactly `0..number_of_modules` and each column has one entry per+ // `Module` node; an index past the end is a `VisitedModule` node, whose data lives in the+ // graph that owns it. Consumers read these tracked: the producing tasks were already+ // resolved here, so the read is cheap while still depending on the value.+ /// Always collected: idents have many consumers (NFT, feature usage, module ids, import+ /// tracing).+ idents: Vec<ResolvedVc<AssetIdent>>,+ /// Always collected: `async_module_info()` runs during chunking for every graph.+ is_self_async: Vec<ResolvedVc<bool>>,+ /// `ident().to_string()`, collected with [`ModuleGraphOptions::include_ident_strings`].+ ident_strings: Option<Vec<ResolvedVc<RcStr>>>,+ /// Collected with [`ModuleGraphOptions::include_side_effects`].+ side_effects: Option<Vec<ResolvedVc<ModuleSideEffects>>>,+ /// Collected with [`ModuleGraphOptions::include_mergeable`]: the module's `is_mergeable()`+ /// for types that implement [`MergeableModule`], `false` for the others.+ is_mergeable: Option<Vec<ResolvedVc<bool>>>,+ // NodeIndex isn't necessarily stable (because of swap_remove), but we never remove nodes. // // HashMaps have nondeterministic order, but this map is only used for lookups (in@@ -301,6 +363,11 @@ impl Clone for SingleModuleGraph { Self { graph: self.graph.clone(), number_of_modules: self.number_of_modules,+ idents: self.idents.clone(),+ is_self_async: self.is_self_async.clone(),+ ident_strings: self.ident_strings.clone(),+ side_effects: self.side_effects.clone(),+ is_mergeable: self.is_mergeable.clone(), modules: self.modules.clone(), entries: self.entries.clone(), // Never carry derived state into a clone that might be modified before being stored.@@ -335,8 +402,7 @@ impl SingleModuleGraph { async fn new_inner( entries: &GraphEntries, visited_modules: &FxIndexMap<ResolvedVc<Box<dyn Module>>, GraphNodeIndex>,- include_traced: bool,- include_binding_usage: bool,+ options: ModuleGraphOptions, ) -> Result<Vc<Self>> { let emit_spans = tracing::enabled!(Level::INFO); let root_nodes = entries@@ -347,61 +413,92 @@ impl SingleModuleGraph { .try_join() .await?; + let node_data = Mutex::new(FxHashMap::default()); let children_nodes_iter = AdjacencyMap::new() .visit( root_nodes, SingleModuleGraphBuilder { visited_modules,+ node_data: &node_data, emit_spans,- include_traced,- include_binding_usage,+ options, }, ) .await .completed()?; let node_count = children_nodes_iter.len(); + let edges = children_nodes_iter+ .into_breadth_first_edges()+ .collect::<Vec<_>>();+ // The traversal resolved every module's node data in `edges()`.+ let mut node_data = node_data.into_inner().unwrap();+ let mut graph: DiGraph<SingleModuleGraphNode, RefData> = DiGraph::with_capacity( node_count, // From real world measurements each module has about 3-4 children // If it has more this would cause an additional allocation, but that's fine node_count * 4, ); - let mut number_of_modules = 0; let mut modules: FxHashMap<ResolvedVc<Box<dyn Module>>, NodeIndex> = FxHashMap::with_capacity_and_hasher(node_count, Default::default());+ let module_count = node_data.len();+ let mut idents = Vec::with_capacity(module_count);+ let mut is_self_async = Vec::with_capacity(module_count);+ let mut ident_strings = options+ .include_ident_strings+ .then(|| Vec::with_capacity(module_count));+ let mut side_effects = options+ .include_side_effects+ .then(|| Vec::with_capacity(module_count));+ let mut is_mergeable = options+ .include_mergeable+ .then(|| Vec::with_capacity(module_count)); { let _span = tracing::info_span!("build module graph").entered();- for (parent, current) in children_nodes_iter.into_breadth_first_edges() {- let (module, graph_node, count) = match current {- SingleModuleGraphBuilderNode::Module {- module,- is_traced: _,- ident: _,- } => (module, SingleModuleGraphNode::Module(module), 1),- SingleModuleGraphBuilderNode::VisitedModule { module, idx } => (+ // Synchronous: no `.await` here (the loop holds the `tracing` span guard and many+ // mutable locals across iterations; the per-module `node_data` was resolved during the+ // traversal).++ // Nodes are added in breadth-first order, but all `Module` nodes before any+ // `VisitedModule` node, so the per-module columns line up with `0..number_of_modules`.+ for (_, current) in &edges {+ if let &SingleModuleGraphBuilderNode::Module { module, .. } = current+ && !modules.contains_key(&module)+ {+ let data = node_data+ .remove(&module)+ .context("the traversal resolves node data for every module it reaches")?;+ let node_idx = graph.add_node(SingleModuleGraphNode::Module(module));+ debug_assert_eq!(+ node_idx.index(),+ idents.len(),+ "node indices must line up with the per-module columns"+ );+ modules.insert(module, node_idx);+ idents.push(data.ident);+ is_self_async.push(data.is_self_async);+ push_collected(&mut ident_strings, data.ident_string);+ push_collected(&mut side_effects, data.side_effects);+ push_collected(&mut is_mergeable, data.is_mergeable);+ }+ }+ for (_, current) in &edges {+ if let &SingleModuleGraphBuilderNode::VisitedModule { module, idx } = current+ && !modules.contains_key(&module)+ {+ modules.insert( module,- SingleModuleGraphNode::VisitedModule { idx, module },- 0,- ),- };-- // Find the current node, if it was already added- let current_idx = if let Some(current_idx) = modules.get(&module) {- *current_idx- } else {- let idx = graph.add_node(graph_node);- number_of_modules += count;- modules.insert(module, idx);- idx- };- // Add the edge+ graph.add_node(SingleModuleGraphNode::VisitedModule { idx, module }),+ );+ }+ }+ for (parent, current) in edges { if let Some((SingleModuleGraphBuilderNode::Module { module, .. }, ref_data)) = parent {- let parent_idx = *modules.get(&module).unwrap();- graph.add_edge(parent_idx, current_idx, ref_data);+ graph.add_edge(modules[&module], modules[¤t.module()], ref_data); } } }@@ -492,9 +589,33 @@ impl SingleModuleGraph { } } + let number_of_modules = idents.len();+ debug_assert!(+ graph+ .node_indices()+ .all(|idx| matches!(graph[idx], SingleModuleGraphNode::Module(_))+ == (idx.index() < number_of_modules)),+ "all Module nodes must come before all VisitedModule nodes"+ );+ debug_assert!(+ [+ ident_strings.as_ref().map(Vec::len),+ side_effects.as_ref().map(Vec::len),+ is_mergeable.as_ref().map(Vec::len),+ ]+ .into_iter()+ .flatten()+ .all(|len| len == number_of_modules),+ "collected columns must have one entry per Module node"+ ); let graph = SingleModuleGraph { graph: TracedDiGraph::new(graph), number_of_modules,+ idents,+ is_self_async,+ ident_strings,+ side_effects,+ is_mergeable, modules, entries: entries.clone(), entry_nodes: OnceLock::new(),@@ -511,11 +632,22 @@ impl SingleModuleGraph { /// Use iter_reachable_modules or one of the .traverse_* functions instead. pub fn iter_nodes(&self) -> impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + '_ { self.graph.node_weights().filter_map(|n| match n {- SingleModuleGraphNode::Module(node) => Some(*node),+ SingleModuleGraphNode::Module(module) => Some(*module), SingleModuleGraphNode::VisitedModule { .. } => None, }) } + /// Iterates the `Module` nodes with their eagerly-resolved idents. The `Module` nodes are the+ /// first `number_of_modules` nodes, in the same order as the `idents` column.+ pub fn iter_modules_with_idents(+ &self,+ ) -> impl Iterator<Item = (ResolvedVc<Box<dyn Module>>, ResolvedVc<AssetIdent>)> + '_ {+ self.graph+ .node_weights()+ .zip(&self.idents)+ .map(|(node, ident)| (node.module(), *ident))+ }+ fn entry_nodes(&self) -> &FxHashSet<NodeIndex> { self.entry_nodes.get_or_init(|| { self.entries@@ -684,12 +816,19 @@ impl ModuleGraphImportTracer { // Compute this mapping on demand since it might not always be needed. #[turbo_tasks::function] async fn path_to_modules(&self) -> Result<Vc<PathToModulesMap>> {- let path_and_modules = self+ let graph = self.graph.await?;+ let path_and_modules = graph .graph- .await?- .modules- .iter()- .map(async |(&module, _)| Ok((module.ident().await?.path.clone(), module)))+ .node_references()+ .map(async |(idx, n)| {+ let (module, ident) = match n {+ SingleModuleGraphNode::Module(module) => (*module, graph.idents[idx.index()]),+ SingleModuleGraphNode::VisitedModule { module, .. } => {+ (*module, module.ident().to_resolved().await?)+ }+ };+ Ok((ident.await?.path.clone(), module))+ }) .try_join() .await?; let mut map: FxHashMap<FileSystemPath, Vec<ResolvedVc<Box<dyn Module>>>> =@@ -1177,6 +1316,93 @@ impl ModuleGraphSnapshot { .collect() } + /// Returns a module's eagerly-resolved `AssetIdent` `Vc`. Idents are always collected, and+ /// `get_entry` only resolves a module to its `Module` node, so this always finds one. The+ /// returned `ResolvedVc` is read tracked by callers.+ pub fn module_ident(+ &self,+ module: ResolvedVc<Box<dyn Module>>,+ ) -> Result<ResolvedVc<AssetIdent>> {+ let idx = self.get_entry(module)?;+ self.node_ident(idx)+ .context("get_entry resolves a module to its Module node")+ }++ /// Returns a module's eagerly-resolved `ident_string()` `Vc`.+ ///+ /// Requires the graph to have been built with [`ModuleGraphOptions::include_ident_strings`];+ /// `bail!`s otherwise. Asking a graph for an ident string it never collected is a programming+ /// error — only the whole-app/CLI-build graphs that run the module-id strategy set the bit, and+ /// those are exactly the graphs this is called on.+ pub fn module_ident_string(+ &self,+ module: ResolvedVc<Box<dyn Module>>,+ ) -> Result<ResolvedVc<RcStr>> {+ let idx = self.get_entry(module)?;+ node_column(+ &self.get_graph(idx.graph_idx).ident_strings,+ idx,+ "module_ident_string() requires the module graph to be built with \+ `ModuleGraphOptions::include_ident_strings`",+ )?+ .context("get_entry resolves a module to its Module node")+ }++ /// The eagerly-resolved ident of a `Module` node, or `None` for a `VisitedModule` node (whose+ /// ident lives in the graph that owns the module).+ pub fn node_ident(&self, node: GraphNodeIndex) -> Option<ResolvedVc<AssetIdent>> {+ self.get_graph(node.graph_idx)+ .idents+ .get(node.node_idx.index())+ .copied()+ }++ /// The eagerly-resolved `is_self_async()` of a `Module` node, or `None` for a `VisitedModule`+ /// node.+ pub fn node_is_self_async(&self, node: GraphNodeIndex) -> Option<ResolvedVc<bool>> {+ self.get_graph(node.graph_idx)+ .is_self_async+ .get(node.node_idx.index())+ .copied()+ }++ /// The eagerly-resolved `side_effects()` of a `Module` node, or `None` for a `VisitedModule`+ /// node. `bail!`s unless the graph was built with+ /// [`ModuleGraphOptions::include_side_effects`].+ pub fn node_side_effects(+ &self,+ node: GraphNodeIndex,+ ) -> Result<Option<ResolvedVc<ModuleSideEffects>>> {+ node_column(+ &self.get_graph(node.graph_idx).side_effects,+ node,+ "side effects require the module graph to be built with \+ `ModuleGraphOptions::include_side_effects`",+ )+ }++ /// The eagerly-resolved `is_mergeable()` of a `Module` node (`false` for modules that don't+ /// implement [`MergeableModule`]), or `None` for a `VisitedModule` node. `bail!`s unless the+ /// graph was built with [`ModuleGraphOptions::include_mergeable`].+ pub fn node_is_mergeable(&self, node: GraphNodeIndex) -> Result<Option<ResolvedVc<bool>>> {+ node_column(+ &self.get_graph(node.graph_idx).is_mergeable,+ node,+ "mergeability requires the module graph to be built with \+ `ModuleGraphOptions::include_mergeable`",+ )+ }++ /// Iterates every `Module` node of every graph with its eagerly-resolved ident. Unlike+ /// [`Self::module_ident`], this needs no lookup per module.+ pub fn iter_modules_with_idents(+ &self,+ ) -> impl Iterator<Item = (ResolvedVc<Box<dyn Module>>, ResolvedVc<AssetIdent>)> + '_ {+ self.graphs+ .iter()+ .flat_map(|g| g.iter_modules_with_idents())+ }+ /// Traverses all reachable nodes exactly once and calls the visitor. /// /// * `entries` - The entry modules to start the traversal from@@ -1770,7 +1996,7 @@ impl ModuleGraphSnapshot { pub fn iter_reachable_modules( &self, ) -> Result<impl Iterator<Item = ResolvedVc<Box<dyn Module>>>> {- Ok(self.iter_reachable_nodes()?.filter_map(|n| match n {+ Ok(self.iter_reachable_nodes()?.filter_map(|(_, n)| match n { SingleModuleGraphNode::Module(m) => Some(*m), SingleModuleGraphNode::VisitedModule { .. } => None, }))@@ -1780,7 +2006,7 @@ impl ModuleGraphSnapshot { /// This includes VisitedModule nodes (which means that some modules are returned twice). pub fn iter_reachable_nodes<'a>( &'a self,- ) -> Result<impl Iterator<Item = &'a SingleModuleGraphNode> + 'a> {+ ) -> Result<impl Iterator<Item = (GraphNodeIndex, &'a SingleModuleGraphNode)> + 'a> { ModuleGraphSnapshotNodeIterator::new(self) } }@@ -1808,7 +2034,7 @@ impl<'a> ModuleGraphSnapshotNodeIterator<'a> { } } impl<'a> Iterator for ModuleGraphSnapshotNodeIterator<'a> {- type Item = &'a SingleModuleGraphNode;+ type Item = (GraphNodeIndex, &'a SingleModuleGraphNode); fn next(&mut self) -> Option<Self::Item> { while let Some(node_idx) = self.visit_queue.pop_front() {@@ -1828,7 +2054,7 @@ impl<'a> Iterator for ModuleGraphSnapshotNodeIterator<'a> { .filter(|succ| !self.visited.contains(succ)), ); }- return Some(node_weight);+ return Some((node_idx, node_weight)); } } None@@ -1841,45 +2067,34 @@ impl SingleModuleGraph { #[turbo_tasks::function(operation)] pub async fn new_with_entry( entry: ChunkGroupEntry,- include_traced: bool,- include_binding_usage: bool,+ options: ModuleGraphOptions, ) -> Result<Vc<Self>> { SingleModuleGraph::new_inner( &GraphEntries::from_chunk_groups(vec![entry]), &Default::default(),- include_traced,- include_binding_usage,+ options, ) .await } #[turbo_tasks::function(operation)] pub async fn new_with_entries( entries: ResolvedVc<GraphEntries>,- include_traced: bool,- include_binding_usage: bool,+ options: ModuleGraphOptions, ) -> Result<Vc<Self>> {- SingleModuleGraph::new_inner(- &*entries.await?,- &Default::default(),- include_traced,- include_binding_usage,- )- .await+ SingleModuleGraph::new_inner(&*entries.await?, &Default::default(), options).await } #[turbo_tasks::function(operation)] pub async fn new_with_entries_visited( entries: ResolvedVc<GraphEntries>, visited_modules: OperationVc<VisitedModules>,- include_traced: bool,- include_binding_usage: bool,+ options: ModuleGraphOptions, ) -> Result<Vc<Self>> { SingleModuleGraph::new_inner( &*entries.await?, &visited_modules.connect().await?.modules,- include_traced,- include_binding_usage,+ options, ) .await }@@ -1889,16 +2104,10 @@ impl SingleModuleGraph { // This must not be a Vc<Vec<_>> to ensure layout segment optimization hits the cache entries: GraphEntries, visited_modules: OperationVc<VisitedModules>,- include_traced: bool,- include_binding_usage: bool,+ options: ModuleGraphOptions, ) -> Result<Vc<Self>> {- SingleModuleGraph::new_inner(- &entries,- &visited_modules.connect().await?.modules,- include_traced,- include_binding_usage,- )- .await+ SingleModuleGraph::new_inner(&entries, &visited_modules.connect().await?.modules, options)+ .await } #[turbo_tasks::function]@@ -1921,6 +2130,18 @@ pub enum SingleModuleGraphNode { module: ResolvedVc<Box<dyn Module>>, }, }+#[cfg(test)]+mod size_test {+ use crate::module_graph::SingleModuleGraphNode;++ /// Guards the per-module node size — there is one of these per module in the graph, so growth+ /// here is multiplied across the whole app. Per-module data belongs in the+ /// `SingleModuleGraph` columns instead.+ #[test]+ pub fn test_size() {+ assert_eq!(16, size_of::<SingleModuleGraphNode>());+ }+} impl SingleModuleGraphNode { pub fn module(&self) -> ResolvedVc<Box<dyn Module>> {@@ -1929,6 +2150,7 @@ impl SingleModuleGraphNode { SingleModuleGraphNode::VisitedModule { module, .. } => *module, } }+ pub fn target_idx(&self, direction: Direction) -> Option<GraphNodeIndex> { match self { SingleModuleGraphNode::VisitedModule { idx, .. } => match direction {@@ -1952,13 +2174,17 @@ pub enum GraphTraversalAction { // These nodes are created while walking the Turbopack modules references, and are used to then // afterwards build the SingleModuleGraph.-#[derive(Clone, Hash, PartialEq, Eq)]+#[derive(Clone)] enum SingleModuleGraphBuilderNode { /// A regular module Module { module: ResolvedVc<Box<dyn Module>>,- /// module.ident().to_string(), eagerly computed for tracing, otherwise None- ident: Option<ReadRef<RcStr>>,+ /// The module's `ident().to_string()`, read (untracked) only when `emit_spans` so [`span`]+ /// can use it as the span name synchronously. Transient (never reaches the graph node) and+ /// excluded from `Hash`/`Eq`.+ ///+ /// [`span`]: SingleModuleGraphBuilder::span+ span_name: Option<ReadRef<RcStr>>, /// whether this module is a tracing context is_traced: bool, },@@ -1969,17 +2195,168 @@ enum SingleModuleGraphBuilderNode { }, } +// `Hash`/`Eq` are implemented manually over the identity-determining fields only (`module` ++// `is_traced` / `idx`), deliberately excluding `ident`. This node is the `AdjacencyMap` visit dedup+// key; `ident` is fully determined by `module`, so including it would only waste cycles hashing a+// full `AssetIdent` per insert.+impl PartialEq for SingleModuleGraphBuilderNode {+ fn eq(&self, other: &Self) -> bool {+ match (self, other) {+ (+ SingleModuleGraphBuilderNode::Module {+ module: l_module,+ is_traced: l_is_traced,+ ..+ },+ SingleModuleGraphBuilderNode::Module {+ module: r_module,+ is_traced: r_is_traced,+ ..+ },+ ) => l_module == r_module && l_is_traced == r_is_traced,+ (+ SingleModuleGraphBuilderNode::VisitedModule {+ module: l_module,+ idx: l_idx,+ },+ SingleModuleGraphBuilderNode::VisitedModule {+ module: r_module,+ idx: r_idx,+ },+ ) => l_module == r_module && l_idx == r_idx,+ _ => false,+ }+ }+}+impl Eq for SingleModuleGraphBuilderNode {}+impl Hash for SingleModuleGraphBuilderNode {+ fn hash<H: Hasher>(&self, state: &mut H) {+ core::mem::discriminant(self).hash(state);+ match self {+ SingleModuleGraphBuilderNode::Module {+ module, is_traced, ..+ } => {+ module.hash(state);+ is_traced.hash(state);+ }+ SingleModuleGraphBuilderNode::VisitedModule { module, idx } => {+ module.hash(state);+ idx.hash(state);+ }+ }+ }+}++/// Reads a node's entry from a per-module column of [`SingleModuleGraph`]: `None` for a+/// `VisitedModule` node, which sits past the end of every column. `bail!`s with `missing` when the+/// graph was built without collecting the column.+fn node_column<T: Copy>(+ column: &Option<Vec<T>>,+ node: GraphNodeIndex,+ missing: &'static str,+) -> Result<Option<T>> {+ Ok(column+ .as_ref()+ .context(missing)?+ .get(node.node_idx.index())+ .copied())+}++/// Appends to a per-module column of [`SingleModuleGraph`] if the graph collects it.+fn push_collected<T>(column: &mut Option<Vec<T>>, value: Option<T>) {+ if let (Some(column), Some(value)) = (column, value) {+ column.push(value);+ }+}++/// The eagerly-resolved per-module data, stored in the per-module columns of [`SingleModuleGraph`].+struct ModuleGraphNodeData {+ ident: ResolvedVc<AssetIdent>,+ ident_string: Option<ResolvedVc<RcStr>>,+ is_self_async: ResolvedVc<bool>,+ side_effects: Option<ResolvedVc<ModuleSideEffects>>,+ is_mergeable: Option<ResolvedVc<bool>>,+}++/// Resolves a module's graph-node data. Each `to_resolved()` resolves the pointer chain (executing+/// the producing task, without reading the cell content).+///+/// This is deliberately a plain `async fn` rather than a `#[turbo_tasks::function]`: the producing+/// tasks become children of the graph-construction task instead of a new task per module. On a+/// large app the extra edges added little to the construction task's batched connect, while a task+/// per module costs a creation, an execution and a connect of its own.+///+/// All calls are issued before the first `.await`: calling a turbo-tasks function schedules its+/// task, so they execute concurrently rather than one after another.+async fn module_graph_node_data(+ module: ResolvedVc<Box<dyn Module>>,+ ModuleGraphOptions {+ include_ident_strings,+ include_side_effects,+ include_mergeable,+ ..+ }: ModuleGraphOptions,+) -> Result<ModuleGraphNodeData> {+ // Always collected: idents have many consumers (NFT, feature usage, module-id strategy, import+ // tracer).+ let ident = module.ident();+ // Always collected: `async_module_info()` runs during chunking for every graph. Cheap to+ // resolve (trait default `Vc::cell(false)`; ecma reads already-cached `references()`).+ let is_self_async = module.is_self_async();+ let side_effects = include_side_effects.then(|| module.side_effects());+ // When collecting mergeability, store `Some` for *every* module so that `None` at the node+ // level unambiguously means "graph not built with `include_mergeable`". Types that implement+ // `MergeableModule` get their real `is_mergeable()`; others get a resolved `false` (the+ // interned `false` cell, so this is cheap and deduped).+ let is_mergeable = include_mergeable.then(|| {+ match ResolvedVc::try_downcast::<Box<dyn MergeableModule>>(module) {+ Some(mergeable) => mergeable.is_mergeable(),+ None => Vc::<bool>::default(),+ }+ });++ let ident = ident.to_resolved().await?;+ Ok(ModuleGraphNodeData {+ ident,+ // Only the module-id strategy needs the stringified ident, and only on the graphs that set+ // this bit. `ident_string()` is real work (recursive stringify), so it's gated.+ ident_string: if include_ident_strings {+ Some(ident.to_string().to_resolved().await?)+ } else {+ None+ },+ is_self_async: is_self_async.to_resolved().await?,+ side_effects: match side_effects {+ Some(side_effects) => Some(side_effects.to_resolved().await?),+ None => None,+ },+ is_mergeable: match is_mergeable {+ Some(is_mergeable) => Some(is_mergeable.to_resolved().await?),+ None => None,+ },+ })+}+ impl SingleModuleGraphBuilderNode {+ fn module(&self) -> ResolvedVc<Box<dyn Module>> {+ match self {+ SingleModuleGraphBuilderNode::Module { module, .. } => *module,+ SingleModuleGraphBuilderNode::VisitedModule { module, .. } => *module,+ }+ }+ async fn new_module( emit_spans: bool, module: ResolvedVc<Box<dyn Module>>, is_traced: bool, ) -> Result<Self> { Ok(Self::Module { module,- ident: if emit_spans {- // INVALIDATION: we don't need to invalidate when the span name changes- Some(module.ident_string().untracked().await?)+ // The span name needs the ident string synchronously. Read it (untracked — its value is+ // non-load-bearing for correctness) only when spans are enabled. Stays on the builder+ // node; never reaches the graph node.+ span_name: if emit_spans {+ Some(module.ident().to_string().untracked().await?) } else { None },@@ -1994,13 +2371,13 @@ impl SingleModuleGraphBuilderNode { struct SingleModuleGraphBuilder<'a> { visited_modules: &'a FxIndexMap<ResolvedVc<Box<dyn Module>>, GraphNodeIndex>, - emit_spans: bool,+ /// Filled by `edges()`, which the traversal calls once per unique module, so each module's+ /// node data is resolved exactly once and concurrently with the traversal.+ node_data: &'a Mutex<FxHashMap<ResolvedVc<Box<dyn Module>>, ModuleGraphNodeData>>, - /// Whether to walk ChunkingType::Traced references- include_traced: bool,+ emit_spans: bool, - /// Whether to read ModuleReference::binding_usage()- include_binding_usage: bool,+ options: ModuleGraphOptions, } impl Visit<SingleModuleGraphBuilderNode, RefData> for SingleModuleGraphBuilder<'_> { type EdgesIntoIter = Vec<(SingleModuleGraphBuilderNode, RefData)>;@@ -2028,16 +2405,24 @@ impl Visit<SingleModuleGraphBuilderNode, RefData> for SingleModuleGraphBuilder<' unreachable!() }; let visited_modules = self.visited_modules;+ let node_data = self.node_data; let emit_spans = self.emit_spans;- let include_traced = self.include_traced;- let include_binding_usage = self.include_binding_usage;+ let options = self.options;+ let ModuleGraphOptions {+ include_traced,+ include_binding_usage,+ ..+ } = options; async move {+ // Issued before resolving the node data so both run concurrently. let refs_cell = if !is_traced { primary_chunkable_referenced_modules(*module, include_traced, include_binding_usage) } else { // Currently we don't care about the binding usage of traced references referenced_modules_and_affecting_sources(*module, false) };+ let data = module_graph_node_data(module, options).await?;+ node_data.lock().unwrap().insert(module, data); let refs = match refs_cell.await { Ok(refs) => refs, Err(e) => {@@ -2108,14 +2493,22 @@ impl Visit<SingleModuleGraphBuilderNode, RefData> for SingleModuleGraphBuilder<' let mut span = match node { SingleModuleGraphBuilderNode::Module {- ident: Some(ident), ..+ span_name: Some(span_name),+ ..+ } => {+ // Use the precomputed `ident().to_string()` (read untracked in `new_module`).+ // `span_name` is populated whenever `emit_spans`, so this arm is taken for every+ // `Module` node here.+ tracing::info_span!("module", name = display(span_name))+ }+ SingleModuleGraphBuilderNode::Module {+ span_name: None, .. } => {- tracing::info_span!("module", name = display(ident))+ tracing::info_span!("module") } SingleModuleGraphBuilderNode::VisitedModule { .. } => { tracing::info_span!("visited module") }- _ => unreachable!(), }; if let Some(edge) = edge {@@ -2224,8 +2617,7 @@ pub mod tests { }], vec![], )),- false,- false,+ ModuleGraphOptions::default(), ) .connect() .to_resolved()@@ -2666,8 +3058,7 @@ pub mod tests { heuristics: EntryHeuristics::default(), }]) .resolved_cell(),- false,- false,+ ModuleGraphOptions::default(), ); let module_graph = ModuleGraph::from_graphs(@@ -2680,8 +3071,7 @@ pub mod tests { }]) .resolved_cell(), VisitedModules::from_graph(parent_graph),- false,- false,+ ModuleGraphOptions::default(), ), ], None,@@ -2848,8 +3238,10 @@ pub mod tests { heuristics: EntryHeuristics::default(), }]) .resolved_cell(),- true,- false,+ ModuleGraphOptions {+ include_traced: true,+ ..Default::default()+ }, ); let module_graph = ModuleGraph::from_graphs(@@ -2862,8 +3254,10 @@ pub mod tests { }]) .resolved_cell(), VisitedModules::from_graph(parent_graph),- true,- false,+ ModuleGraphOptions {+ include_traced: true,+ ..Default::default()+ }, ), ], None,@@ -2875,7 +3269,7 @@ pub mod tests { iter_nodes: module_graph .await? .iter_reachable_nodes()?- .map(async |node| {+ .map(async |(_, node)| { Ok(match node { SingleModuleGraphNode::Module(module) => { module.ident_string().owned().await?@@ -2901,7 +3295,7 @@ pub mod tests { .connect() .await? .iter_reachable_nodes()?- .map(async |node| {+ .map(async |(_, node)| { Ok(match node { SingleModuleGraphNode::Module(module) => { module.ident_string().owned().await?@@ -3343,8 +3737,7 @@ pub mod tests { }], vec![], )),- false,- false,+ ModuleGraphOptions::default(), ); // Create a simple name mapping to make analyzing the visitors easier.@@ -3566,8 +3959,7 @@ pub mod tests { heuristics: EntryHeuristics::default(), }]) .resolved_cell(),- false,- false,+ ModuleGraphOptions::default(), ); let module_graph = ModuleGraph::from_graphs( vec![@@ -3579,8 +3971,7 @@ pub mod tests { }]) .resolved_cell(), VisitedModules::from_graph(parent_graph),- false,- false,+ ModuleGraphOptions::default(), ), ], None,@@ -3702,8 +4093,7 @@ pub mod tests { heuristics: EntryHeuristics::default(), }]) .resolved_cell(),- false,- false,+ ModuleGraphOptions::default(), ); let graph = ModuleGraph::from_graphs(vec![graph], None) .connect()@@ -3774,8 +4164,7 @@ pub mod tests { heuristics: EntryHeuristics::default(), }]) .resolved_cell(),- false,- false,+ ModuleGraphOptions::default(), ); let layers = ModuleGraph::from_graphs(vec![graph], None) .connect()@@ -3883,8 +4272,7 @@ pub mod tests { } let graph = SingleModuleGraph::new_with_entries( GraphEntries::resolved_cell(GraphEntries::new(entries, vec![])),- false,- false,+ ModuleGraphOptions::default(), ); let chunk_group_info = ModuleGraph::from_graphs(vec![graph], None) .connect()@@ -4016,8 +4404,7 @@ pub mod tests { ]; let graph = SingleModuleGraph::new_with_entries( GraphEntries::resolved_cell(GraphEntries::new(entries, vec![])),- false,- false,+ ModuleGraphOptions::default(), ); let module_to_name = graph .connect()@@ -4216,8 +4603,7 @@ pub mod tests { } let graph = SingleModuleGraph::new_with_entries( GraphEntries::resolved_cell(GraphEntries::new(graph_entries, vec![])),- false,- false,+ ModuleGraphOptions::default(), ); let modules = graph .connect()turbopack/crates/turbopack-core/src/module_graph/side_effect_module_info.rs12 + / 6 −
@@ -1,4 +1,4 @@-use anyhow::Result;+use anyhow::{Context, Result}; use rustc_hash::{FxHashMap, FxHashSet}; use turbo_tasks::{JoinIterExt, ResolvedVc, Vc}; @@ -44,13 +44,19 @@ async fn compute_side_effect_free_module_info_single( let graph = graph.await?; let module_side_effects = graph .iter_reachable_nodes()?- .map(async |node| {+ .map(async |(idx, node)| { Ok(match node { SingleModuleGraphNode::Module(module) => {- // This turbo task always has a cache hit since it is called when building the- // module graph. we could consider moving this information- // into to the module graph, but then changes would invalidate the whole graph.- (*module, *module.side_effects().await?)+ // Read the eagerly-resolved `side_effects` from the graph. This pass only runs+ // when removing unused imports, and we build those graphs with+ // `ModuleGraphOptions::include_side_effects`, so the value is always present;+ // `node_side_effects` bails otherwise. Tracked read: the producing task was+ // resolved at graph construction, so this is cheap while still depending on+ // the value correctly.+ let side_effects = graph+ .node_side_effects(idx)?+ .context("Module nodes always have side effects when collected")?;+ (*module, *side_effects.await?) } SingleModuleGraphNode::VisitedModule { idx: _, module } => ( *module,turbopack/crates/turbopack-ecmascript/src/lib.rs1 + / 1 −
@@ -934,7 +934,7 @@ impl Module for EcmascriptModuleAsset { // Check package.json first, so that we can skip parsing the module if it's marked that way. // We need to respect package.json configuration over any static analysis we might do. Ok((match *get_side_effect_free_declaration(- self.ident().await?.path.clone(),+ this.source.ident().await?.path.clone(), this.side_effect_free_packages.map(|g| *g), ) .await?turbopack/crates/turbopack-ecmascript/src/module_fragments/part/module.rs1 + / 1 −
@@ -326,7 +326,7 @@ impl Module for EcmascriptModulePartAsset { } #[turbo_tasks::function]- async fn side_effects(&self) -> Vc<ModuleSideEffects> {+ fn side_effects(&self) -> Vc<ModuleSideEffects> { match self.part { ModulePart::Exports | ModulePart::Export(_) | ModulePart::PartialExport { .. } => { ModuleSideEffects::SideEffectFree.cell()turbopack/crates/turbopack-node/src/transforms/postcss.rs2 + / 3 −
@@ -16,7 +16,7 @@ use turbopack_core::{ context::{AssetContext, ProcessResult}, file_source::FileSource, ident::AssetIdent,- module_graph::{ModuleGraph, SingleModuleGraph},+ module_graph::{ModuleGraph, ModuleGraphOptions, SingleModuleGraph}, reference_type::{EntryReferenceSubType, InnerAssets, ReferenceType}, resolve::{FindContextFileResult, find_context_file_or_package_key, options::ImportMapping}, source::Source,@@ -531,8 +531,7 @@ impl PostCssTransformedAsset { let module_graph = ModuleGraph::from_graphs( vec![SingleModuleGraph::new_with_entries( entries.graph_entries().to_resolved().await?,- false,- false,+ ModuleGraphOptions::default(), )], None, )turbopack/crates/turbopack-node/src/transforms/webpack.rs3 + / 5 −
@@ -33,7 +33,7 @@ use turbopack_core::{ issue::{Issue, IssueExt, IssueSeverity, IssueSource, IssueStage, StyledString}, module::Module, module_graph::{- ModuleGraph, SingleModuleGraph,+ ModuleGraph, ModuleGraphOptions, SingleModuleGraph, chunk_group_info::{ChunkGroupEntry, ChunkGroupKey, EntryHeuristics}, }, output::{ExpandOutputAssetsInput, OutputAsset, OutputAssets, expand_output_assets},@@ -491,8 +491,7 @@ impl WebpackLoadersProcessedAsset { let module_graph = ModuleGraph::from_graphs( vec![SingleModuleGraph::new_with_entries( entries.graph_entries().to_resolved().await?,- false,- false,+ ModuleGraphOptions::default(), )], None, )@@ -1002,8 +1001,7 @@ impl EvaluateContext for WebpackLoaderContext { modules: vec![module], heuristics: EntryHeuristics::default(), },- false,- false,+ ModuleGraphOptions::default(), ); let import_module_graph = ModuleGraph::from_graphs(vec![single_graph], None) .connect()turbopack/crates/turbopack-tests/tests/execution.rs15 + / 3 −
@@ -43,7 +43,8 @@ use turbopack_core::{ ident::Layer, issue::{CollectibleIssuesExt, IssueFilter}, module_graph::{- ModuleGraph, SingleModuleGraph, binding_usage_info::compute_binding_usage_info,+ ModuleGraph, ModuleGraphOptions, SingleModuleGraph,+ binding_usage_info::compute_binding_usage_info, }, reference_type::{InnerAssets, ReferenceType}, resolve::{@@ -589,8 +590,19 @@ async fn run_test_operation(prepared_test: ResolvedVc<PreparedTest>) -> Result<V let single_graph = SingleModuleGraph::new_with_entries( entries.graph_entries().to_resolved().await?,- false,- true,+ ModuleGraphOptions {+ include_binding_usage: true,+ // `compute_side_effect_free_module_info` (run below via `compute_binding_usage_info`+ // when removing unused imports) reads `side_effects` from the graph node.+ include_side_effects: options.remove_unused_imports,+ // Module merging (driven by `scope_hoisting` in the chunking context below) reads+ // `is_mergeable` from the graph node and bails if absent.+ include_mergeable: options.scope_hoisting,+ // `get_global_module_id_strategy` (used for global/numeric ids below) reads the+ // stringified ident from the graph node.+ include_ident_strings: options.global_module_ids || options.numeric_module_ids,+ ..Default::default()+ }, ); let mut module_graph = ModuleGraph::from_graphs(vec![single_graph], None); turbopack/crates/turbopack-tests/tests/snapshot.rs11 + / 3 −
@@ -47,7 +47,7 @@ use turbopack_core::{ issue::{CollectibleIssuesExt, IssueFilter, IssueSeverity}, module::Module, module_graph::{- GraphEntries, ModuleGraph, SingleModuleGraph,+ GraphEntries, ModuleGraph, ModuleGraphOptions, SingleModuleGraph, binding_usage_info::compute_binding_usage_info, chunk_group_info::{ChunkGroupEntry, ChunkGroupKey, EntryHeuristics}, },@@ -500,8 +500,16 @@ async fn run_test_operation(resource: RcStr) -> Result<Vc<FileSystemPath>> { heuristics: EntryHeuristics::default(), }]) .resolved_cell(),- false,- true,+ ModuleGraphOptions {+ include_binding_usage: true,+ // `compute_side_effect_free_module_info` (run below via `compute_binding_usage_info`+ // when removing unused imports) reads `side_effects` from the graph node.+ include_side_effects: options.remove_unused_imports,+ // Module merging (driven by `scope_hoisting` in the chunking context below) reads+ // `is_mergeable` from the graph node and bails if absent.+ include_mergeable: options.scope_hoisting,+ ..Default::default()+ }, ); let mut module_graph = ModuleGraph::from_graphs(vec![single_graph], None); turbopack/crates/turbopack/src/global_module_ids.rs24 + / 6 −
@@ -11,7 +11,6 @@ use turbopack_core::{ chunk_id_strategy::{ModuleIdFallback, ModuleIdStrategy}, }, ident::AssetIdent,- module::Module, module_graph::{ModuleGraph, RefData}, }; use turbopack_ecmascript::async_chunk::module::AsyncLoaderModule;@@ -25,7 +24,11 @@ pub async fn get_global_module_id_strategy( let module_graph = module_graph.await?; // All modules in the graph and additionally, all the modules that are inserted by chunking- // (i.e. async loaders)+ // (i.e. async loaders). For graph modules we read the eagerly-resolved ident AND its+ // pre-resolved `ident_string` from the node, so we don't fan out a `to_string()` task per+ // module here (the production/CLI-build graphs that run this are built with+ // `include_ident_strings`). Async-loader idents are synthesized and not in the graph, so+ // their strings are still computed via `to_string()`. let mut modules = FxHashSet::default(); let mut async_idents = vec![]; module_graph.traverse_edges_unordered(|parent, current| {@@ -45,19 +48,34 @@ pub async fn get_global_module_id_strategy( Ok(()) })?; - let mut module_id_map = modules+ // Resolve graph-module idents to `(ident, (ident_str, hash))`, reading the pre-resolved+ // `ident_string` from the node (the graph is built with `include_ident_strings`, else+ // `module_ident_string` bails — no silent per-module `to_string()` fallback).+ let graph_entries = modules+ .into_iter()+ .map(async |module| {+ let ident = module_graph.module_ident(module)?;+ let ident_str = module_graph.module_ident_string(module)?.await?;+ let hash = hash_xxh3_hash64(&ident_str);+ Ok((ident, (ident_str, hash)))+ })+ .join()+ .await;+ // Async-loader idents are synthesized (not in the graph); compute their strings directly.+ let async_entries = async_idents .into_iter()- .map(|m| m.ident())- .chain(async_idents.into_iter()) .map(async |ident| { let ident = ident.to_resolved().await?; let ident_str = ident.to_string().await?; let hash = hash_xxh3_hash64(&ident_str); Ok((ident, (ident_str, hash))) }) .join()- .await+ .await;++ let mut module_id_map = graph_entries .into_iter()+ .chain(async_entries) .collect::<Result<FxHashMap<_, _>>>()?; finalize_module_ids(&mut module_id_map);