vercel/next.js · #100272
Support fs.stat in Turbopack webpack loaders
docs/01-app/03-api-reference/05-config/01-next-config-js/turbopack.mdx1 + / 1 −
@@ -76,7 +76,7 @@ Turbopack uses the [`loader-runner`](https://github.com/webpack/loader-runner) l **File system and output:** -- [`fs`](https://webpack.js.org/api/loaders/#thisfs) - Partial support: only `fs.readFile` is currently implemented.+- [`fs`](https://webpack.js.org/api/loaders/#thisfs) - Partial support: asynchronous `fs.readFile` and `fs.stat` are implemented. Other methods, including synchronous filesystem methods, are not supported. - [`emitFile`](https://webpack.js.org/api/loaders/#thisemitfile) - No support **Utilities:**test/e2e/app-dir/webpack-loader-fs/app/input.statadded1 + / 0 −
@@ -0,0 +1 @@+stat fixturetest/e2e/app-dir/webpack-loader-fs/app/page.js2 + / 0 −
@@ -1,9 +1,11 @@ import data from './test.txt'+import stat from './input.stat' export default function Page() { return ( <div> <p id="test">{data}</p>+ <pre id="stat">{JSON.stringify(stat)}</pre> </div> ) }test/e2e/app-dir/webpack-loader-fs/app/parent/input.parent-statadded1 + / 0 −
@@ -0,0 +1 @@+stat fixturetest/e2e/app-dir/webpack-loader-fs/app/parent/page.jsadded5 + / 0 −
@@ -0,0 +1,5 @@+import stat from './input.parent-stat'++export default function Page() {+ return <pre id="parent-stat">{JSON.stringify(stat)}</pre>+}test/e2e/app-dir/webpack-loader-fs/app/path/to/stat-parent-target.binadded1 + / 0 −
@@ -0,0 +1 @@+hellotest/e2e/app-dir/webpack-loader-fs/app/stat-parent-target.binadded1 + / 0 −
@@ -0,0 +1 @@+hellotest/e2e/app-dir/webpack-loader-fs/app/stat-target.binadded1 + / 0 −
@@ -0,0 +1 @@+hellotest/e2e/app-dir/webpack-loader-fs/next.config.js13 + / 0 −
@@ -4,13 +4,26 @@ const nextConfig = { turbopack: { rules: {+ '*.parent-stat': {+ loaders: ['./stat-parent-loader.js'],+ as: '*.js',+ },+ '*.stat': {+ loaders: ['./stat-loader.js'],+ as: '*.js',+ }, '*.txt': { loaders: ['./test-file-loader.js'], as: '*.js', }, }, }, webpack(config) {+ config.module.rules.push({+ test: /\.parent-stat$/,+ use: './stat-parent-loader.js',+ })+ config.module.rules.push({ test: /\.stat$/, use: './stat-loader.js' }) config.module.rules.push({ test: /\.txt/, use: './test-file-loader.js',test/e2e/app-dir/webpack-loader-fs/stat-loader.jsadded81 + / 0 −
@@ -0,0 +1,81 @@+const fs = require('fs')+const path = require('path')+const { pathToFileURL } = require('url')++module.exports = async function () {+ const dir = path.dirname(this.resourcePath)+ const target = path.join(dir, 'stat-target.bin')+ const stat = (p, options) =>+ new Promise((resolve, reject) => {+ const callback = (err, stats) => (err ? reject(err) : resolve(stats))+ if (options === undefined) this.fs.stat(p, callback)+ else this.fs.stat(p, options, callback)+ })+ const error = async (p) => {+ try {+ await stat(p)+ return null+ } catch (err) {+ let nodeError+ try {+ fs.statSync(p)+ } catch (error) {+ nodeError = error+ }+ return {+ code: err.code,+ syscall: err.syscall,+ correctPath: err.path === p,+ matchesNode:+ err.code === nodeError.code && err.syscall === nodeError.syscall,+ }+ }+ }++ const file = await stat(target)+ const bigint = await stat(target, { bigint: true })+ const buffer = await stat(Buffer.from(target))+ const url = await stat(pathToFileURL(target))+ const directory = await stat(dir)+ const root = await stat(this.rootContext)+ const linkedDirectory = await stat(+ path.join(dir, 'path/to/symlink/inner/path')+ )+ const linkedFile = await stat(+ path.join(dir, 'path/to/symlink/inner/path/one.txt')+ )+ const optionalPath = path.join(dir, 'optional.bin')+ let optional+ try {+ const stats = await stat(optionalPath)+ optional = stats.isDirectory() ? 'directory' : 'file'+ } catch (err) {+ if (err.code !== 'ENOENT') throw err+ optional = 'missing'+ }++ return `module.exports = ${JSON.stringify({+ file: file.isFile(),+ directory: directory.isDirectory(),+ root: root.isDirectory(),+ symlink: linkedDirectory.isSymbolicLink(),+ linkedDirectory: linkedDirectory.isDirectory(),+ linkedSize: linkedFile.size,+ size: file.size,+ mtimeMs: file.mtimeMs,+ date: file.mtime instanceof Date,+ bigint: typeof bigint.size,+ bigintSize: bigint.size.toString(),+ bufferSize: buffer.size,+ urlSize: url.size,+ mode: file.mode,+ matchesNode:+ file instanceof fs.Stats && file.size === fs.statSync(target).size,+ missing: await error(path.join(dir, 'missing/child.bin')),+ notDirectory: await error(path.join(target, 'child.bin')),+ notDirectoryDeep: await error(path.join(target, 'child/grandchild.bin')),+ missingParent: await error(`${dir}/missing/../stat-target.bin`),+ notDirectoryParent: await error(`${target}/../stat-target.bin`),+ optional,+ })}`+}test/e2e/app-dir/webpack-loader-fs/stat-parent-loader.jsadded20 + / 0 −
@@ -0,0 +1,20 @@+const fs = require('fs')++// Directory stat dependencies would mask tracking the wrong target when a file is replaced.+module.exports = async function () {+ const stat = (p) =>+ new Promise((resolve, reject) => {+ this.fs.stat(p, (err, stats) => (err ? reject(err) : resolve(stats)))+ })+ const relativePath = 'app/path/to/symlink/../stat-parent-target.bin'+ const absolutePath = `${this.rootContext}/${relativePath}`+ const absolute = await stat(absolutePath)+ const relative = await stat(relativePath)+ const buffer = await stat(Buffer.from(absolutePath))+ return `module.exports = ${JSON.stringify({+ parentSize: absolute.size,+ relativeParentSize: relative.size,+ bufferParentSize: buffer.size,+ parentMatchesNode: absolute.size === fs.statSync(absolutePath).size,+ })}`+}test/e2e/app-dir/webpack-loader-fs/webpack-loader-fs.test.ts126 + / 0 −
@@ -1,4 +1,7 @@ import { nextTestSetup } from 'e2e-utils'+import { retry } from 'next-test-utils'+import { utimes } from 'fs/promises'+import path from 'path' describe('webpack-loader-fs', () => { const { next } = nextTestSetup({@@ -11,4 +14,127 @@ describe('webpack-loader-fs', () => { "Buffer read: 18, string read: 'this is some data', binary read: 6765, glob read: 'one.txt'" ) })++ const readStat = async () => {+ const $ = await next.render$('/')+ return JSON.parse($('#stat').text())+ }++ const readParentStat = async () => {+ const $ = await next.render$('/parent')+ return JSON.parse($('#parent-stat').text())+ }++ it('exposes Node stat results, options, path types, and callback errors', async () => {+ const stat = await readStat()+ expect(stat).toMatchObject({+ file: true,+ directory: true,+ root: true,+ symlink: false,+ linkedDirectory: true,+ size: 6,+ date: true,+ bigint: 'bigint',+ bigintSize: '6',+ bufferSize: 6,+ urlSize: 6,+ matchesNode: true,+ missing: {+ code: 'ENOENT',+ syscall: 'stat',+ correctPath: true,+ matchesNode: true,+ },+ notDirectory: { syscall: 'stat', correctPath: true, matchesNode: true },+ notDirectoryDeep: {+ syscall: 'stat',+ correctPath: true,+ matchesNode: true,+ },+ missingParent:+ process.platform === 'win32'+ ? null+ : {+ code: 'ENOENT',+ syscall: 'stat',+ correctPath: true,+ matchesNode: true,+ },+ notDirectoryParent:+ process.platform === 'win32'+ ? null+ : { syscall: 'stat', correctPath: true, matchesNode: true },+ optional: 'missing',+ })+ expect(stat.mtimeMs).toEqual(expect.any(Number))+ expect(stat.mode).toEqual(expect.any(Number))+ expect(await readParentStat()).toMatchObject({+ parentSize: 6,+ relativeParentSize: 6,+ bufferParentSize: 6,+ parentMatchesNode: true,+ })+ })++ // Stat tracking is automatic in Turbopack; webpack loaders must register their own dependencies.+ // @force-gate dev && turbopack+ it('invalidates a stat-only read when the file size changes', async () => {+ await readStat()+ await next.patchFile('app/stat-target.bin', 'a longer file', async () => {+ await retry(async () => expect((await readStat()).size).toBe(13))+ })+ await retry(async () => expect((await readStat()).size).toBe(6))+ })++ // @force-gate dev && turbopack+ it('tracks metadata through a symlinked parent directory', async () => {+ const initial = await readStat()+ await next.patchFile(+ 'app/glob-target/inner/path/one.txt',+ 'a longer symlink target',+ async () => {+ await retry(async () => expect((await readStat()).linkedSize).toBe(23))+ }+ )+ await retry(async () =>+ expect((await readStat()).linkedSize).toBe(initial.linkedSize)+ )+ })++ // @force-gate dev && turbopack+ it('invalidates metadata-only changes without reading file contents', async () => {+ const initial = await readStat()+ const target = path.join(next.testDir, 'app/stat-target.bin')+ const modified = new Date('2000-01-01T00:00:00Z')+ try {+ await utimes(target, modified, modified)+ await retry(async () =>+ expect((await readStat()).mtimeMs).toBe(modified.getTime())+ )+ } finally {+ const previous = new Date(initial.mtimeMs)+ await utimes(target, previous, previous)+ }+ await retry(async () =>+ expect((await readStat()).mtimeMs).not.toBe(modified.getTime())+ )+ })++ // @force-gate dev && turbopack+ it('tracks missing-path creation, file-to-directory replacement, and deletion', async () => {+ expect((await readStat()).optional).toBe('missing')+ await next.patchFile('app/optional.bin', 'created')+ try {+ await retry(async () => expect((await readStat()).optional).toBe('file'))+ await next.remove('app/optional.bin')+ await next.patchFile('app/optional.bin/child', 'child')+ await retry(async () =>+ expect((await readStat()).optional).toBe('directory')+ )+ } finally {+ await next.remove('app/optional.bin')+ }+ await retry(async () => expect((await readStat()).optional).toBe('missing'))+ }) })turbopack/crates/turbo-tasks-fs/src/disk.rs79 + / 15 −
@@ -25,8 +25,8 @@ use tokio::{ use tracing::Instrument; use turbo_rcstr::{RcStr, rcstr}; use turbo_tasks::{- CapturedEffect, Effect, EffectExt, EffectStateStorage, InvalidationReason, Invalidator,- NonLocalValue, OperationVc, ReadRef, ResolvedVc, TurboTasksApi, ValueToString, Vc,+ CapturedEffect, Completion, Effect, EffectExt, EffectStateStorage, InvalidationReason,+ Invalidator, NonLocalValue, OperationVc, ReadRef, ResolvedVc, TurboTasksApi, ValueToString, Vc, debug::ValueDebugFormat, turbo_tasks_weak, turbobail, }; use turbo_tasks_hash::{hash_xxh3_hash64, hash_xxh3_hash128};@@ -36,8 +36,9 @@ use turbo_unix_path::{normalize_path, sys_to_unix, unix_to_sys}; use crate::windows::{is_link_junction_point, to_verbatim_with_case_folded_disk}; use crate::{ AnyhowWrapper, DiskFileSystemMap, File, FileComparison, FileContent, FileMeta, FileSystem,- FileSystemPath, FileSystemPathOption, LinkContent, LinkTarget, PersistedFileContent,- RawDirectoryContent, RawDirectoryEntry, WriteLinkContent, WriteLinkTargetType,+ FileSystemEntryType, FileSystemPath, FileSystemPathOption, LinkContent, LinkTarget,+ PersistedFileContent, RawDirectoryContent, RawDirectoryEntry, WriteLinkContent,+ WriteLinkTargetType, canonicalized_path_cache::CanonicalizedPathWalkCache, invalidation::Write, invalidator_map::InvalidatorMap,@@ -227,6 +228,9 @@ pub(crate) struct DiskFileSystemInner { #[turbo_tasks(debug_ignore, unsafe_ignore)] #[bincode(skip)] pub(crate) dir_invalidator_map: InvalidatorMap,+ #[turbo_tasks(debug_ignore, unsafe_ignore)]+ #[bincode(skip)]+ pub(crate) metadata_invalidator_map: InvalidatorMap, /// Lock that makes invalidation atomic. It will keep a write lock during /// watcher invalidation and a read lock during other operations. #[turbo_tasks(debug_ignore, unsafe_ignore)]@@ -300,16 +304,17 @@ impl DiskFileSystemInner { /// re-read the updated content. This is necessary because the file watcher may not be active /// (e.g., in tests or build-only scenarios). fn invalidate_from_write(&self, full_path: &Path) {- let mut invalidator_map = self.invalidator_map.lock().unwrap();- if let Some(invalidators) = invalidator_map.remove(full_path) {- let Some(turbo_tasks) = self.turbo_tasks.upgrade() else {- return;- };- let _guard = self.tokio_handle.enter();- let reason = Write {- path: full_path.to_string_lossy().into_owned(),- };- Invalidator::invalidate_all_with_reason(invalidators, &*turbo_tasks, reason);+ for map in [&self.invalidator_map, &self.metadata_invalidator_map] {+ if let Some(invalidators) = map.lock().unwrap().remove(full_path) {+ let Some(turbo_tasks) = self.turbo_tasks.upgrade() else {+ return;+ };+ let _guard = self.tokio_handle.enter();+ let reason = Write {+ path: full_path.to_string_lossy().into_owned(),+ };+ Invalidator::invalidate_all_with_reason(invalidators, &*turbo_tasks, reason);+ } } } @@ -340,9 +345,11 @@ impl DiskFileSystemInner { let invalidator_map = take(&mut *self.invalidator_map.lock().unwrap()); let dir_invalidator_map = take(&mut *self.dir_invalidator_map.lock().unwrap());+ let metadata_invalidator_map = take(&mut *self.metadata_invalidator_map.lock().unwrap()); let invalidators = invalidator_map .into_iter() .chain(dir_invalidator_map)+ .chain(metadata_invalidator_map) .flat_map(|(_, invalidators)| invalidators.into_iter()); Invalidator::invalidate_all(invalidators, &*turbo_tasks); }@@ -362,16 +369,22 @@ impl DiskFileSystemInner { let invalidator_map = take(&mut *self.invalidator_map.lock().unwrap()); let dir_invalidator_map = take(&mut *self.dir_invalidator_map.lock().unwrap());+ let metadata_invalidator_map = take(&mut *self.metadata_invalidator_map.lock().unwrap()); let path_invalidator_counts = || { invalidator_map .values() .chain(dir_invalidator_map.values())+ .chain(metadata_invalidator_map.values()) .map(|path_invalidators| path_invalidators.len()) .filter(|&count| count > 0) }; let mut invalidators = Vec::with_capacity(path_invalidator_counts().sum()); let mut reasons = Vec::with_capacity(path_invalidator_counts().count());- for (path, path_invalidators) in invalidator_map.into_iter().chain(dir_invalidator_map) {+ for (path, path_invalidators) in invalidator_map+ .into_iter()+ .chain(dir_invalidator_map)+ .chain(metadata_invalidator_map)+ { if path_invalidators.is_empty() { continue; }@@ -400,6 +413,7 @@ impl DiskFileSystemInner { let path_count = paths.size_hint().0; let mut invalidator_map = self.invalidator_map.lock().unwrap(); let mut dir_invalidator_map = self.dir_invalidator_map.lock().unwrap();+ let mut metadata_invalidator_map = self.metadata_invalidator_map.lock().unwrap(); let mut invalidators = Vec::with_capacity(path_count); let mut reasons = Vec::with_capacity(path_count); let mut add = |path: &Path, path_invalidators: FxHashSet<Invalidator>| {@@ -428,16 +442,25 @@ impl DiskFileSystemInner { { add(&invalidated_path, path_invalidators); }+ for (invalidated_path, path_invalidators) in+ metadata_invalidator_map.extract_path_with_children(&path)+ {+ add(&invalidated_path, path_invalidators);+ } } for path in parent_dirs_to_invalidate { if let Some(path_invalidators) = dir_invalidator_map.remove(path.as_path()) { add(&path, path_invalidators); }+ if let Some(path_invalidators) = metadata_invalidator_map.remove(path.as_path()) {+ add(&path, path_invalidators);+ } } drop(invalidator_map); drop(dir_invalidator_map);+ drop(metadata_invalidator_map); Invalidator::invalidate_all_with_reasons(invalidators, &*turbo_tasks, reasons); }@@ -844,6 +867,7 @@ impl DiskFileSystem { invalidation_lock: Default::default(), invalidator_map: InvalidatorMap::new(), dir_invalidator_map: InvalidatorMap::new(),+ metadata_invalidator_map: InvalidatorMap::new(), read_semaphore: create_read_semaphore(), write_semaphore: create_write_semaphore(), watcher: DiskWatcher::new(watcher_config),@@ -865,6 +889,46 @@ impl Debug for DiskFileSystem { } } +#[turbo_tasks::value_impl]+impl DiskFileSystem {+ /// Tracks changes relevant to an external stat call, including sizes and timestamps.+ /// This registers dependencies; it does not return stat results. `metadata()` returns+ /// `FileMeta`, which omits sizes and timestamps. Metadata-only changes must invalidate stat+ /// calls without invalidating ordinary content reads.+ ///+ /// The caller must resolve symlinks, including parent directories, before calling this.+ #[turbo_tasks::function(fs, session_dependent)]+ pub async fn full_metadata(&self, fs_path: FileSystemPath) -> Result<Vc<Completion>> {+ if self.inner.is_path_denied(&fs_path) {+ turbobail!("Cannot track metadata from denied path: {fs_path}");+ }+ let full_path = Arc::new(self.to_sys_path_raw(&fs_path));+ if let Some(invalidator) = turbo_tasks::get_invalidator() {+ self.inner+ .metadata_invalidator_map+ .insert(full_path.clone(), invalidator);+ let entry_type = *fs_path.get_type().await?;+ debug_assert!(+ !matches!(entry_type, FileSystemEntryType::Symlink),+ "full_metadata requires a path with symlinks already resolved"+ );+ if !fs_path.is_root() {+ self.inner+ .watcher+ .ensure_watched_file(&full_path, self.inner.root_path())+ .await?;+ }+ if matches!(entry_type, FileSystemEntryType::Directory) {+ self.inner+ .watcher+ .ensure_watched_dir(&full_path, self.inner.root_path())+ .await?;+ }+ }+ Ok(Completion::new())+ }+}+ #[turbo_tasks::value_impl] impl FileSystem for DiskFileSystem { #[turbo_tasks::function(fs, session_dependent)]turbopack/crates/turbo-tasks-fs/src/watcher/fs_api.rs5 + / 0 −
@@ -23,6 +23,7 @@ pub(crate) trait DiskFileSystemWatcherApi: Send + Sync + 'static { fn tokio_handle(&self) -> &Handle; fn invalidator_map(&self) -> &InvalidatorMap; fn dir_invalidator_map(&self) -> &InvalidatorMap;+ fn metadata_invalidator_map(&self) -> &InvalidatorMap; fn invalidation_lock(&self) -> &RwLock<()>; fn invalidate_all(&self); fn invalidate_all_with_reason<R: InvalidationReason + Clone>(@@ -60,6 +61,10 @@ impl DiskFileSystemWatcherApi for DiskFileSystemInner { &self.dir_invalidator_map } + fn metadata_invalidator_map(&self) -> &InvalidatorMap {+ &self.metadata_invalidator_map+ }+ fn invalidation_lock(&self) -> &RwLock<()> { &self.invalidation_lock }turbopack/crates/turbo-tasks-fs/src/watcher/mock_fs_api.rs6 + / 0 −
@@ -24,6 +24,7 @@ pub struct MockFileSystem { pub tokio_handle: Handle, pub invalidator_map: InvalidatorMap, pub dir_invalidator_map: InvalidatorMap,+ pub metadata_invalidator_map: InvalidatorMap, pub invalidation_lock: RwLock<()>, pub run_counts: Mutex<FxHashMap<Arc<PathBuf>, u64>>, /// Tracked reads wait for a read lock on this before reading. Tests can hold the write lock to@@ -50,6 +51,7 @@ impl MockFileSystem { tokio_handle: Handle::current(), invalidator_map: InvalidatorMap::new(), dir_invalidator_map: InvalidatorMap::new(),+ metadata_invalidator_map: InvalidatorMap::new(), invalidation_lock: RwLock::new(()), run_counts: Mutex::new(FxHashMap::default()), read_gate: RwLock::new(()),@@ -131,6 +133,10 @@ impl DiskFileSystemWatcherApi for MockFileSystem { &self.dir_invalidator_map } + fn metadata_invalidator_map(&self) -> &InvalidatorMap {+ &self.metadata_invalidator_map+ }+ fn invalidation_lock(&self) -> &RwLock<()> { &self.invalidation_lock }turbopack/crates/turbo-tasks-fs/src/watcher/mod.rs146 + / 8 −
@@ -690,7 +690,12 @@ impl DiskWatcher { delay = delay.max(config.extended_batch_delay_duration); } - if batch.add_event(event, fs.invalidator_map(), fs.dir_invalidator_map()) {+ if batch.add_event(+ event,+ fs.invalidator_map(),+ fs.dir_invalidator_map(),+ fs.metadata_invalidator_map(),+ ) { schedule.extend(delay); } else if !batch.has_pending_invalidations() && batch.has_new_paths() { // No batch is pending, so `recv_event` would wait for the next event,@@ -761,6 +766,7 @@ impl DiskWatcher { batch.execute( fs.invalidator_map(), fs.dir_invalidator_map(),+ fs.metadata_invalidator_map(), |invalidation_reason_path, invalidator| { invalidators.push(invalidator); if changed_paths.last().map(|p| p.as_path()) != Some(invalidation_reason_path) {@@ -810,6 +816,8 @@ bitflags! { const PATH_AND_CHILDREN = 1 << 2; /// Invalidate this path and all of its children in the directory-listing invalidator map. const PATH_AND_CHILDREN_DIR = 1 << 3;+ /// Invalidate metadata readers of exactly this path.+ const PATH_METADATA = 1 << 4; } } @@ -947,7 +955,10 @@ impl BatchedInvalidations { fn mark_parent_dir(&mut self, path: &Path) { if let Some(parent) = path.parent() {- self.mark(Cow::Borrowed(parent), InvalidationFlags::PATH_DIR);+ self.mark(+ Cow::Borrowed(parent),+ InvalidationFlags::PATH_DIR | InvalidationFlags::PATH_METADATA,+ ); } } @@ -962,10 +973,13 @@ impl BatchedInvalidations { event: notify::Event, invalidator_map: &InvalidatorMap, dir_invalidator_map: &InvalidatorMap,+ metadata_invalidator_map: &InvalidatorMap, ) -> bool { // if we need to lock an invalidator map, hold and cache the guard for the whole event let invalidator_map_guard = LazyCell::new(|| invalidator_map.lock().unwrap()); let dir_invalidator_map_guard = LazyCell::new(|| dir_invalidator_map.lock().unwrap());+ let metadata_invalidator_map_guard =+ LazyCell::new(|| metadata_invalidator_map.lock().unwrap()); // If this path has no reader, we should not add it to the batch, and we should not extend // the batch schedule. This function is best-effort and marks some events as relevant that@@ -979,10 +993,13 @@ impl BatchedInvalidations { // (some events invalidate the parent directory) invalidator_map_guard.contains_key(path) || dir_invalidator_map_guard.contains_key(path)+ || metadata_invalidator_map_guard.contains_key(path) || parent.is_some_and(|parent| dir_invalidator_map_guard.contains_key(parent))+ || parent.is_some_and(|parent| metadata_invalidator_map_guard.contains_key(parent)) || (recursive && (invalidator_map_guard.contains_path_or_children(path)- || dir_invalidator_map_guard.contains_path_or_children(path)))+ || dir_invalidator_map_guard.contains_path_or_children(path)+ || metadata_invalidator_map_guard.contains_path_or_children(path))) }; let paths = event.paths;@@ -1005,6 +1022,14 @@ impl BatchedInvalidations { } } }+ EventKind::Modify(ModifyKind::Metadata(_)) => {+ for path in paths {+ if metadata_invalidator_map_guard.contains_key(path.as_path()) {+ last_updated_index =+ Some(self.mark(Cow::Owned(path), InvalidationFlags::PATH_METADATA));+ }+ }+ } EventKind::Create(_) => { for path in paths { self.mark_new_path(&path);@@ -1068,9 +1093,7 @@ impl BatchedInvalidations { )); } }- EventKind::Modify(ModifyKind::Metadata(..) | ModifyKind::Other)- | EventKind::Access(_)- | EventKind::Other => {}+ EventKind::Modify(ModifyKind::Other) | EventKind::Access(_) | EventKind::Other => {} } if let Some(index) = last_updated_index { self.last_updated_index = Some(index);@@ -1105,6 +1128,7 @@ impl BatchedInvalidations { &mut self, invalidator_map: &InvalidatorMap, dir_invalidator_map: &InvalidatorMap,+ metadata_invalidator_map: &InvalidatorMap, mut invalidate: impl FnMut(&Path, Invalidator), ) { for (map, exact_flag, recursive_flag) in [@@ -1118,6 +1142,11 @@ impl BatchedInvalidations { InvalidationFlags::PATH_DIR, InvalidationFlags::PATH_AND_CHILDREN_DIR, ),+ (+ metadata_invalidator_map,+ InvalidationFlags::PATH | InvalidationFlags::PATH_METADATA,+ InvalidationFlags::PATH_AND_CHILDREN,+ ), ] { let mut map = map.lock().unwrap(); for (path, flags) in &self.paths {@@ -1127,7 +1156,7 @@ impl BatchedInvalidations { invalidate(path, invalidator); } }- } else if flags.contains(exact_flag)+ } else if flags.intersects(exact_flag) && let Some(invalidators) = map.remove(&**path) { for invalidator in invalidators {@@ -1292,6 +1321,7 @@ mod tests { tt.run_once(async move { let invalidator_map = InvalidatorMap::new(); let dir_invalidator_map = InvalidatorMap::new();+ let metadata_invalidator_map = InvalidatorMap::new(); let invalidator = turbo_tasks::get_invalidator().unwrap(); invalidator_map.insert(Arc::new(PathBuf::from("root/file")), invalidator); dir_invalidator_map.insert(Arc::new(PathBuf::from("root/listed")), invalidator);@@ -1315,7 +1345,12 @@ mod tests { BatchedInvalidations::new(DiskWatcherRecursiveMode::NonRecursive, false); let event = notify::Event::new(kind).add_path(PathBuf::from(path)); assert_eq!(- batch.add_event(event, &invalidator_map, &dir_invalidator_map),+ batch.add_event(+ event,+ &invalidator_map,+ &dir_invalidator_map,+ &metadata_invalidator_map+ ), relevant, "{kind:?} at {path}", );@@ -1327,6 +1362,109 @@ mod tests { .unwrap(); } + #[cfg(not(miri))]+ #[tokio::test]+ async fn metadata_events_only_invalidate_metadata_readers() {+ let tt = TurboTasks::new(TurboTasksBackend::new(+ BackendOptions::default(),+ noop_backing_storage(),+ ));+ tt.run_once(async move {+ let invalidator_map = InvalidatorMap::new();+ let dir_invalidator_map = InvalidatorMap::new();+ let metadata_invalidator_map = InvalidatorMap::new();+ let path = PathBuf::from("root/file");+ let invalidator = turbo_tasks::get_invalidator().unwrap();+ invalidator_map.insert(Arc::new(path.clone()), invalidator);+ metadata_invalidator_map.insert(Arc::new(path.clone()), invalidator);++ let mut batch =+ BatchedInvalidations::new(DiskWatcherRecursiveMode::NonRecursive, false);+ let event = notify::Event::new(EventKind::Modify(ModifyKind::Metadata(+ MetadataKind::Permissions,+ )))+ .add_path(path.clone());+ assert!(batch.add_event(+ event,+ &invalidator_map,+ &dir_invalidator_map,+ &metadata_invalidator_map+ ));+ batch.execute(+ &invalidator_map,+ &dir_invalidator_map,+ &metadata_invalidator_map,+ |_, _| {},+ );+ assert!(invalidator_map.lock().unwrap().contains_key(path.as_path()));+ assert!(metadata_invalidator_map.lock().unwrap().is_empty());++ metadata_invalidator_map.insert(Arc::new(path.clone()), invalidator);+ let event = notify::Event::new(EventKind::Modify(ModifyKind::Data(DataChange::Any)))+ .add_path(path);+ assert!(batch.add_event(+ event,+ &invalidator_map,+ &dir_invalidator_map,+ &metadata_invalidator_map+ ));+ batch.execute(+ &invalidator_map,+ &dir_invalidator_map,+ &metadata_invalidator_map,+ |_, _| {},+ );+ assert!(invalidator_map.lock().unwrap().is_empty());+ assert!(metadata_invalidator_map.lock().unwrap().is_empty());+ anyhow::Ok(())+ })+ .await+ .unwrap();+ }++ #[cfg(not(miri))]+ #[tokio::test]+ async fn entry_changes_invalidate_directory_and_missing_metadata_reads() {+ let tt = TurboTasks::new(TurboTasksBackend::new(+ BackendOptions::default(),+ noop_backing_storage(),+ ));+ tt.run_once(async move {+ for kind in [+ EventKind::Create(CreateKind::Any),+ EventKind::Remove(RemoveKind::Any),+ EventKind::Modify(ModifyKind::Name(RenameMode::From)),+ ] {+ let invalidator_map = InvalidatorMap::new();+ let dir_invalidator_map = InvalidatorMap::new();+ let metadata_invalidator_map = InvalidatorMap::new();+ let invalidator = turbo_tasks::get_invalidator().unwrap();+ for path in ["root", "root/entry", "root/entry/child"] {+ metadata_invalidator_map.insert(Arc::new(PathBuf::from(path)), invalidator);+ }+ let mut batch =+ BatchedInvalidations::new(DiskWatcherRecursiveMode::NonRecursive, false);+ let event = notify::Event::new(kind).add_path(PathBuf::from("root/entry"));+ assert!(batch.add_event(+ event,+ &invalidator_map,+ &dir_invalidator_map,+ &metadata_invalidator_map+ ));+ batch.execute(+ &invalidator_map,+ &dir_invalidator_map,+ &metadata_invalidator_map,+ |_, _| {},+ );+ assert!(metadata_invalidator_map.lock().unwrap().is_empty());+ }+ anyhow::Ok(())+ })+ .await+ .unwrap();+ }+ /// Polls [`tracked_read`] until it has executed more than `previous_runs` times, i.e. until the /// watcher has invalidated it. async fn wait_for_rerun(fs: &Arc<MockFileSystem>, path: &Path, previous_runs: u64) {turbopack/crates/turbopack-node/js/src/transforms/transforms.ts4 + / 0 −
@@ -41,6 +41,10 @@ export type IpcRequestMessage = type: 'trackFileRead' file: string }+ | {+ type: 'trackFileStat'+ file: string+ } | { type: 'importModule' lookupPath: stringturbopack/crates/turbopack-node/js/src/transforms/webpack-loaders.ts33 + / 1 −
@@ -17,6 +17,7 @@ import { import fs from 'fs' import path from 'path' import { absolutify, contextify, createHash } from './webpack-loaders-utils'+import { fileURLToPath } from 'url' export type IpcInfoMessage = | {@@ -205,11 +206,42 @@ const transform = ( buildDependencies.add(dependency) }, fs: {+ stat(+ p: fs.PathLike,+ optionsOrCallback:+ | fs.StatOptions+ | undefined+ | ((+ err: NodeJS.ErrnoException | null,+ stats: fs.Stats | fs.BigIntStats+ ) => void),+ callback?: (+ err: NodeJS.ErrnoException | null,+ stats: fs.Stats | fs.BigIntStats+ ) => void+ ) {+ const file = p instanceof URL ? fileURLToPath(p) : p.toString()+ ipc+ .sendRequest({+ type: 'trackFileStat',+ file,+ })+ .then(+ () => {+ if (typeof optionsOrCallback === 'function') {+ fs.stat(p, optionsOrCallback)+ } else {+ fs.stat(p, optionsOrCallback, callback!)+ }+ },+ (err) => ipc.sendError(err)+ )+ }, readFile(p: string, optionsOrCb: any, maybeCb: any) { ipc .sendRequest({ type: 'trackFileRead',- file: pathResolve(p),+ file: p, }) .then( () => {turbopack/crates/turbopack-node/src/transforms/webpack.rs37 + / 2 −
@@ -17,7 +17,7 @@ use turbo_tasks::{ }; use turbo_tasks_env::ProcessEnv; use turbo_tasks_fs::{- DiskFileSystem, File, FileContent, FileSystemEntryType, FileSystemPath,+ DiskFileSystem, File, FileContent, FileSystemEntryType, FileSystemPath, RealPathErrorType, glob::{Glob, GlobOptions}, json::parse_json_with_source_context, rope::Rope,@@ -242,6 +242,30 @@ async fn loader_path(cwd: &FileSystemPath, path: &str) -> Result<FileSystemPath> .with_context(|| format!("webpack loader path {path:?} is outside configured roots")) } +/// Resolves a path passed to the loader's `fs`. As elsewhere in Turbopack, `..` is applied+/// lexically before symlinks are followed. Returns `None` when Node's callback will report the+/// error instead.+async fn resolve_loader_path(cwd: &FileSystemPath, path: &str) -> Result<Option<FileSystemPath>> {+ match loader_path(cwd, path).await?.realpath().await? {+ Ok(path) => Ok(Some(path)),+ // Changes behind a symlink leaving the configured roots can't be tracked.+ Err(error) if matches!(error.kind(), RealPathErrorType::Invalid { .. }) => {+ Err(error.into())+ }+ Err(_) => Ok(None),+ }+}++async fn register_loader_stat_dependencies(cwd: &FileSystemPath, file: &str) -> Result<()> {+ let Some(path) = resolve_loader_path(cwd, file).await? else {+ return Ok(());+ };+ let fs = ResolvedVc::try_downcast_type::<DiskFileSystem>(path.fs)+ .expect("resolved loader paths are backed by DiskFileSystem");+ fs.full_metadata(path).await?;+ Ok(())+}+ // TODO: This creates a task that's keyed on the list of file paths, which might churn through tasks // (creating more pressure for task GC). #[turbo_tasks::function]@@ -667,6 +691,8 @@ pub enum RequestMessage { #[serde(rename_all = "camelCase")] TrackFileRead { file: RcStr }, #[serde(rename_all = "camelCase")]+ TrackFileStat { file: RcStr },+ #[serde(rename_all = "camelCase")] ImportModule { lookup_path: RcStr, request: RcStr }, } @@ -690,6 +716,7 @@ pub enum ResponseMessage { }, // Only used for tracking invalidations, no content is returned. TrackFileRead {},+ TrackFileStat {}, #[serde(rename_all = "camelCase")] ImportModule { entry_path: RcStr,@@ -935,9 +962,17 @@ impl EvaluateContext for WebpackLoaderContext { RequestMessage::TrackFileRead { file } => { // Ignore result, we read on the JS side again to prevent some IPC overhead. Still // await the read though to cover at least one class of race conditions.- let _ = &*loader_path(&self.cwd, &file).await?.read().await?;+ if let Some(path) = resolve_loader_path(&self.cwd, &file).await? {+ let _ = &*path.read().await?;+ } Ok(ResponseMessage::TrackFileRead {}) }+ RequestMessage::TrackFileStat { file } => {+ register_loader_stat_dependencies(&self.cwd, &file)+ .await+ .with_context(|| format!("tracking webpack loader stat of {file}"))?;+ Ok(ResponseMessage::TrackFileStat {})+ } RequestMessage::ImportModule { lookup_path, request,