Serve legacy module aliases through a sys.meta_path finder

@@ -20,9 +20,8 @@  __version__ = "5.18.0.dev0" -import importlib+import re import sys-import types from pathlib import Path from typing import TYPE_CHECKING @@ -54,7 +53,7 @@ from .utils import is_torchvision_available as is_torchvision_available from .utils import is_vision_available as is_vision_available from .utils import logging as logging-from .utils.import_utils import define_import_structure+from .utils.import_utils import _LegacyModuleAliasFinder, define_import_structure   logger = logging.get_logger(__name__)  # pylint: disable=invalid-name@@ -808,7 +807,16 @@     from .utils.quantization_config import VptqConfig as VptqConfig     from .video_processing_utils import BaseVideoProcessor as BaseVideoProcessor else:+    # Deprecated top-level module paths -> replacements. Declared here so the legacy names are also part of the lazy+    # import structure, keeping `transformers.tokenization_utils_fast` attribute access working.+    _LEGACY_MODULE_ALIASES = {+        "tokenization_utils_fast": "tokenization_utils_tokenizers",+        "tokenization_utils": "tokenization_utils_sentencepiece",+        "image_processing_utils_fast": "image_processing_backends",+    }     _import_structure = {k: set(v) for k, v in _import_structure.items()}+    for _alias in _LEGACY_MODULE_ALIASES:+        _import_structure.setdefault(_alias, set())      import_structure = define_import_structure(Path(__file__).parent / "models", prefix="models")     import_structure[frozenset({})].update(_import_structure)@@ -821,54 +829,22 @@         extra_objects={"__version__": __version__},     ) -    def _create_module_alias(alias: str, target: str) -> None:-        """-        Lazily redirect legacy module paths to their replacements without importing heavy deps.-        """-        module = types.ModuleType(alias)-        module.__doc__ = f"Alias module for backward compatibility with `{target}`."-        # Set __file__ explicitly so that inspect.py's hasattr(module, '__file__') check-        # never falls through to __getattr__ and triggers a premature (possibly circular) import.-        module.__file__ = None--        def _get_target():-            return importlib.import_module(target, __name__)--        module.__getattr__ = lambda name: getattr(_get_target(), name)-        module.__dir__ = lambda: dir(_get_target())--        sys.modules[alias] = module-        setattr(sys.modules[__name__], alias.rsplit(".", 1)[-1], module)--    _create_module_alias(f"{__name__}.tokenization_utils_fast", ".tokenization_utils_tokenizers")-    _create_module_alias(f"{__name__}.tokenization_utils", ".tokenization_utils_sentencepiece")-    _create_module_alias(f"{__name__}.image_processing_utils_fast", ".image_processing_backends")--    for _proc_file in sorted((Path(__file__).parent / "models").rglob("image_processing_*.py")):-        _model = _proc_file.parent.name-        _module = _proc_file.stem-        _target = f".models.{_model}.{_module}"-        _create_module_alias(f"{__name__}.models.{_model}.{_module}_fast", _target)--        # Also map XImageProcessorFast -> XImageProcessor for backward compat with old class names.-        def getattr_factory(target):-            def _getattr(name):-                if name.endswith("Fast"):-                    new_name = name.removesuffix("Fast")-                    logger.warning_once(-                        "Accessing `%s` from `%s`. Returning `%s` instead. Behavior may be "-                        "different and this alias will be removed in future versions.",-                        name,-                        target,-                        new_name,-                    )-                    return getattr(importlib.import_module(target, __name__), new_name)-                # Silently forward non-Fast names to target (transparent alias behavior)-                return getattr(importlib.import_module(target, __name__), name)--            return _getattr--        sys.modules[f"{__name__}.models.{_model}.{_module}_fast"].__getattr__ = getattr_factory(_target)+    # Deprecated module paths, served on demand by `_LegacyModuleAliasFinder`. `models/x/image_processing_x_fast.py`+    # became `models/x/image_processing_x.py` when torchvision became the default backend (PIL: `image_processing_pil_x.py`).+    _LEGACY_FAST_IMAGE_PROCESSOR_MODULE = re.compile(+        rf"{re.escape(__name__)}\.models\.(\w+)\.image_processing_(?!pil_)(\w+)_fast"+    )++    def _resolve_legacy_module_alias(fullname: str) -> str | None:+        if (match := _LEGACY_FAST_IMAGE_PROCESSOR_MODULE.fullmatch(fullname)) is not None:+            return f"{__name__}.models.{match[1]}.image_processing_{match[2]}"+        package, _, name = fullname.rpartition(".")+        if package == __name__ and name in _LEGACY_MODULE_ALIASES:+            return f"{__name__}.{_LEGACY_MODULE_ALIASES[name]}"+        return None++    if not any(isinstance(finder, _LegacyModuleAliasFinder) for finder in sys.meta_path):  # this init can run twice+        sys.meta_path.append(_LegacyModuleAliasFinder(_resolve_legacy_module_alias))  if not is_torch_available():     logger.warning_advice(

@@ -16,6 +16,7 @@ """  import functools+import importlib.abc import importlib.machinery import importlib.metadata import importlib.util@@ -2802,6 +2803,38 @@ class OptionalDependencyNotAvailable(BaseException):     """Internally used error class for signalling an optional dependency was not found."""  +class _LegacyModuleAliasFinder(importlib.abc.MetaPathFinder, importlib.abc.Loader):+    """+    Serves deprecated module paths as copies of their replacement, created only when actually imported. Nothing is+    pre-registered in `sys.modules`, so tools that scan it and probe attributes (unittest's `assertWarns`, `pickle`,+    `inspect`, ...) never trigger imports of optional heavy dependencies.+    """++    def __init__(self, resolve: Callable[[str], str | None]):+        self.resolve = resolve  # legacy module name -> replacement module name, `None` if not an alias++    def find_spec(self, fullname, path=None, target=None):+        replacement = self.resolve(fullname)+        if replacement is None or importlib.util.find_spec(replacement) is None:+            return None+        return importlib.machinery.ModuleSpec(fullname, self, loader_state=replacement)++    def exec_module(self, module):+        # Import the replacement first: a missing optional dependency fails right here, at the user's import statement.+        replacement = importlib.import_module(module.__spec__.loader_state)+        vars(module).update({k: v for k, v in vars(replacement).items() if k == "__all__" or not k.startswith("__")})++        def __getattr__(name):  # `XImageProcessorFast` classes were renamed `XImageProcessor`+            if name.endswith("ImageProcessorFast") and hasattr(replacement, name[:-4]):+                logger.warning_once(+                    f"`{name}` is deprecated, use `{name[:-4]}` from `{replacement.__name__}` instead."+                )+                return getattr(replacement, name[:-4])+            raise AttributeError(f"module {module.__name__!r} has no attribute {name!r}")++        module.__getattr__ = __getattr__++ def direct_transformers_import(path: str, file="__init__.py") -> ModuleType:     """Imports transformers directly 

@@ -3,11 +3,18 @@ from types import ModuleType from unittest.mock import DEFAULT, MagicMock, patch +import pytest from packaging.version import parse as parse_version from parameterized import parameterized  from transformers import logging-from transformers.testing_utils import CaptureLogger, LoggingLevel, require_torch, run_test_using_subprocess+from transformers.testing_utils import (+    CaptureLogger,+    LoggingLevel,+    require_torch,+    require_torchvision,+    run_test_using_subprocess,+) from transformers.utils.import_utils import (     _candidate_distribution_names,     _is_package_available,@@ -419,3 +426,62 @@ def run(x):         return x + 1 if helper(*args) else x - 1      run(torch.zeros(3))  # a graph break inside the helper would raise here+++@run_test_using_subprocess+def test_legacy_module_aliases_are_served_on_demand():+    """+    Deprecated module paths (`transformers.tokenization_utils_fast`, `transformers.models.x.image_processing_x_fast`,+    ...) are served by a `sys.meta_path` finder and must not be pre-registered in `sys.modules`: tools that scan+    `sys.modules` and probe attributes (unittest's `assertWarns`, `pickle`, `inspect`) would otherwise trigger imports+    of optional heavy dependencies. Regression test for https://github.com/huggingface/transformers/issues/48966.+    """+    import unittest+    import warnings++    import transformers+    from transformers.utils import import_utils++    finders = [f for f in sys.meta_path if isinstance(f, import_utils._LegacyModuleAliasFinder)]+    assert len(finders) == 1+    finder = finders[0]+    pre_registered = [name for name in sys.modules if finder.resolve(name) is not None]+    assert pre_registered == [], f"legacy module paths pre-registered in sys.modules: {pre_registered}"+    assert not any(name.startswith("image_processing_") for name in vars(transformers))++    # Scanning `sys.modules` (what `assertWarns` does on entry) must not import anything.+    modules_before = len(sys.modules)+    with unittest.TestCase().assertWarns(UserWarning):+        warnings.warn("probe", UserWarning)+    assert len(sys.modules) == modules_before++    # Legacy paths still work, as thin shims over the replacement module.+    from transformers.tokenization_utils_fast import PreTrainedTokenizerFast++    assert PreTrainedTokenizerFast is transformers.PreTrainedTokenizerFast+    shim = sys.modules["transformers.tokenization_utils_fast"]+    assert shim is transformers.tokenization_utils_fast+    assert shim.__spec__.loader is finder+    assert "PreTrainedTokenizerFast" in vars(shim)+    # Module metadata is the shim's own, so attribute probes behave like on any other module.+    assert getattr(shim, "__warningregistry__", None) is None+    assert getattr(shim, "does_not_exist", None) is None++    with pytest.raises(ModuleNotFoundError):+        import transformers.models.aria.image_processing_nope_fast  # noqa: F401+++@require_torchvision+def test_legacy_fast_image_processor_module_alias():+    """`models/x/image_processing_x_fast.py` became `models/x/image_processing_x.py`; the old path is an alias."""+    import transformers.models.aria.image_processing_aria_fast as legacy_module+    from transformers.models.aria.image_processing_aria import AriaImageProcessor+    from transformers.models.aria.image_processing_aria_fast import AriaImageProcessorFast  # legacy class name++    assert AriaImageProcessorFast is AriaImageProcessor+    assert legacy_module.AriaImageProcessor is AriaImageProcessor+    assert legacy_module.__all__ == ["AriaImageProcessor"]++    # Only paths that actually existed are aliased.+    with pytest.raises(ModuleNotFoundError):+        import transformers.models.aria.image_processing_pil_aria_fast  # noqa: F401