Stop importing yt-dlp, bs4 and lxml when importing zim or inputs

CHANGELOG.md

@@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `VideoMp4High` now encodes at CRF 28 instead of 20, which regularly produced a file larger than its input, and adds `-movflags +faststart` (#271)
 - All Webm presets (`VideoWebmLow`, `VideoWebmMedium`, `VideoWebmHigh`) now use `-row-mt 1` for faster VP9 encoding (#271)
 - Bumped `VideoWebmLow` to v4 and `VideoWebmHigh` to v3 following the `-row-mt` addition (#271)
+- `import zimscraperlib.zim` no longer imports yt-dlp, beautifulsoup4 or lxml, and `import zimscraperlib.inputs` no longer imports yt-dlp: `stream_file`, `get_retry_adapter` and `find_title_in_file` are now imported where they are used (#244)
 
 ### Fixed
 

pyproject.toml

@@ -234,6 +234,9 @@ ignore = [
   "S603",
   # Ignore complexity
   "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915",
+  # Modules of optional dependencies are imported where they are used so that
+  # importing a family does not load another family's dependencies
+  "PLC0415",
   # To be fixed once more recent Python versions are available
   "UP006", "UP007"
 ]

src/zimscraperlib/inputs.py

@@ -11,7 +11,6 @@
 from zimscraperlib.constants import (
     MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH as MAX_LONG_DESC_LENGTH,
 )
-from zimscraperlib.download import stream_file
 
 
 def handle_user_provided_file(
@@ -41,6 +40,8 @@ def handle_user_provided_file(
         )
 
     if str(source).startswith("http"):
+        from zimscraperlib.download import stream_file
+
         logger.debug(f"download {source} -> {dest}")
         headers = {"User-Agent": user_agent} if user_agent else None
         stream_file(url=str(source), fpath=dest, headers=headers)

src/zimscraperlib/zim/filesystem.py

@@ -31,7 +31,6 @@
 
 from zimscraperlib import logger
 from zimscraperlib.filesystem import get_file_mimetype
-from zimscraperlib.html import find_title_in_file
 from zimscraperlib.types import get_mime_for_name
 from zimscraperlib.zim import metadata
 from zimscraperlib.zim.creator import Creator
@@ -59,6 +58,8 @@ def get_path(self) -> str:
         return str(self.filepath.relative_to(self.root))
 
     def get_title(self) -> str:
+        from zimscraperlib.html import find_title_in_file
+
         return find_title_in_file(self.filepath, self.mimetype)
 
 

src/zimscraperlib/zim/items.py

@@ -10,7 +10,6 @@
 
 import libzim.writer  # pyright: ignore[reportMissingModuleSource]
 
-from zimscraperlib.download import stream_file
 from zimscraperlib.filesystem import get_content_mimetype, get_file_mimetype
 from zimscraperlib.zim.indexing import (
     IndexData,
@@ -230,6 +229,8 @@ def download_for_size(
         on_disk: bool,
     ):
         """Download URL to a temp file and return its tempfile and size"""
+        from zimscraperlib.download import stream_file
+
         fpath = stream = None
         if on_disk:
             suffix = pathlib.Path(re.sub(r"^/", "", url.path)).suffix
@@ -254,6 +255,8 @@ def __init__(
         use_disk: bool | None = None,
         **kwargs: Any,
     ):
+        from zimscraperlib.download import stream_file
+
         if use_disk is not None:
             kwargs["use_disk"] = use_disk
         super().__init__(

src/zimscraperlib/zim/providers.py

@@ -13,8 +13,6 @@
 import libzim.writer  # pyright: ignore[reportMissingModuleSource]
 import requests
 
-from zimscraperlib.download import get_retry_adapter, stream_file
-
 
 class FileProvider(libzim.writer.FileProvider):
     def __init__(
@@ -67,6 +65,8 @@ class URLProvider(libzim.writer.ContentProvider):
     Useful for non-indexed content for which feed() is called only once"""
 
     def __init__(self, url: str, size: int | None = None, ref: object | None = None):
+        from zimscraperlib.download import get_retry_adapter
+
         super().__init__()
         self.url = url
         self.size = size if size is not None else self.get_size_of(url)
@@ -79,6 +79,8 @@ def __init__(self, url: str, size: int | None = None, ref: object | None = None)
 
     @staticmethod
     def get_size_of(url: str) -> int | None:
+        from zimscraperlib.download import stream_file
+
         _, headers = stream_file(url, byte_stream=io.BytesIO(), only_first_block=True)
         try:
             return int(headers["Content-Length"])

tests/inputs/test_inputs.py

@@ -3,7 +3,7 @@
 
 import pytest
 
-import zimscraperlib
+import zimscraperlib.download
 from zimscraperlib.constants import CONTACT
 from zimscraperlib.constants import NAME as PROJECT_NAME
 from zimscraperlib.inputs import (
@@ -96,7 +96,7 @@ def mock_stream_file(**kwargs: Any):
         assert user_agent.endswith(f"({CONTACT})")
 
     monkeypatch.setattr(
-        zimscraperlib.inputs,  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType]
+        zimscraperlib.download,  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType]
         "stream_file",
         mock_stream_file,
         raising=True,
@@ -115,7 +115,7 @@ def mock_stream_file(**kwargs: Any):
         assert user_agent == valid_user_agent
 
     monkeypatch.setattr(
-        zimscraperlib.inputs,  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType]
+        zimscraperlib.download,  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType]
         "stream_file",
         mock_stream_file,
         raising=True,
@@ -130,7 +130,7 @@ def mock_stream_file(**kwargs: Any):
         assert kwargs.get("headers") is None
 
     monkeypatch.setattr(
-        zimscraperlib.inputs,  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType]
+        zimscraperlib.download,  # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType]
         "stream_file",
         mock_stream_file,
         raising=True,

tests/test_import_boundaries.py

@@ -0,0 +1,58 @@
+"""Import-graph boundaries between the library's families.
+
+Importing `zim` or `inputs` must not require yt-dlp, beautifulsoup4 or lxml,
+so those are hidden from a child interpreter and the module imported anyway.
+"""
+
+import subprocess
+import sys
+
+SCRIPT = """
+import sys
+
+BLOCKED = {blocked!r}
+
+
+class Blocker:
+    def find_spec(self, fullname, path=None, target=None):
+        if fullname.split(".")[0] in BLOCKED:
+            raise ModuleNotFoundError(f"No module named {{fullname!r}}")
+        return None
+
+
+sys.meta_path.insert(0, Blocker())
+import {module}
+"""
+
+
+def import_with_blocked(
+    module: str, blocked: set[str]
+) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        [sys.executable, "-c", SCRIPT.format(module=module, blocked=blocked)],
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+
+
+def test_zim_import_needs_no_yt_dlp():
+    result = import_with_blocked("zimscraperlib.zim", {"yt_dlp"})
+    assert result.returncode == 0, result.stderr
+
+
+def test_zim_import_needs_no_bs4_or_lxml():
+    result = import_with_blocked("zimscraperlib.zim", {"bs4", "lxml"})
+    assert result.returncode == 0, result.stderr
+
+
+def test_inputs_import_needs_no_yt_dlp():
+    result = import_with_blocked("zimscraperlib.inputs", {"yt_dlp"})
+    assert result.returncode == 0, result.stderr
+
+
+def test_download_does_need_yt_dlp():
+    """The blocker works, so the tests above are not vacuous"""
+    result = import_with_blocked("zimscraperlib.download", {"yt_dlp"})
+    assert result.returncode != 0
+    assert "No module named 'yt_dlp'" in result.stderr