diff --git a/CHANGELOG.md b/CHANGELOG.md index 2034c694..412d1994 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `VideoMp4High` now encodes at CRF 28 instead of 20, which regularly produced a file larger than its input, and adds `-movflags +faststart` (#271) - All Webm presets (`VideoWebmLow`, `VideoWebmMedium`, `VideoWebmHigh`) now use `-row-mt 1` for faster VP9 encoding (#271) - Bumped `VideoWebmLow` to v4 and `VideoWebmHigh` to v3 following the `-row-mt` addition (#271) +- `import zimscraperlib.zim` no longer imports yt-dlp, beautifulsoup4 or lxml, and `import zimscraperlib.inputs` no longer imports yt-dlp: `stream_file`, `get_retry_adapter` and `find_title_in_file` are now imported where they are used (#244) ### Fixed diff --git a/pyproject.toml b/pyproject.toml index 5b9cb159..44dc40eb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -234,6 +234,9 @@ ignore = [ "S603", # Ignore complexity "C901", "PLR0911", "PLR0912", "PLR0913", "PLR0915", + # Modules of optional dependencies are imported where they are used so that + # importing a family does not load another family's dependencies + "PLC0415", # To be fixed once more recent Python versions are available "UP006", "UP007" ] diff --git a/src/zimscraperlib/inputs.py b/src/zimscraperlib/inputs.py index 19472868..5f1abb56 100644 --- a/src/zimscraperlib/inputs.py +++ b/src/zimscraperlib/inputs.py @@ -11,7 +11,6 @@ from zimscraperlib.constants import ( MAXIMUM_LONG_DESCRIPTION_METADATA_LENGTH as MAX_LONG_DESC_LENGTH, ) -from zimscraperlib.download import stream_file def handle_user_provided_file( @@ -41,6 +40,8 @@ def handle_user_provided_file( ) if str(source).startswith("http"): + from zimscraperlib.download import stream_file + logger.debug(f"download {source} -> {dest}") headers = {"User-Agent": user_agent} if user_agent else None stream_file(url=str(source), fpath=dest, headers=headers) diff --git a/src/zimscraperlib/zim/filesystem.py b/src/zimscraperlib/zim/filesystem.py index 10a12790..5493369c 100644 --- a/src/zimscraperlib/zim/filesystem.py +++ b/src/zimscraperlib/zim/filesystem.py @@ -31,7 +31,6 @@ from zimscraperlib import logger from zimscraperlib.filesystem import get_file_mimetype -from zimscraperlib.html import find_title_in_file from zimscraperlib.types import get_mime_for_name from zimscraperlib.zim import metadata from zimscraperlib.zim.creator import Creator @@ -59,6 +58,8 @@ def get_path(self) -> str: return str(self.filepath.relative_to(self.root)) def get_title(self) -> str: + from zimscraperlib.html import find_title_in_file + return find_title_in_file(self.filepath, self.mimetype) diff --git a/src/zimscraperlib/zim/items.py b/src/zimscraperlib/zim/items.py index 418cc8a1..02d2a1f8 100644 --- a/src/zimscraperlib/zim/items.py +++ b/src/zimscraperlib/zim/items.py @@ -10,7 +10,6 @@ import libzim.writer # pyright: ignore[reportMissingModuleSource] -from zimscraperlib.download import stream_file from zimscraperlib.filesystem import get_content_mimetype, get_file_mimetype from zimscraperlib.zim.indexing import ( IndexData, @@ -230,6 +229,8 @@ def download_for_size( on_disk: bool, ): """Download URL to a temp file and return its tempfile and size""" + from zimscraperlib.download import stream_file + fpath = stream = None if on_disk: suffix = pathlib.Path(re.sub(r"^/", "", url.path)).suffix @@ -254,6 +255,8 @@ def __init__( use_disk: bool | None = None, **kwargs: Any, ): + from zimscraperlib.download import stream_file + if use_disk is not None: kwargs["use_disk"] = use_disk super().__init__( diff --git a/src/zimscraperlib/zim/providers.py b/src/zimscraperlib/zim/providers.py index 6e700ffd..f62db392 100644 --- a/src/zimscraperlib/zim/providers.py +++ b/src/zimscraperlib/zim/providers.py @@ -13,8 +13,6 @@ import libzim.writer # pyright: ignore[reportMissingModuleSource] import requests -from zimscraperlib.download import get_retry_adapter, stream_file - class FileProvider(libzim.writer.FileProvider): def __init__( @@ -67,6 +65,8 @@ class URLProvider(libzim.writer.ContentProvider): Useful for non-indexed content for which feed() is called only once""" def __init__(self, url: str, size: int | None = None, ref: object | None = None): + from zimscraperlib.download import get_retry_adapter + super().__init__() self.url = url self.size = size if size is not None else self.get_size_of(url) @@ -79,6 +79,8 @@ def __init__(self, url: str, size: int | None = None, ref: object | None = None) @staticmethod def get_size_of(url: str) -> int | None: + from zimscraperlib.download import stream_file + _, headers = stream_file(url, byte_stream=io.BytesIO(), only_first_block=True) try: return int(headers["Content-Length"]) diff --git a/tests/inputs/test_inputs.py b/tests/inputs/test_inputs.py index 652d599c..f7684810 100644 --- a/tests/inputs/test_inputs.py +++ b/tests/inputs/test_inputs.py @@ -3,7 +3,7 @@ import pytest -import zimscraperlib +import zimscraperlib.download from zimscraperlib.constants import CONTACT from zimscraperlib.constants import NAME as PROJECT_NAME from zimscraperlib.inputs import ( @@ -96,7 +96,7 @@ def mock_stream_file(**kwargs: Any): assert user_agent.endswith(f"({CONTACT})") monkeypatch.setattr( - zimscraperlib.inputs, # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType] + zimscraperlib.download, # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType] "stream_file", mock_stream_file, raising=True, @@ -115,7 +115,7 @@ def mock_stream_file(**kwargs: Any): assert user_agent == valid_user_agent monkeypatch.setattr( - zimscraperlib.inputs, # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType] + zimscraperlib.download, # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType] "stream_file", mock_stream_file, raising=True, @@ -130,7 +130,7 @@ def mock_stream_file(**kwargs: Any): assert kwargs.get("headers") is None monkeypatch.setattr( - zimscraperlib.inputs, # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType] + zimscraperlib.download, # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType, reportUnknownArgumentType] "stream_file", mock_stream_file, raising=True, diff --git a/tests/test_import_boundaries.py b/tests/test_import_boundaries.py new file mode 100644 index 00000000..a421094a --- /dev/null +++ b/tests/test_import_boundaries.py @@ -0,0 +1,58 @@ +"""Import-graph boundaries between the library's families. + +Importing `zim` or `inputs` must not require yt-dlp, beautifulsoup4 or lxml, +so those are hidden from a child interpreter and the module imported anyway. +""" + +import subprocess +import sys + +SCRIPT = """ +import sys + +BLOCKED = {blocked!r} + + +class Blocker: + def find_spec(self, fullname, path=None, target=None): + if fullname.split(".")[0] in BLOCKED: + raise ModuleNotFoundError(f"No module named {{fullname!r}}") + return None + + +sys.meta_path.insert(0, Blocker()) +import {module} +""" + + +def import_with_blocked( + module: str, blocked: set[str] +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-c", SCRIPT.format(module=module, blocked=blocked)], + capture_output=True, + text=True, + check=False, + ) + + +def test_zim_import_needs_no_yt_dlp(): + result = import_with_blocked("zimscraperlib.zim", {"yt_dlp"}) + assert result.returncode == 0, result.stderr + + +def test_zim_import_needs_no_bs4_or_lxml(): + result = import_with_blocked("zimscraperlib.zim", {"bs4", "lxml"}) + assert result.returncode == 0, result.stderr + + +def test_inputs_import_needs_no_yt_dlp(): + result = import_with_blocked("zimscraperlib.inputs", {"yt_dlp"}) + assert result.returncode == 0, result.stderr + + +def test_download_does_need_yt_dlp(): + """The blocker works, so the tests above are not vacuous""" + result = import_with_blocked("zimscraperlib.download", {"yt_dlp"}) + assert result.returncode != 0 + assert "No module named 'yt_dlp'" in result.stderr