diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b3e14c..6953058 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix JS URL rewriter confusing a percent-encoded `%2B` with a raw `+` in query strings (#389) - Patch vendored `wombat.js` so `retrieveWBOSRCAsURL` (the helper behind the above element-property fix) resolves a recorded original against the frame's true original URL (`wb_info.url`) instead of its replay-time `ownerDocument`, fixing it returning/using a URL under the wrong host for a page whose own URL was changed by a fuzzy rule (e.g. YouTube embeds), which broke playback (#398) - Add a host-gated `SITE_JS_PATCHES` mechanism to `JsRewriter`, fixing MDN's `` widget missing a deterministic `permalink` attribute (#394) +- Fix `HtmlRewriter` title extraction so it reassembles the whole text of a `` containing character references (`&`, `'`, `'`, …), which were truncated at the first one, and the ZIM entry titles built from it (#401) ## [5.5.2] - 2026-11-06 diff --git a/src/zimscraperlib/rewriting/html.py b/src/zimscraperlib/rewriting/html.py index 35247d4..e9e997b 100644 --- a/src/zimscraperlib/rewriting/html.py +++ b/src/zimscraperlib/rewriting/html.py @@ -18,7 +18,7 @@ from collections.abc import Callable from dataclasses import dataclass from functools import cache -from html import escape +from html import escape, unescape from html.parser import HTMLParser from inspect import Signature, signature from typing import Any, NamedTuple @@ -194,6 +194,7 @@ def __init__( super().__init__(convert_charrefs=False) self.url_rewriter = url_rewriter self.title = None + self.title_chunks: list[str] = [] self.output = None # This works only for tag without children. # But as we use it to get the title, we are ok @@ -275,6 +276,8 @@ def handle_starttag(self, tag: str, attrs: AttrsList, *, auto_close: bool = Fals def handle_endtag(self, tag: str): """Overwrite handle_endtag from HTMLParser""" + if tag == "title": + self._save_title() self.html_rewrite_context = None if tag == "head" and self.post_head_insert: self.send(self.post_head_insert) @@ -287,8 +290,7 @@ def handle_startendtag(self, tag: str, attrs: AttrsList): def handle_data(self, data: str): """Overwrite handle_data from HTMLParser""" - if self.html_rewrite_context == "title" and self.title is None: - self.title = data.strip() + self._collect_title_data(data) if ( data.strip() and ( @@ -308,10 +310,12 @@ def handle_data(self, data: str): def handle_entityref(self, name: str): """Overwrite handle_entityref from HTMLParser""" + self._collect_title_data(unescape(f"&{name};")) self.send(f"&{name};") def handle_charref(self, name: str): """Overwrite handle_charref from HTMLParser""" + self._collect_title_data(unescape(f"&#{name};")) self.send(f"&#{name};") def handle_comment(self, data: str): @@ -330,6 +334,33 @@ def unknown_decl(self, data: str): """Overwrite unknown_decl from HTMLParser""" self.handle_decl(data) # pragma: no cover + def close(self): + """Overwrite close from HTMLParser + + Also saves a title whose closing tag is missing from the document. + """ + super().close() + self._save_title() + + def _collect_title_data(self, data: str) -> None: + """Collect one piece of the text of the title currently being read + + The parser is fed with character reference conversion disabled, so the text of a + title containing one or more of them is handed over in several pieces. + """ + if self.html_rewrite_context == "title" and self.title is None: + self.title_chunks.append(data) + + def _save_title(self) -> None: + """Keep the whole text of the first title which has some + + A title element without text leaves the title unset, so that a later title, + typically the one of an inline SVG, still gets used. + """ + if self.title is not None or not self.title_chunks: + return + self.title = "".join(self.title_chunks).strip() + DropAttributeCallable = Callable[..., bool] RewriteAttributeCallable = Callable[..., AttrNameAndValue | None] diff --git a/tests/rewriting/test_html_rewriting.py b/tests/rewriting/test_html_rewriting.py index 914bcb8..5bfce73 100644 --- a/tests/rewriting/test_html_rewriting.py +++ b/tests/rewriting/test_html_rewriting.py @@ -9,6 +9,7 @@ AttrsList, HtmlRewriter, HTMLRewritingRules, + RewritenHtml, extract_base_href, format_attr, get_attr_value_from, @@ -391,6 +392,68 @@ def test_extract_title(): ) +def rewritten_html(content: str) -> RewritenHtml: + return HtmlRewriter( + ArticleUrlRewriter(article_url=HttpUrl("http://kiwix.org/")), + None, + None, + None, + ).rewrite(content) + + +@pytest.mark.parametrize( + "content,expected_title", + [ + ("<html><head><title>Tom & Jerry", "Tom & Jerry"), + ( + "Déjà Vu & More", + "Déjà Vu & More", + ), + ( + "L'Illustre Maurin", + "L'Illustre Maurin", + ), + ( + "L'Illustre Maurin", + "L'Illustre Maurin", + ), + ( + "Nice & clean & fresh", + "Nice & clean & fresh", + ), + ("Rock & Roll", "Rock & Roll"), + ( + " spaced & out ", + "spaced & out", + ), + ( + ( + "Tom & Jerry" + "Second & only" + ), + "Tom & Jerry", + ), + ( + ( + "" + "SVG & only" + ), + "SVG & only", + ), + ("Unclosed & still", "Unclosed & still"), + ], +) +def test_extract_title_with_char_refs(content: str, expected_title: str): + assert rewritten_html(content).title == expected_title + + +def test_extract_title_keeps_char_refs_in_content(): + content = "<html><head><title>Tom & Jerry é 2026" + rewritten = rewritten_html(content) + assert rewritten.title == "Tom & Jerry é 2026" + assert rewritten.content == content + + def test_rewrite_attributes(): rewriter = HtmlRewriter( ArticleUrlRewriter(