From efc73b4860579fe54e4b5f118d5e6a3fbfec9b10 Mon Sep 17 00:00:00 2001 From: pawank925 Date: Wed, 11 Nov 2026 20:34:35 +0530 Subject: [PATCH 1/2] Fix title extraction truncated at the first character reference HtmlRewriter parses with character reference conversion disabled so that references are re-emitted verbatim in the rewritten content. A consequence is that the text of a reaches the parser in several pieces, of which only the first was kept: "Tom & Jerry" gave "Tom" and "L'Illustre Maurin" gave "L". warc2zim uses this value as the ZIM entry title, so any title with an ampersand or an escaped apostrophe came out truncated in title search and suggestions. Title pieces are now collected, with references decoded, and joined when the element closes. Rewritten content is untouched, and the first title which has text still wins: a document with an empty title in head keeps using the title of a later inline SVG. A document whose title element was never closed now gets its whole title instead of the first piece. --- CHANGELOG.md | 1 + src/zimscraperlib/rewriting/html.py | 37 +++++++++++++-- tests/rewriting/test_html_rewriting.py | 63 ++++++++++++++++++++++++++ 3 files changed, 98 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b3e14c..6953058 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Fix JS URL rewriter confusing a percent-encoded `%2B` with a raw `+` in query strings (#389) - Patch vendored `wombat.js` so `retrieveWBOSRCAsURL` (the helper behind the above element-property fix) resolves a recorded original against the frame's true original URL (`wb_info.url`) instead of its replay-time `ownerDocument`, fixing it returning/using a URL under the wrong host for a page whose own URL was changed by a fuzzy rule (e.g. YouTube embeds), which broke playback (#398) - Add a host-gated `SITE_JS_PATCHES` mechanism to `JsRewriter`, fixing MDN's `<interactive-example>` widget missing a deterministic `permalink` attribute (#394) +- Fix `HtmlRewriter` title extraction so it reassembles the whole text of a `<title>` containing character references (`&`, `'`, `'`, …), which were truncated at the first one, and the ZIM entry titles built from it (#401) ## [5.5.2] - 2026-11-06 diff --git a/src/zimscraperlib/rewriting/html.py b/src/zimscraperlib/rewriting/html.py index 35247d4..e9e997b 100644 --- a/src/zimscraperlib/rewriting/html.py +++ b/src/zimscraperlib/rewriting/html.py @@ -18,7 +18,7 @@ from collections.abc import Callable from dataclasses import dataclass from functools import cache -from html import escape +from html import escape, unescape from html.parser import HTMLParser from inspect import Signature, signature from typing import Any, NamedTuple @@ -194,6 +194,7 @@ def __init__( super().__init__(convert_charrefs=False) self.url_rewriter = url_rewriter self.title = None + self.title_chunks: list[str] = [] self.output = None # This works only for tag without children. # But as we use it to get the title, we are ok @@ -275,6 +276,8 @@ def handle_starttag(self, tag: str, attrs: AttrsList, *, auto_close: bool = Fals def handle_endtag(self, tag: str): """Overwrite handle_endtag from HTMLParser""" + if tag == "title": + self._save_title() self.html_rewrite_context = None if tag == "head" and self.post_head_insert: self.send(self.post_head_insert) @@ -287,8 +290,7 @@ def handle_startendtag(self, tag: str, attrs: AttrsList): def handle_data(self, data: str): """Overwrite handle_data from HTMLParser""" - if self.html_rewrite_context == "title" and self.title is None: - self.title = data.strip() + self._collect_title_data(data) if ( data.strip() and ( @@ -308,10 +310,12 @@ def handle_data(self, data: str): def handle_entityref(self, name: str): """Overwrite handle_entityref from HTMLParser""" + self._collect_title_data(unescape(f"&{name};")) self.send(f"&{name};") def handle_charref(self, name: str): """Overwrite handle_charref from HTMLParser""" + self._collect_title_data(unescape(f"&#{name};")) self.send(f"&#{name};") def handle_comment(self, data: str): @@ -330,6 +334,33 @@ def unknown_decl(self, data: str): """Overwrite unknown_decl from HTMLParser""" self.handle_decl(data) # pragma: no cover + def close(self): + """Overwrite close from HTMLParser + + Also saves a title whose closing tag is missing from the document. + """ + super().close() + self._save_title() + + def _collect_title_data(self, data: str) -> None: + """Collect one piece of the text of the title currently being read + + The parser is fed with character reference conversion disabled, so the text of a + title containing one or more of them is handed over in several pieces. + """ + if self.html_rewrite_context == "title" and self.title is None: + self.title_chunks.append(data) + + def _save_title(self) -> None: + """Keep the whole text of the first title which has some + + A title element without text leaves the title unset, so that a later title, + typically the one of an inline SVG, still gets used. + """ + if self.title is not None or not self.title_chunks: + return + self.title = "".join(self.title_chunks).strip() + DropAttributeCallable = Callable[..., bool] RewriteAttributeCallable = Callable[..., AttrNameAndValue | None] diff --git a/tests/rewriting/test_html_rewriting.py b/tests/rewriting/test_html_rewriting.py index 914bcb8..5bfce73 100644 --- a/tests/rewriting/test_html_rewriting.py +++ b/tests/rewriting/test_html_rewriting.py @@ -9,6 +9,7 @@ AttrsList, HtmlRewriter, HTMLRewritingRules, + RewritenHtml, extract_base_href, format_attr, get_attr_value_from, @@ -391,6 +392,68 @@ def test_extract_title(): ) +def rewritten_html(content: str) -> RewritenHtml: + return HtmlRewriter( + ArticleUrlRewriter(article_url=HttpUrl("http://kiwix.org/")), + None, + None, + None, + ).rewrite(content) + + +@pytest.mark.parametrize( + "content,expected_title", + [ + ("<html><head><title>Tom & Jerry", "Tom & Jerry"), + ( + "Déjà Vu & More", + "Déjà Vu & More", + ), + ( + "L'Illustre Maurin", + "L'Illustre Maurin", + ), + ( + "L'Illustre Maurin", + "L'Illustre Maurin", + ), + ( + "Nice & clean & fresh", + "Nice & clean & fresh", + ), + ("Rock & Roll", "Rock & Roll"), + ( + " spaced & out ", + "spaced & out", + ), + ( + ( + "Tom & Jerry" + "Second & only" + ), + "Tom & Jerry", + ), + ( + ( + "" + "SVG & only" + ), + "SVG & only", + ), + ("Unclosed & still", "Unclosed & still"), + ], +) +def test_extract_title_with_char_refs(content: str, expected_title: str): + assert rewritten_html(content).title == expected_title + + +def test_extract_title_keeps_char_refs_in_content(): + content = "<html><head><title>Tom & Jerry é 2026" + rewritten = rewritten_html(content) + assert rewritten.title == "Tom & Jerry é 2026" + assert rewritten.content == content + + def test_rewrite_attributes(): rewriter = HtmlRewriter( ArticleUrlRewriter( From ba828638edd3862dd0cf24d2c7166f432dac16f4 Mon Sep 17 00:00:00 2001 From: pawank925 Date: Thu, 12 Nov 2026 07:12:54 +0530 Subject: [PATCH 2/2] Trigger CI