Fix title extraction truncated at the first character reference

CHANGELOG.md

@@ -17,6 +17,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix JS URL rewriter confusing a percent-encoded `%2B` with a raw `+` in query strings (#389)
 - Patch vendored `wombat.js` so `retrieveWBOSRCAsURL` (the helper behind the above element-property fix) resolves a recorded original against the frame's true original URL (`wb_info.url`) instead of its replay-time `ownerDocument`, fixing it returning/using a URL under the wrong host for a page whose own URL was changed by a fuzzy rule (e.g. YouTube embeds), which broke playback (#398)
 - Add a host-gated `SITE_JS_PATCHES` mechanism to `JsRewriter`, fixing MDN's `<interactive-example>` widget missing a deterministic `permalink` attribute (#394)
+- Fix `HtmlRewriter` title extraction so it reassembles the whole text of a `<title>` containing character references (`&amp;`, `&#039;`, `&#x27;`, …), which were truncated at the first one, and the ZIM entry titles built from it (#401)
 
 ## [5.5.2] - 2026-11-06
 

src/zimscraperlib/rewriting/html.py

@@ -18,7 +18,7 @@
 from collections.abc import Callable
 from dataclasses import dataclass
 from functools import cache
-from html import escape
+from html import escape, unescape
 from html.parser import HTMLParser
 from inspect import Signature, signature
 from typing import Any, NamedTuple
@@ -194,6 +194,7 @@ def __init__(
         super().__init__(convert_charrefs=False)
         self.url_rewriter = url_rewriter
         self.title = None
+        self.title_chunks: list[str] = []
         self.output = None
         # This works only for tag without children.
         # But as we use it to get the title, we are ok
@@ -275,6 +276,8 @@ def handle_starttag(self, tag: str, attrs: AttrsList, *, auto_close: bool = Fals
 
     def handle_endtag(self, tag: str):
         """Overwrite handle_endtag from HTMLParser"""
+        if tag == "title":
+            self._save_title()
         self.html_rewrite_context = None
         if tag == "head" and self.post_head_insert:
             self.send(self.post_head_insert)
@@ -287,8 +290,7 @@ def handle_startendtag(self, tag: str, attrs: AttrsList):
 
     def handle_data(self, data: str):
         """Overwrite handle_data from HTMLParser"""
-        if self.html_rewrite_context == "title" and self.title is None:
-            self.title = data.strip()
+        self._collect_title_data(data)
         if (
             data.strip()
             and (
@@ -308,10 +310,12 @@ def handle_data(self, data: str):
 
     def handle_entityref(self, name: str):
         """Overwrite handle_entityref from HTMLParser"""
+        self._collect_title_data(unescape(f"&{name};"))
         self.send(f"&{name};")
 
     def handle_charref(self, name: str):
         """Overwrite handle_charref from HTMLParser"""
+        self._collect_title_data(unescape(f"&#{name};"))
         self.send(f"&#{name};")
 
     def handle_comment(self, data: str):
@@ -330,6 +334,33 @@ def unknown_decl(self, data: str):
         """Overwrite unknown_decl from HTMLParser"""
         self.handle_decl(data)  # pragma: no cover
 
+    def close(self):
+        """Overwrite close from HTMLParser
+
+        Also saves a title whose closing tag is missing from the document.
+        """
+        super().close()
+        self._save_title()
+
+    def _collect_title_data(self, data: str) -> None:
+        """Collect one piece of the text of the title currently being read
+
+        The parser is fed with character reference conversion disabled, so the text of a
+        title containing one or more of them is handed over in several pieces.
+        """
+        if self.html_rewrite_context == "title" and self.title is None:
+            self.title_chunks.append(data)
+
+    def _save_title(self) -> None:
+        """Keep the whole text of the first title which has some
+
+        A title element without text leaves the title unset, so that a later title,
+        typically the one of an inline SVG, still gets used.
+        """
+        if self.title is not None or not self.title_chunks:
+            return
+        self.title = "".join(self.title_chunks).strip()
+
 
 DropAttributeCallable = Callable[..., bool]
 RewriteAttributeCallable = Callable[..., AttrNameAndValue | None]

tests/rewriting/test_html_rewriting.py

@@ -9,6 +9,7 @@
     AttrsList,
     HtmlRewriter,
     HTMLRewritingRules,
+    RewritenHtml,
     extract_base_href,
     format_attr,
     get_attr_value_from,
@@ -391,6 +392,68 @@ def test_extract_title():
     )
 
 
+def rewritten_html(content: str) -> RewritenHtml:
+    return HtmlRewriter(
+        ArticleUrlRewriter(article_url=HttpUrl("http://kiwix.org/")),
+        None,
+        None,
+        None,
+    ).rewrite(content)
+
+
+@pytest.mark.parametrize(
+    "content,expected_title",
+    [
+        ("<html><head><title>Tom &amp; Jerry</title></head></html>", "Tom & Jerry"),
+        (
+            "<html><head><title>D&eacute;j&agrave; Vu &amp; More</title></head></html>",
+            "Déjà Vu & More",
+        ),
+        (
+            "<html><head><title>L&#039;Illustre Maurin</title></head></html>",
+            "L'Illustre Maurin",
+        ),
+        (
+            "<html><head><title>L&#x27;Illustre Maurin</title></head></html>",
+            "L'Illustre Maurin",
+        ),
+        (
+            "<html><head><title>Nice &amp; clean &#38; fresh</title></head></html>",
+            "Nice & clean & fresh",
+        ),
+        ("<html><head><title>Rock & Roll</title></head></html>", "Rock & Roll"),
+        (
+            "<html><head><title>  spaced &amp; out  </title></head></html>",
+            "spaced & out",
+        ),
+        (
+            (
+                "<html><head><title>Tom &amp; Jerry</title></head>"
+                "<body><title>Second &amp; only</title></body></html>"
+            ),
+            "Tom & Jerry",
+        ),
+        (
+            (
+                "<html><head><title></title></head>"
+                "<body><svg><title>SVG &amp; only</title></svg></body></html>"
+            ),
+            "SVG & only",
+        ),
+        ("<html><head><title>Unclosed &amp; still", "Unclosed & still"),
+    ],
+)
+def test_extract_title_with_char_refs(content: str, expected_title: str):
+    assert rewritten_html(content).title == expected_title
+
+
+def test_extract_title_keeps_char_refs_in_content():
+    content = "<html><head><title>Tom &amp; Jerry &eacute; 2026</title></head></html>"
+    rewritten = rewritten_html(content)
+    assert rewritten.title == "Tom & Jerry é 2026"
+    assert rewritten.content == content
+
+
 def test_rewrite_attributes():
     rewriter = HtmlRewriter(
         ArticleUrlRewriter(