Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Detect `export const`/`let`/`var`/`function`/`async function`/`*` as module code (`EXPORT_RX` only recognized `export {...}`, `export default` and `export class`), fixing JS rewriting wrongly injecting a non-module `let arguments;` wrapper into ES modules using these common export forms, which is a syntax error in strict/module code (#387)
- Fix JS URL rewriter confusing a percent-encoded `%2B` with a raw `+` in query strings (#389)
- Patch vendored `wombat.js` so `retrieveWBOSRCAsURL` (the helper behind the above element-property fix) resolves a recorded original against the frame's true original URL (`wb_info.url`) instead of its replay-time `ownerDocument`, fixing it returning/using a URL under the wrong host for a page whose own URL was changed by a fuzzy rule (e.g. YouTube embeds), which broke playback (#398)
- Add a host-gated `SITE_JS_PATCHES` mechanism to `JsRewriter`, fixing MDN's `<interactive-example>` widget missing a deterministic `permalink` attribute (#394)

## [5.5.2] - 2026-11-06

Expand Down
50 changes: 50 additions & 0 deletions src/zimscraperlib/rewriting/js.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import re
from collections.abc import Callable, Iterable
from typing import Any, Literal
from urllib.parse import urlsplit

from zimscraperlib.rewriting.js_ast import parse_top_level
from zimscraperlib.rewriting.rx_replacer import (
Expand Down Expand Up @@ -267,6 +268,52 @@ def create_js_rules() -> list[TransformationRule]:

REWRITE_JS_RULES = create_js_rules()

# Site-specific content patches: narrow, deliberately host-gated fixes for a
# particular known-broken behavior on a given site, applied to the raw JS
# text before any other rewriting. Keyed by exact hostname (the host of the
# article URL currently being rewritten, i.e. this JS file's own URL) so a
# patch can never affect any other site's content, even if its marker
# string were to appear there by coincidence.
#
# Each entry is (marker, pattern, replacement): `marker` is a cheap
# substring check, tested before the regex runs, so that a visit to this
# host only pays for a `str.find` on the (vast majority of) its files that
# don't match and don't need the patch at all.
SiteJsPatch = tuple[str, re.Pattern[str], str]

SITE_JS_PATCHES: dict[str, list[SiteJsPatch]] = {
"developer.mozilla.org": [
# developer.mozilla.org's top "Try it" widget (<interactive-example>)
# renders its inner <mdn-play-runner> without the `permalink`
# attribute, so that component falls back to crypto.randomUUID() for
# a session-random id instead of the content-derived hash
# <mdn-live-sample-result> uses elsewhere on the same page (whose own
# template already always sets `permalink`). A fresh random id can
# never match anything captured in a WARC, so the widget fails to
# load on replay. The *initial*, pre-edit code is just as
# content-stable as the other live-sample widgets that already work;
# only the id source (hash vs crypto.randomUUID()) differs. Forcing
# `permalink` here makes it use that same deterministic,
# content-derived id (live editing by a reader still won't resolve
# beyond that initial state, same as it already wouldn't without this
# patch -- no regression, since that's the same inherent limitation
# as any other dynamically-generated URL).
(
'customElements.define("interactive-example"',
re.compile(r"<mdn-play-runner(\s)"),
r"<mdn-play-runner permalink\1",
),
],
}


def apply_site_js_patches(text: str, host: str) -> str:
"""Apply any narrow, host-gated content patches registered for `host`."""
for marker, pattern, replacement in SITE_JS_PATCHES.get(host, []):
if marker in text:
text = pattern.sub(replacement, text)
return text


class JsRewriter(RxRewriter):
"""
Expand All @@ -285,6 +332,7 @@ def __init__(
self.url_rewriter = url_rewriter
self.notify_js_module = notify_js_module
self.base_href = base_href
self.host = urlsplit(url_rewriter.article_url.value).hostname or ""

def _init_local_declaration(self, local_decls: Iterable[str]) -> str:
"""
Expand Down Expand Up @@ -341,6 +389,8 @@ def rewrite(self, text: str | bytes, opts: dict[str, Any] | None = None) -> str:
if isinstance(text, bytes):
text = text.decode()

text = apply_site_js_patches(text, self.host)

opts = opts or {}

if "isModule" not in opts:
Expand Down
64 changes: 64 additions & 0 deletions tests/rewriting/test_js_rewriting.py
Original file line number Diff line number Diff line change
Expand Up @@ -641,3 +641,67 @@ def custom_notify(zim_path: ZimPath):

assert len(js_modules) == 1
assert js_modules[0].value == expected_js_module_path


# Marker string present in the test fixtures below, matching the one real
# chunk this patch targets (customElements.define("interactive-example"...).
MDN_MARKER = 'customElements.define("interactive-example",X)'


def _rewrite_as(url: str, text: str) -> str:
return JsRewriter(
url_rewriter=ArticleUrlRewriter(article_url=HttpUrl(url)),
base_href=None,
notify_js_module=None,
).rewrite(text)


def test_mdn_permalink_patch_applies_on_mdn_host():
text = f'{MDN_MARKER};x=<mdn-play-runner foo="bar"></mdn-play-runner>;'
result = _rewrite_as("https://developer.mozilla.org/static/client/x.js", text)
assert "<mdn-play-runner permalink foo=" in result


def test_mdn_permalink_patch_does_not_apply_on_other_hosts():
# Same content, same marker, different host: must NOT be patched. This is
# the whole point of gating SITE_JS_PATCHES by exact hostname.
text = f'{MDN_MARKER};x=<mdn-play-runner foo="bar"></mdn-play-runner>;'
result = _rewrite_as("https://example.com/static/client/x.js", text)
assert "permalink" not in result
assert result == text


def test_mdn_permalink_patch_does_not_apply_on_lookalike_hosts():
# Hostname match must be exact, not a substring/prefix/suffix match, so a
# completely unrelated host that merely contains "developer.mozilla.org"
# as a substring is never affected.
text = f'{MDN_MARKER};x=<mdn-play-runner foo="bar"></mdn-play-runner>;'
for url in [
"https://developer.mozilla.org.evil.example/static/client/x.js",
"https://notdeveloper.mozilla.org/static/client/x.js",
"https://developer.mozilla.org.example.com/static/client/x.js",
]:
result = _rewrite_as(url, text)
assert "permalink" not in result, url


def test_mdn_permalink_patch_requires_marker():
# Right host, but the file doesn't define <interactive-example> (e.g.
# this is <mdn-live-sample-result>'s own chunk, which already always
# sets `permalink` itself): the patch must not touch unrelated chunks
# that merely happen to also mention <mdn-play-runner>.
text = 'x=<mdn-play-runner foo="bar"></mdn-play-runner>;'
result = _rewrite_as("https://developer.mozilla.org/static/client/x.js", text)
assert result == text


def test_mdn_permalink_patch_does_not_touch_closing_tag_or_css_selector():
text = (
f"{MDN_MARKER};"
'<mdn-play-runner foo="bar"></mdn-play-runner>'
".template-console mdn-play-runner{display:none}"
)
result = _rewrite_as("https://developer.mozilla.org/static/client/x.js", text)
assert result.count("permalink") == 1
assert "</mdn-play-runner permalink>" not in result
assert "mdn-play-runner{display:none}" in result