Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added

- `ArticleUrlRewriter.set_zim_path_prefix()`: an opt-in ZIM path prefix which stores every ZIM path under it and makes rewritten URIs walk up to it instead of stopping at the closest common directory. A rewritten URI then always contains the prefix as a literal segment, so a replay reader can tell a rewritten URL from an original one (and recover its ZIM path) without relying on the `__wb_orig_*` attributes recorded at scrape time. Disabled by default, keeping the current unprefixed entry paths and "as-up-as-needed" URIs (#396)

### Fixed

- Fix JS rewriting of `this.<prop>` so it reliably detects a leading newline (and prepends the needed `;`) regardless of the amount of whitespace, and no longer wrongly rewrites `this` when preceded by `.`/`$` and several spaces (#381)
Expand Down
9 changes: 9 additions & 0 deletions docs/technical_architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,15 @@ Below is an example case of the rewrite operation on an image URL found in an HT
- Image rewritten URL: `../../../ex%C3%A9mple.com/a/resource/image.png%3Ffoo=bar`
- Image ZIM Path: `exémple.com/a/resource/image.png?foo=bar`

`ArticleUrlRewriter.set_zim_path_prefix("zimreplay")` is an opt-in variant of this scheme:

- every ZIM path gains the prefix, e.g. `kiwix.org/a/article/document.html` becomes `zimreplay/kiwix.org/a/article/document.html`. Scrapers have to store their entries under those prefixed paths.
- a rewritten URI walks all the way up to the ZIM root and comes back down through the prefix, instead of stopping as soon as it can: the image above is then addressed as `../../../../zimreplay/ex%C3%A9mple.com/a/resource/image.png%3Ffoo=bar`.

The point is that a rewritten URI then always contains the prefix as a literal segment, while an original URI found in the document cannot. A replay reader (wombat) can hence tell the two apart and recover the ZIM path from a rewritten URI alone, without relying on the `__wb_orig_*` attributes recorded at scrape time for the cases where it cannot.

The prefix is disabled by default, and its validation is deliberately strict (one unencoded URI path segment, made of `a-z`, `0-9`, `.`, `-`, `_`) because it is the string readers have to match.

## JS Rewriting

JS Rewriting is a bit special because rules to apply are different wether we are using "classic" Javascript or "module" Javascript.
Expand Down
53 changes: 50 additions & 3 deletions src/zimscraperlib/rewriting/url_rewriting.py
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,16 @@
and not url-encoded. '=' and ',' are kept unencoded because some sites (e.g. YouTube)
embed scripts that parse their own script URL and expect these characters to appear
literally in it.

A replay reader looking at a rewritten URI cannot tell it from an original relative one
(`./page2.html` in `page1.html` is both a valid original link and a valid rewritten
link), which is why the original value of every rewritten URL attribute has to be
recorded separately. Setting `ArticleUrlRewriter.zim_path_prefix` removes that
ambiguity: every ZIM path is then stored under that prefix and every rewritten URI
walks all the way up to the ZIM root before coming back down through it, so a rewritten
URI always contains the prefix as a literal path segment
(`../../../zimreplay/example.com/page2.html`) while an original one cannot. It is off by
default, which keeps the historical "as-up-as-needed" URIs and unprefixed entry paths.
"""

import re
Expand Down Expand Up @@ -78,6 +88,8 @@ class AdditionalRule:
for rule in FUZZY_RULES
]

ZIM_PATH_PREFIX_RX = re.compile(r"[a-zA-Z0-9._-]+")


def _remove_dot_segments(path: str) -> str:
"""Remove '.' and '..' segments from a path, per RFC 3986 section 5.2.4
Expand Down Expand Up @@ -246,6 +258,31 @@ class ArticleUrlRewriter:
"""

additional_rules: ClassVar[list[AdditionalRule]] = COMPILED_FUZZY_RULES
zim_path_prefix: ClassVar[str | None] = None

@classmethod
def set_zim_path_prefix(cls, zim_path_prefix: str | None) -> None:
"""Store every ZIM path under `zim_path_prefix` and root-anchor URIs on it

Both halves of the behaviour are set together: entry paths gain the prefix, and
rewritten URIs are made to walk up to it, which is what makes a rewritten URI
recognizable. Passing None (or an empty string) restores the default unprefixed
entry paths and "as-up-as-needed" URIs.

The prefix must be one URI path segment of `a-z`, `0-9`, `.`, `-` or `_`, kept
unencoded in the rewritten URI because it is the literal string a replay reader
matches on: a `:` would be parsed as a scheme, a `/` would make it several
segments, and anything else would be percent-encoded by `quote`.
"""
if not zim_path_prefix:
cls.zim_path_prefix = None
return
if not ZIM_PATH_PREFIX_RX.fullmatch(zim_path_prefix) or zim_path_prefix in (
".",
"..",
):
raise ValueError(f"Invalid ZIM path prefix: {zim_path_prefix!r}")
cls.zim_path_prefix = zim_path_prefix

def __init__(
self,
Expand Down Expand Up @@ -411,8 +448,14 @@ def get_document_uri(self, item_path: ZimPath, item_fragment: str) -> str:
if article_path.endswith("/")
else article_path.rpartition("/")[0]
)
relative_path = ArticleUrlRewriter._relative_path(
item_url.removesuffix("/"), article_dir
walk_up_from = "" if self.zim_path_prefix else article_dir
ups_to_zim_root = (
"../" * len([part for part in article_dir.split("/") if part != "."])
if self.zim_path_prefix
else ""
)
relative_path = ups_to_zim_root + ArticleUrlRewriter._relative_path(
item_url.removesuffix("/"), walk_up_from
)
# the last '/' of the path has been removed above, we add it back
if item_path.value.endswith("/"):
Expand Down Expand Up @@ -558,4 +601,8 @@ def normalize(cls, url: HttpUrl) -> ZimPath:
f"{hostname}{path}{query}"
)

return ZimPath(ArticleUrlRewriter.escape_dot_segments(fuzzified_url))
zim_path = ArticleUrlRewriter.escape_dot_segments(fuzzified_url)

return ZimPath(
f"{cls.zim_path_prefix}/{zim_path}" if cls.zim_path_prefix else zim_path
)
17 changes: 17 additions & 0 deletions tests/rewriting/test_html_rewriting.py
Original file line number Diff line number Diff line change
Expand Up @@ -365,6 +365,23 @@ def test_rewrite(rewrite_url: ContentForTests):
)


def test_rewrite_with_zim_path_prefix(monkeypatch: pytest.MonkeyPatch):
"""A prefixed URI is recognizable as rewritten, original still recorded"""
monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", "zimreplay")
assert HtmlRewriter(
ArticleUrlRewriter(
article_url=HttpUrl("http://kiwix.org/another/path/page.html"),
existing_zim_paths={ZimPath("zimreplay/exemple.com/a/long/path")},
),
None,
None,
None,
).rewrite('<a href="http://exemple.com/a/long/path">A link</a>').content == (
'<a href="../../../../zimreplay/exemple.com/a/long/path"'
' __wb_orig_href="http://exemple.com/a/long/path">A link</a>'
)


def test_extract_title():
content = """<html>
<head>
Expand Down
197 changes: 197 additions & 0 deletions tests/rewriting/test_url_rewriting.py
Original file line number Diff line number Diff line change
Expand Up @@ -1188,6 +1188,191 @@ def test_base_href(
)


class TestZimPathPrefix:
"""`zim_path_prefix` makes a rewritten URI recognizable as such (#396)"""

PREFIX = "zimreplay"

def test_disabled_by_default(self):
assert ArticleUrlRewriter.zim_path_prefix is None
rewriter = ArticleUrlRewriter(
article_url=HttpUrl("https://example.com/d/p.html")
)
assert rewriter("p2.html", None).zim_path == ZimPath("example.com/d/p2.html")
assert rewriter("p2.html", None).rewriten_url == "p2.html"

def test_prefix_makes_a_sibling_link_recognizable(
self, monkeypatch: pytest.MonkeyPatch
):
"""The same link, without and with the prefix

A rewritten `./p2.html` is textually indistinguishable from an original
relative link, which is what forces the original to be recorded separately.
Walking up to the prefix instead makes the rewritten URI carry a marker no
original URI can have.
"""
article_url = HttpUrl("https://example.com/d/p.html")
assert (
ArticleUrlRewriter(article_url=article_url)("./p2.html", None).rewriten_url
== "p2.html"
)
monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", self.PREFIX)
rewritten = ArticleUrlRewriter(article_url=article_url)(
"./p2.html", None
).rewriten_url
assert rewritten == "../../../zimreplay/example.com/d/p2.html"
assert self.PREFIX in rewritten
assert self.PREFIX not in "./p2.html"

@pytest.mark.parametrize(
"url,zim_path",
[
("https://example.com", "zimreplay/example.com/"),
("http://example.com/resource", "zimreplay/example.com/resource"),
("http://example.com/a/b?foo=bar", "zimreplay/example.com/a/b?foo=bar"),
("http://example.com/a/../b", "zimreplay/example.com/a/_../b"),
(
"https://www.youtube.com/youtubei/v1/foo?videoId=x&other=y",
"zimreplay/youtube.fuzzy.replayweb.page/youtubei/v1/foo?videoId=x",
),
],
)
def test_normalize_adds_prefix(
self, monkeypatch: pytest.MonkeyPatch, url: str, zim_path: str
):
monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", self.PREFIX)
assert ArticleUrlRewriter.normalize(HttpUrl(url)) == ZimPath(zim_path)

@pytest.mark.parametrize(
"article_url,original_content_url,expected_rewritten_url",
[
pytest.param(
"https://example.com/dir/page1.html",
"./page2.html",
"../../../zimreplay/example.com/dir/page2.html",
id="sibling_document_shows_the_whole_path",
),
(
"https://example.com/dir/page1.html",
"https://other.org/a/b.css",
"../../../zimreplay/other.org/a/b.css",
),
pytest.param(
"https://example.com/dir/",
"index.html",
"../../../zimreplay/example.com/dir/index.html",
id="directory_document",
),
pytest.param(
"https://example.com/dir/page.html",
"https://example.com/dir/page.html?foo=a b#frag",
"../../../zimreplay/example.com/dir/page.html%3Ffoo=a%20b#frag",
id="querystring_and_fragment_keep_their_place",
),
pytest.param(
"https://example.com/index.html",
"//cdn.example.com/logo.png",
"../../zimreplay/cdn.example.com/logo.png",
id="article_at_host_root_walks_up_one_segment_less",
),
pytest.param(
"https://example.com/x//y/page.html",
"https://example.com/x//y/z.html",
"../../../../../zimreplay/example.com/x//y/z.html",
id="empty_segments_count_as_directories_to_walk_up_from",
),
pytest.param(
"https://example.com/dir/page1.html",
"#anchor",
"#anchor",
id="fragment_only_reference_is_not_rewritten",
),
pytest.param(
"https://example.com/dir/page1.html",
"mailto:a@b.com",
"mailto:a@b.com",
id="other_scheme_is_not_rewritten",
),
],
)
def test_rewritten_url(
self,
monkeypatch: pytest.MonkeyPatch,
article_url: str,
original_content_url: str,
expected_rewritten_url: str,
):
monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", self.PREFIX)
rewriter = ArticleUrlRewriter(article_url=HttpUrl(article_url))
assert (
rewriter(
original_content_url, base_href=None, rewrite_all_url=True
).rewriten_url
== expected_rewritten_url
)

@pytest.mark.parametrize(
"article_url,original_content_url",
[
("https://example.com/dir/page1.html", "./page2.html"),
("https://example.com/dir/page1.html", "../other/x.png"),
("https://example.com/dir/", "index.html"),
("https://example.com/x//y/page.html", "https://example.com/x//y/z.html"),
("https://example.com/a/b/c/d/e.html", "https://other.org/f?g=h//i"),
],
)
def test_prefix_uri_resolves_back_to_zim_path(
self,
monkeypatch: pytest.MonkeyPatch,
article_url: str,
original_content_url: str,
):
"""Walking up to the prefix must not change which entry is addressed"""
monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", self.PREFIX)
rewriter = ArticleUrlRewriter(article_url=HttpUrl(article_url))
result = rewriter(original_content_url, base_href=None, rewrite_all_url=True)
assert result.zim_path is not None
assert (
_resolve_from_zim_root(rewriter.article_path.value, result.rewriten_url)
== result.zim_path.value
)

@pytest.mark.parametrize(
"invalid_prefix",
["a/b", "with space", "co:lon", "p%C3%A9fix", ".", "..", "%2F"],
)
def test_set_zim_path_prefix_rejects_invalid(
self, monkeypatch: pytest.MonkeyPatch, invalid_prefix: str
):
monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", None)
with pytest.raises(ValueError, match="Invalid ZIM path prefix"):
ArticleUrlRewriter.set_zim_path_prefix(invalid_prefix)
assert ArticleUrlRewriter.zim_path_prefix is None

@pytest.mark.parametrize(
"valid_prefix", ["z", "zim-replay", "zim_replay", "zim.replay"]
)
def test_set_zim_path_prefix_accepts_valid(
self, monkeypatch: pytest.MonkeyPatch, valid_prefix: str
):
monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", None)
ArticleUrlRewriter.set_zim_path_prefix(valid_prefix)
assert ArticleUrlRewriter.zim_path_prefix == valid_prefix
assert ArticleUrlRewriter.normalize(
HttpUrl("https://example.com/a")
).value.startswith(f"{valid_prefix}/example.com/a")

def test_set_zim_path_prefix_disables(self, monkeypatch: pytest.MonkeyPatch):
monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", self.PREFIX)
for falsy in (None, ""):
ArticleUrlRewriter.set_zim_path_prefix(falsy)
assert ArticleUrlRewriter.zim_path_prefix is None
assert (
ArticleUrlRewriter.normalize(HttpUrl("https://example.com/a")).value
== "example.com/a"
)


class CustomRewriter(ArticleUrlRewriter):
"""Sample class using the rewriting result to process items to download

Expand Down Expand Up @@ -1378,3 +1563,15 @@ def _resolve_relative_reference(base_url: str, reference: str) -> str:
base = urlsplit(base_url)
merged_path = base.path.rpartition("/")[0] + "/" + reference.partition("#")[0]
return f"{base.netloc}{_remove_dot_segments(merged_path)}"


def _resolve_from_zim_root(article_zim_path: str, reference: str) -> str:
"""Resolve a document URI against the ZIM path of its document, as a reader does

A ZIM has no host, so the ZIM root is served behind a dummy one to reuse the
resolution rules.
"""
resolved = _resolve_relative_reference(
f"https://zimroot.example.test/{article_zim_path}", reference
)
return unquote(resolved).removeprefix("zimroot.example.test/")