diff --git a/CHANGELOG.md b/CHANGELOG.md index ce781f9e..9e1e9e6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- `ArticleUrlRewriter.set_zim_path_prefix()`: an opt-in ZIM path prefix which stores every ZIM path under it and makes rewritten URIs walk up to it instead of stopping at the closest common directory. A rewritten URI then always contains the prefix as a literal segment, so a replay reader can tell a rewritten URL from an original one (and recover its ZIM path) without relying on the `__wb_orig_*` attributes recorded at scrape time. Disabled by default, keeping the current unprefixed entry paths and "as-up-as-needed" URIs (#396) + ### Fixed - Fix JS rewriting of `this.` so it reliably detects a leading newline (and prepends the needed `;`) regardless of the amount of whitespace, and no longer wrongly rewrites `this` when preceded by `.`/`$` and several spaces (#381) diff --git a/docs/technical_architecture.md b/docs/technical_architecture.md index a6244c98..d46aa72f 100644 --- a/docs/technical_architecture.md +++ b/docs/technical_architecture.md @@ -47,6 +47,15 @@ Below is an example case of the rewrite operation on an image URL found in an HT - Image rewritten URL: `../../../ex%C3%A9mple.com/a/resource/image.png%3Ffoo=bar` - Image ZIM Path: `exémple.com/a/resource/image.png?foo=bar` +`ArticleUrlRewriter.set_zim_path_prefix("zimreplay")` is an opt-in variant of this scheme: + +- every ZIM path gains the prefix, e.g. `kiwix.org/a/article/document.html` becomes `zimreplay/kiwix.org/a/article/document.html`. Scrapers have to store their entries under those prefixed paths. +- a rewritten URI walks all the way up to the ZIM root and comes back down through the prefix, instead of stopping as soon as it can: the image above is then addressed as `../../../../zimreplay/ex%C3%A9mple.com/a/resource/image.png%3Ffoo=bar`. + +The point is that a rewritten URI then always contains the prefix as a literal segment, while an original URI found in the document cannot. A replay reader (wombat) can hence tell the two apart and recover the ZIM path from a rewritten URI alone, without relying on the `__wb_orig_*` attributes recorded at scrape time for the cases where it cannot. + +The prefix is disabled by default, and its validation is deliberately strict (one unencoded URI path segment, made of `a-z`, `0-9`, `.`, `-`, `_`) because it is the string readers have to match. + ## JS Rewriting JS Rewriting is a bit special because rules to apply are different wether we are using "classic" Javascript or "module" Javascript. diff --git a/src/zimscraperlib/rewriting/url_rewriting.py b/src/zimscraperlib/rewriting/url_rewriting.py index cf1bfb3c..e19b1fc7 100644 --- a/src/zimscraperlib/rewriting/url_rewriting.py +++ b/src/zimscraperlib/rewriting/url_rewriting.py @@ -48,6 +48,16 @@ and not url-encoded. '=' and ',' are kept unencoded because some sites (e.g. YouTube) embed scripts that parse their own script URL and expect these characters to appear literally in it. + +A replay reader looking at a rewritten URI cannot tell it from an original relative one +(`./page2.html` in `page1.html` is both a valid original link and a valid rewritten +link), which is why the original value of every rewritten URL attribute has to be +recorded separately. Setting `ArticleUrlRewriter.zim_path_prefix` removes that +ambiguity: every ZIM path is then stored under that prefix and every rewritten URI +walks all the way up to the ZIM root before coming back down through it, so a rewritten +URI always contains the prefix as a literal path segment +(`../../../zimreplay/example.com/page2.html`) while an original one cannot. It is off by +default, which keeps the historical "as-up-as-needed" URIs and unprefixed entry paths. """ import re @@ -78,6 +88,8 @@ class AdditionalRule: for rule in FUZZY_RULES ] +ZIM_PATH_PREFIX_RX = re.compile(r"[a-zA-Z0-9._-]+") + def _remove_dot_segments(path: str) -> str: """Remove '.' and '..' segments from a path, per RFC 3986 section 5.2.4 @@ -246,6 +258,31 @@ class ArticleUrlRewriter: """ additional_rules: ClassVar[list[AdditionalRule]] = COMPILED_FUZZY_RULES + zim_path_prefix: ClassVar[str | None] = None + + @classmethod + def set_zim_path_prefix(cls, zim_path_prefix: str | None) -> None: + """Store every ZIM path under `zim_path_prefix` and root-anchor URIs on it + + Both halves of the behaviour are set together: entry paths gain the prefix, and + rewritten URIs are made to walk up to it, which is what makes a rewritten URI + recognizable. Passing None (or an empty string) restores the default unprefixed + entry paths and "as-up-as-needed" URIs. + + The prefix must be one URI path segment of `a-z`, `0-9`, `.`, `-` or `_`, kept + unencoded in the rewritten URI because it is the literal string a replay reader + matches on: a `:` would be parsed as a scheme, a `/` would make it several + segments, and anything else would be percent-encoded by `quote`. + """ + if not zim_path_prefix: + cls.zim_path_prefix = None + return + if not ZIM_PATH_PREFIX_RX.fullmatch(zim_path_prefix) or zim_path_prefix in ( + ".", + "..", + ): + raise ValueError(f"Invalid ZIM path prefix: {zim_path_prefix!r}") + cls.zim_path_prefix = zim_path_prefix def __init__( self, @@ -411,8 +448,14 @@ def get_document_uri(self, item_path: ZimPath, item_fragment: str) -> str: if article_path.endswith("/") else article_path.rpartition("/")[0] ) - relative_path = ArticleUrlRewriter._relative_path( - item_url.removesuffix("/"), article_dir + walk_up_from = "" if self.zim_path_prefix else article_dir + ups_to_zim_root = ( + "../" * len([part for part in article_dir.split("/") if part != "."]) + if self.zim_path_prefix + else "" + ) + relative_path = ups_to_zim_root + ArticleUrlRewriter._relative_path( + item_url.removesuffix("/"), walk_up_from ) # the last '/' of the path has been removed above, we add it back if item_path.value.endswith("/"): @@ -558,4 +601,8 @@ def normalize(cls, url: HttpUrl) -> ZimPath: f"{hostname}{path}{query}" ) - return ZimPath(ArticleUrlRewriter.escape_dot_segments(fuzzified_url)) + zim_path = ArticleUrlRewriter.escape_dot_segments(fuzzified_url) + + return ZimPath( + f"{cls.zim_path_prefix}/{zim_path}" if cls.zim_path_prefix else zim_path + ) diff --git a/tests/rewriting/test_html_rewriting.py b/tests/rewriting/test_html_rewriting.py index 914bcb8b..17af8d96 100644 --- a/tests/rewriting/test_html_rewriting.py +++ b/tests/rewriting/test_html_rewriting.py @@ -365,6 +365,23 @@ def test_rewrite(rewrite_url: ContentForTests): ) +def test_rewrite_with_zim_path_prefix(monkeypatch: pytest.MonkeyPatch): + """A prefixed URI is recognizable as rewritten, original still recorded""" + monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", "zimreplay") + assert HtmlRewriter( + ArticleUrlRewriter( + article_url=HttpUrl("http://kiwix.org/another/path/page.html"), + existing_zim_paths={ZimPath("zimreplay/exemple.com/a/long/path")}, + ), + None, + None, + None, + ).rewrite('A link').content == ( + 'A link' + ) + + def test_extract_title(): content = """ diff --git a/tests/rewriting/test_url_rewriting.py b/tests/rewriting/test_url_rewriting.py index 7d2197d7..36e17a2e 100644 --- a/tests/rewriting/test_url_rewriting.py +++ b/tests/rewriting/test_url_rewriting.py @@ -1188,6 +1188,191 @@ def test_base_href( ) +class TestZimPathPrefix: + """`zim_path_prefix` makes a rewritten URI recognizable as such (#396)""" + + PREFIX = "zimreplay" + + def test_disabled_by_default(self): + assert ArticleUrlRewriter.zim_path_prefix is None + rewriter = ArticleUrlRewriter( + article_url=HttpUrl("https://example.com/d/p.html") + ) + assert rewriter("p2.html", None).zim_path == ZimPath("example.com/d/p2.html") + assert rewriter("p2.html", None).rewriten_url == "p2.html" + + def test_prefix_makes_a_sibling_link_recognizable( + self, monkeypatch: pytest.MonkeyPatch + ): + """The same link, without and with the prefix + + A rewritten `./p2.html` is textually indistinguishable from an original + relative link, which is what forces the original to be recorded separately. + Walking up to the prefix instead makes the rewritten URI carry a marker no + original URI can have. + """ + article_url = HttpUrl("https://example.com/d/p.html") + assert ( + ArticleUrlRewriter(article_url=article_url)("./p2.html", None).rewriten_url + == "p2.html" + ) + monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", self.PREFIX) + rewritten = ArticleUrlRewriter(article_url=article_url)( + "./p2.html", None + ).rewriten_url + assert rewritten == "../../../zimreplay/example.com/d/p2.html" + assert self.PREFIX in rewritten + assert self.PREFIX not in "./p2.html" + + @pytest.mark.parametrize( + "url,zim_path", + [ + ("https://example.com", "zimreplay/example.com/"), + ("http://example.com/resource", "zimreplay/example.com/resource"), + ("http://example.com/a/b?foo=bar", "zimreplay/example.com/a/b?foo=bar"), + ("http://example.com/a/../b", "zimreplay/example.com/a/_../b"), + ( + "https://www.youtube.com/youtubei/v1/foo?videoId=x&other=y", + "zimreplay/youtube.fuzzy.replayweb.page/youtubei/v1/foo?videoId=x", + ), + ], + ) + def test_normalize_adds_prefix( + self, monkeypatch: pytest.MonkeyPatch, url: str, zim_path: str + ): + monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", self.PREFIX) + assert ArticleUrlRewriter.normalize(HttpUrl(url)) == ZimPath(zim_path) + + @pytest.mark.parametrize( + "article_url,original_content_url,expected_rewritten_url", + [ + pytest.param( + "https://example.com/dir/page1.html", + "./page2.html", + "../../../zimreplay/example.com/dir/page2.html", + id="sibling_document_shows_the_whole_path", + ), + ( + "https://example.com/dir/page1.html", + "https://other.org/a/b.css", + "../../../zimreplay/other.org/a/b.css", + ), + pytest.param( + "https://example.com/dir/", + "index.html", + "../../../zimreplay/example.com/dir/index.html", + id="directory_document", + ), + pytest.param( + "https://example.com/dir/page.html", + "https://example.com/dir/page.html?foo=a b#frag", + "../../../zimreplay/example.com/dir/page.html%3Ffoo=a%20b#frag", + id="querystring_and_fragment_keep_their_place", + ), + pytest.param( + "https://example.com/index.html", + "//cdn.example.com/logo.png", + "../../zimreplay/cdn.example.com/logo.png", + id="article_at_host_root_walks_up_one_segment_less", + ), + pytest.param( + "https://example.com/x//y/page.html", + "https://example.com/x//y/z.html", + "../../../../../zimreplay/example.com/x//y/z.html", + id="empty_segments_count_as_directories_to_walk_up_from", + ), + pytest.param( + "https://example.com/dir/page1.html", + "#anchor", + "#anchor", + id="fragment_only_reference_is_not_rewritten", + ), + pytest.param( + "https://example.com/dir/page1.html", + "mailto:a@b.com", + "mailto:a@b.com", + id="other_scheme_is_not_rewritten", + ), + ], + ) + def test_rewritten_url( + self, + monkeypatch: pytest.MonkeyPatch, + article_url: str, + original_content_url: str, + expected_rewritten_url: str, + ): + monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", self.PREFIX) + rewriter = ArticleUrlRewriter(article_url=HttpUrl(article_url)) + assert ( + rewriter( + original_content_url, base_href=None, rewrite_all_url=True + ).rewriten_url + == expected_rewritten_url + ) + + @pytest.mark.parametrize( + "article_url,original_content_url", + [ + ("https://example.com/dir/page1.html", "./page2.html"), + ("https://example.com/dir/page1.html", "../other/x.png"), + ("https://example.com/dir/", "index.html"), + ("https://example.com/x//y/page.html", "https://example.com/x//y/z.html"), + ("https://example.com/a/b/c/d/e.html", "https://other.org/f?g=h//i"), + ], + ) + def test_prefix_uri_resolves_back_to_zim_path( + self, + monkeypatch: pytest.MonkeyPatch, + article_url: str, + original_content_url: str, + ): + """Walking up to the prefix must not change which entry is addressed""" + monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", self.PREFIX) + rewriter = ArticleUrlRewriter(article_url=HttpUrl(article_url)) + result = rewriter(original_content_url, base_href=None, rewrite_all_url=True) + assert result.zim_path is not None + assert ( + _resolve_from_zim_root(rewriter.article_path.value, result.rewriten_url) + == result.zim_path.value + ) + + @pytest.mark.parametrize( + "invalid_prefix", + ["a/b", "with space", "co:lon", "p%C3%A9fix", ".", "..", "%2F"], + ) + def test_set_zim_path_prefix_rejects_invalid( + self, monkeypatch: pytest.MonkeyPatch, invalid_prefix: str + ): + monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", None) + with pytest.raises(ValueError, match="Invalid ZIM path prefix"): + ArticleUrlRewriter.set_zim_path_prefix(invalid_prefix) + assert ArticleUrlRewriter.zim_path_prefix is None + + @pytest.mark.parametrize( + "valid_prefix", ["z", "zim-replay", "zim_replay", "zim.replay"] + ) + def test_set_zim_path_prefix_accepts_valid( + self, monkeypatch: pytest.MonkeyPatch, valid_prefix: str + ): + monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", None) + ArticleUrlRewriter.set_zim_path_prefix(valid_prefix) + assert ArticleUrlRewriter.zim_path_prefix == valid_prefix + assert ArticleUrlRewriter.normalize( + HttpUrl("https://example.com/a") + ).value.startswith(f"{valid_prefix}/example.com/a") + + def test_set_zim_path_prefix_disables(self, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(ArticleUrlRewriter, "zim_path_prefix", self.PREFIX) + for falsy in (None, ""): + ArticleUrlRewriter.set_zim_path_prefix(falsy) + assert ArticleUrlRewriter.zim_path_prefix is None + assert ( + ArticleUrlRewriter.normalize(HttpUrl("https://example.com/a")).value + == "example.com/a" + ) + + class CustomRewriter(ArticleUrlRewriter): """Sample class using the rewriting result to process items to download @@ -1378,3 +1563,15 @@ def _resolve_relative_reference(base_url: str, reference: str) -> str: base = urlsplit(base_url) merged_path = base.path.rpartition("/")[0] + "/" + reference.partition("#")[0] return f"{base.netloc}{_remove_dot_segments(merged_path)}" + + +def _resolve_from_zim_root(article_zim_path: str, reference: str) -> str: + """Resolve a document URI against the ZIM path of its document, as a reader does + + A ZIM has no host, so the ZIM root is served behind a dummy one to reuse the + resolution rules. + """ + resolved = _resolve_relative_reference( + f"https://zimroot.example.test/{article_zim_path}", reference + ) + return unquote(resolved).removeprefix("zimroot.example.test/")