From 186b0ef7af175ea8b66c3576028237da9eb8694a Mon Sep 17 00:00:00 2001 From: August Vi Date: Mon, 2 Nov 2026 14:56:20 -0800 Subject: [PATCH] Read the remote URL cache through diskcache-community's SafeJSONDisk --- CHANGELOG.md | 1 + glean_parser/util.py | 16 +++++++---- pyproject.toml | 2 +- tests/test_utils.py | 68 ++++++++++++++++++++++++++++++++++++++++++++ uv.lock | 12 ++++---- 5 files changed, 87 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 83123f8e6..b4a009326 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Depend on `diskcache-community` instead of `diskcache`, which is unmaintained and flagged for CVE-2025-69872. `fetch_remote_url` reads its cache through `SafeJSONDisk`, which reads the existing cache files but never unpickles anything. - Support `flows.yaml` files (`moz://mozilla.org/schemas/glean/flows/1-0-0`), declaring multi-step tasks measured as a unit over existing event metrics. Flows are validated against the metrics passed to the parser, with warnings for events that aren't defined in any of them (`UNRESOLVED_FLOW_EVENT`) and for flows with no exit events (`EMPTY_ENDS_ON`). Kotlin, Swift, and Rust output gains the flow definitions to pass to Glean at initialization (`GleanFlows`), and Markdown output documents flows and the flows each event belongs to. Flows are passed to external translators as the `flows` option ([bug 2077133](https://bugzilla.mozilla.org/show_bug.cgi?id=2077133)) - Fix tests that failed when `SOURCE_DATE_EPOCH` is set in the environment - Add a `skip_schema_validation` option to `parse_objects` to skip schema validation of input files ([bug 2076540](https://bugzilla.mozilla.org/show_bug.cgi?id=2076540)) diff --git a/glean_parser/util.py b/glean_parser/util.py index 4f35b6159..186e549f1 100644 --- a/glean_parser/util.py +++ b/glean_parser/util.py @@ -16,7 +16,7 @@ from typing import Any, Callable, Iterable, Sequence, Tuple, Union, Optional import urllib.request -import diskcache # type: ignore +import diskcache_community as diskcache import jinja2 import jsonschema # type: ignore from jsonschema import _utils # type: ignore @@ -283,6 +283,11 @@ def resolve_remote(self, uri): return NullResolver.from_schema(schema) +# SafeJSONDisk reads the same files as JSONDisk but never unpickles anything, +# and on_error="miss" turns an unreadable entry into a cache miss. +_CACHE_DISK = diskcache.SafeJSONDisk.with_options(on_error="miss") + + def fetch_remote_url(url: str, cache: bool = True): """ Fetches the contents from an HTTP url or local file path, and optionally @@ -300,14 +305,15 @@ def fetch_remote_url(url: str, cache: bool = True): if cache: cache_dir = platformdirs.user_cache_dir("glean_parser", "mozilla") - with diskcache.Cache(cache_dir, disk=diskcache.JSONDisk) as dc: - if key in dc: - return dc[key] + with diskcache.Cache(cache_dir, disk=_CACHE_DISK) as dc: + contents = dc.get(key) + if contents is not None: + return contents contents = urllib.request.urlopen(url).read().decode("utf-8") if cache: - with diskcache.Cache(cache_dir, disk=diskcache.JSONDisk) as dc: + with diskcache.Cache(cache_dir, disk=_CACHE_DISK) as dc: dc[key] = contents return contents diff --git a/pyproject.toml b/pyproject.toml index 84d4fbcec..62609c322 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,7 +20,7 @@ classifiers = [ requires-python = ">=3.9" dependencies = [ "Click>=7", - "diskcache>=4", + "diskcache-community>=5.8.2", "Jinja2>=2.10.1", "jsonschema>=3.0.2", "platformdirs>=2.4.0", diff --git a/tests/test_utils.py b/tests/test_utils.py index 4bf14df65..a2d8ac98f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,6 +3,14 @@ # Any copyright is dedicated to the Public Domain. # http://creativecommons.org/publicdomain/zero/1.0/ +import io +import os +import pickle +import sqlite3 + +import diskcache_community as diskcache + +from glean_parser import util from glean_parser.util import to_camel_case, remove_output_params @@ -39,3 +47,63 @@ def test_removing_output_params(): test = remove_output_params(d, "defined_in") assert test == output_removed + + +def _run_planted(path): + with open(path, "w") as fd: + fd.write("ran") + return "planted" + + +class _Planted: + def __init__(self, path): + self.path = path + + def __reduce__(self): + return (_run_planted, (self.path,)) + + +def test_fetch_remote_url_cache(tmp_path, monkeypatch): + cache_dir = tmp_path / "cache" + monkeypatch.setattr(util.platformdirs, "user_cache_dir", lambda *a: str(cache_dir)) + calls = [] + + def urlopen(url): + calls.append(url) + return io.BytesIO(b"schema contents") + + monkeypatch.setattr(util.urllib.request, "urlopen", urlopen) + url = "https://example.com/schema.json" + + assert util.fetch_remote_url(url) == "schema contents" + assert util.fetch_remote_url(url) == "schema contents" + assert len(calls) == 1 + + # A row rewritten by someone else to hold a pickle is a miss, not code + # that runs: the contents are fetched again and the pickle never loads. + marker = tmp_path / "marker" + con = sqlite3.connect(os.path.join(str(cache_dir), "cache.db")) + con.execute( + "UPDATE Cache SET mode = 4, value = ?", + (pickle.dumps(_Planted(str(marker)), protocol=2),), + ) + con.commit() + con.close() + + assert util.fetch_remote_url(url) == "schema contents" + assert len(calls) == 2 + assert not marker.exists() + + +def test_fetch_remote_url_reads_existing_jsondisk_cache(tmp_path, monkeypatch): + cache_dir = tmp_path / "cache" + monkeypatch.setattr(util.platformdirs, "user_cache_dir", lambda *a: str(cache_dir)) + monkeypatch.setattr(util.urllib.request, "urlopen", None) + url = "https://example.com/schema.json" + + # Entries written by earlier releases, through JSONDisk, are still read. + key = (url, str(util.sys.version_info)) + with diskcache.Cache(str(cache_dir), disk=diskcache.JSONDisk) as dc: + dc[key] = "cached earlier" + + assert util.fetch_remote_url(url) == "cached earlier" diff --git a/uv.lock b/uv.lock index 8d861b345..17ddddaae 100644 --- a/uv.lock +++ b/uv.lock @@ -257,12 +257,12 @@ wheels = [ ] [[package]] -name = "diskcache" -version = "5.6.3" +name = "diskcache-community" +version = "5.8.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3f/21/1c1ffc1a039ddcc459db43cc108658f32c57d271d7289a2794e401d0fdb6/diskcache-5.6.3.tar.gz", hash = "sha256:2c3a3fa2743d8535d832ec61c2054a1641f41775aa7c556758a109941e33e4fc", size = 67916, upload-time = "2023-08-31T06:12:00.316Z" } +sdist = { url = "https://files.pythonhosted.org/packages/0d/f4/35b9ad1bc1637d5680e739f9891f01951af16e34c6be2d4d94f37f390bcc/diskcache_community-5.8.2.tar.gz", hash = "sha256:440b558c4841688c05f11c28bf21a18662a3f2f5f90afc0a1f0d277cd7e90403", size = 130095, upload-time = "2026-11-02T22:33:05.946Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/27/4570e78fc0bf5ea0ca45eb1de3818a23787af9b390c0b0a0033a1b8236f9/diskcache-5.6.3-py3-none-any.whl", hash = "sha256:5e31b2d5fbad117cc363ebaf6b689474db18a1f6438bc82358b024abd4c2ca19", size = 45550, upload-time = "2023-08-31T06:11:58.822Z" }, + { url = "https://files.pythonhosted.org/packages/1f/da/1205bee8c006c896475f7c8aa629ca9f7ce0ff1f6b233f2bb2e72b59248e/diskcache_community-5.8.2-py3-none-any.whl", hash = "sha256:1e079c4eb12672cd432029929a8b338ef855d737bc51324edb098d36dfd2f632", size = 78990, upload-time = "2026-11-02T22:33:04.625Z" }, ] [[package]] @@ -292,7 +292,7 @@ source = { editable = "." } dependencies = [ { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "click", version = "8.2.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "diskcache" }, + { name = "diskcache-community" }, { name = "jinja2" }, { name = "jsonschema" }, { name = "platformdirs" }, @@ -317,7 +317,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "click", specifier = ">=7" }, - { name = "diskcache", specifier = ">=4" }, + { name = "diskcache-community", specifier = ">=5.8.2" }, { name = "jinja2", specifier = ">=2.10.1" }, { name = "jsonschema", specifier = ">=3.0.2" }, { name = "platformdirs", specifier = ">=2.4.0" },