diff --git a/.claude/launch.json b/.claude/launch.json index 4dd9dbd..e9add90 100644 --- a/.claude/launch.json +++ b/.claude/launch.json @@ -4,14 +4,48 @@ { "name": "properdocs-en", "runtimeExecutable": "properdocs", - "runtimeArgs": ["serve", "-f", "config/en/properdocs.yml", "--dev-addr", "127.0.0.1:8000", "--watch", "overrides", "--watch", "hooks", "--watch", "data", "--watch", "scripts"], + "runtimeArgs": [ + "serve", + "-f", + "config/en/properdocs.yml", + "--dev-addr", + "127.0.0.1:8000", + "--watch", + "overrides", + "--watch", + "hooks", + "--watch", + "data", + "--watch", + "scripts" + ], "port": 8000 }, { "name": "properdocs-en-alt", "runtimeExecutable": "properdocs", - "runtimeArgs": ["serve", "-f", "config/en/properdocs.yml", "--dev-addr", "127.0.0.1:8001"], + "runtimeArgs": [ + "serve", + "-f", + "config/en/properdocs.yml", + "--dev-addr", + "127.0.0.1:8001" + ], "port": 8001 + }, + { + "name": "review-preview", + "runtimeExecutable": "python3", + "runtimeArgs": [ + "-m", + "http.server", + "8010", + "--bind", + "127.0.0.1", + "--directory", + "/private/tmp/claude-502/-Users-etiennepro--claude-spares-spare-mHZoyW/ba05716d-799e-4788-8abc-098b3b01bdae/scratchpad/site" + ], + "port": 8010 } ] -} +} \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/maintainer-review.yml b/.github/ISSUE_TEMPLATE/maintainer-review.yml new file mode 100644 index 0000000..c4cd11d --- /dev/null +++ b/.github/ISSUE_TEMPLATE/maintainer-review.yml @@ -0,0 +1,10 @@ +name: Maintainer review +description: Maintainer review +title: "Maintainer review: " +body: + - type: input + id: spec_id + attributes: + label: Specification id + validations: + required: true diff --git a/.github/workflows/maintainer_review_redeploy.yml b/.github/workflows/maintainer_review_redeploy.yml new file mode 100644 index 0000000..80431ea --- /dev/null +++ b/.github/workflows/maintainer_review_redeploy.yml @@ -0,0 +1,65 @@ +name: Maintainer review status + +# The maintainer review status of each specification is the `review: ...` +# label of its `maintainer-review` issue (see scripts/maintainer_review.py). +# When a status label is added, this keeps it the only one, writes today's +# date into the issue, then starts the existing deploy so the site shows it. +# +# Changes made here with GITHUB_TOKEN do not start this workflow again. The +# deploy is started with workflow_dispatch, which GITHUB_TOKEN is allowed to do. + +on: + issues: + types: [labeled, unlabeled, closed, reopened, deleted] + +permissions: + actions: write + issues: write + +jobs: + status: + if: >- + github.event.action == 'labeled' && + startsWith(github.event.label.name, 'review: ') && + contains(github.event.issue.labels.*.name, 'maintainer-review') + runs-on: ubuntu-latest + steps: + - name: Keep one status label and date it + uses: actions/github-script@v7 + with: + script: | + const issue = context.payload.issue; + const added = context.payload.label.name; + for (const l of issue.labels) { + if (l.name.startsWith('review: ') && l.name !== added) { + await github.rest.issues.removeLabel({ + ...context.repo, issue_number: issue.number, name: l.name }); + } + } + const today = new Date().toISOString().slice(0, 10); + const re = /(### Review date\s*\n\s*\n)([^\n]*)/; + const body = re.test(issue.body || '') + ? issue.body.replace(re, `$1${today}`) + : `${(issue.body || '').trimEnd()}\n\n### Review date\n\n${today}\n`; + await github.rest.issues.update({ + ...context.repo, issue_number: issue.number, body }); + + redeploy: + needs: status + # Runs after `status`, or on its own when `status` was skipped. + if: >- + always() && needs.status.result != 'failure' && + (contains(github.event.issue.labels.*.name, 'maintainer-review') || + github.event.label.name == 'maintainer-review') + runs-on: ubuntu-latest + # Several changes in a row queue one deploy instead of one per change. + concurrency: + group: maintainer-review-redeploy + cancel-in-progress: true + steps: + - name: Debounce + run: sleep 60 + - name: Start the website deploy + env: + GH_TOKEN: ${{ github.token }} + run: gh workflow run update_website_contents_on_server.yml --repo "$GITHUB_REPOSITORY" --ref main diff --git a/hooks/specifications.py b/hooks/specifications.py index aa63f1e..cede448 100644 --- a/hooks/specifications.py +++ b/hooks/specifications.py @@ -36,7 +36,8 @@ ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(ROOT, "scripts")) -import spec_api # noqa: E402 (path set above) +import maintainer_review # noqa: E402 (path set above) +import spec_api # noqa: E402 import spec_data # noqa: E402 import spec_history # noqa: E402 @@ -637,6 +638,19 @@ def dl_row(ico, label, value_html, field=None, empty=False, view=None, edit=True % (edit_only(empty), icon(ico), escape(label), dd, value_html)) +def render_review(review): + """The line under "Maintained by": where the assessment stands in its review + by the maintainer. The status lives in a GitHub issue, not in the file the + editor changes (see scripts/maintainer_review.py).""" + if not review: + return "" + ico = "check-circle" if review["status"] == "reviewed" else "progress-clock" + date = (' %s' % escape(review["date"]) + if review["date"] else "") + return ('

%s%s%s

' + % (escape(review["status_tone"]), icon(ico), escape(review["status_label"]), date)) + + def render_detail(spec, data, base, new=False): """One specification's page, or with `new` the blank page on which a specification is proposed: the same layout, every field empty, opened in @@ -681,6 +695,7 @@ def render_detail(spec, data, base, new=False): o.append('%s%s' % (org_logo(org, base), escape(org["name"]))) o.append("") + o.append(render_review(spec.get("review"))) o.append("") # ---- in-page nav @@ -1228,6 +1243,9 @@ def blank_spec(): def on_files(files, config): data = spec_data.build_dataset() + reviews = maintainer_review.load(spec_data.GITHUB_REPO, {s["id"] for s in data["specs"]}) + for spec in data["specs"]: + spec["review"] = reviews.get(spec["id"]) for issue in data["issues"]: log("specifications: %s in %s: %s" diff --git a/overrides/assets/stylesheets/specifications.css b/overrides/assets/stylesheets/specifications.css index c487e38..86f8815 100644 --- a/overrides/assets/stylesheets/specifications.css +++ b/overrides/assets/stylesheets/specifications.css @@ -2060,3 +2060,7 @@ buttons :is(.mdip-specs, .mdip-spec) .specs-hero__glow, :is(.mdip-specs, .mdip-spec) .spec-hero__glow { animation: none; } } + +/* Maintainer review of the assessment, under "Maintained by" */ +:is(.mdip-specs, .mdip-spec) .spec-hero__review { margin: .7rem 0 0; } +:is(.mdip-specs, .mdip-spec) .spec-hero__review-date { font-weight: 400; opacity: .8; } diff --git a/scripts/maintainer_review.py b/scripts/maintainer_review.py new file mode 100644 index 0000000..27fdb70 --- /dev/null +++ b/scripts/maintainer_review.py @@ -0,0 +1,167 @@ +"""Where each specification's assessment stands in its review by the maintainer. + +The status is not kept in data/: it lives in one GitHub issue per +specification, labelled `maintainer-review`, so the team changes it on GitHub +with no commit and no pull request. + +- The status is the issue's `review: ...` label (see REVIEW_STATUS). +- The issue body holds only the specification id and the date of the last + status change. The redeploy workflow rewrites that date whenever a status + label is added, so nobody edits the body by hand: + + ### Specification id + + gbfs + + ### Review date + + 2026-10-24 + +Trust: only issues carrying the `maintainer-review` label are read, and only +people with triage access can add labels, so a visitor cannot publish a status. + +`load()` is called once per build. It never fails the build: when GitHub cannot +be reached it logs a warning and the pages simply show no review status. + + MDIP_REVIEWS=off skip the lookup (offline builds) + MDIP_REVIEWS_FILE=path read issues from a JSON file instead of GitHub, + in the shape the GitHub issues API returns + GITHUB_TOKEN optional; raises the API rate limit +""" + +from __future__ import annotations + +import json +import logging +import os +import re +import urllib.request +from collections import OrderedDict + +LABEL = "maintainer-review" +API = "https://api.github.com/repos/%s/issues?labels=%s&state=all&per_page=100&page=%d" +MAX_PAGES = 5 + +# key: GitHub label, text on the page, tone, whether the date is shown. +REVIEW_STATUS = OrderedDict([ + ("requested", {"github_label": "review: requested", + "label": "Assessment review requested", "tone": "warn", "dated": True}), + ("in_progress", {"github_label": "review: in progress", + "label": "Assessment under review by maintainer", "tone": "warn", + "dated": True}), + ("no_answer", {"github_label": "review: no answer", + "label": "Assessment review requested", "tone": "warn", "dated": False}), + ("reviewed", {"github_label": "review: reviewed", + "label": "Assessment reviewed by maintainer", "tone": "pass", "dated": True}), +]) +BY_GITHUB_LABEL = {v["github_label"]: k for k, v in REVIEW_STATUS.items()} + +_HEADING_RE = re.compile(r"^###\s+(.+?)\s*$", re.M) +_DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +_NO_RESPONSE = "_no response_" + +log = logging.getLogger("mkdocs.hooks.maintainer_review") + + +def parse_body(body): + """{field heading (lowercase): answer} from an issue form body.""" + fields = {} + parts = _HEADING_RE.split(body or "") + for i in range(1, len(parts) - 1, 2): + value = parts[i + 1].strip() + fields[parts[i].strip().lower()] = "" if value.lower() == _NO_RESPONSE else value + return fields + + +def make_body(spec_id, date): + return "### Specification id\n\n%s\n\n### Review date\n\n%s\n" % (spec_id, date or "_No response_") + + +def review_from_issue(issue): + """The published review record for one issue, or (None, reason).""" + fields = parse_body(issue.get("body")) + sid = fields.get("specification id", "").strip() + if not sid: + return None, "no specification id" + names = [l["name"] if isinstance(l, dict) else l for l in issue.get("labels") or []] + statuses = [BY_GITHUB_LABEL[n] for n in names if n in BY_GITHUB_LABEL] + if len(statuses) != 1: + return None, "%d review labels instead of 1" % len(statuses) + status = statuses[0] + meta = REVIEW_STATUS[status] + date = fields.get("review date", "").strip() + if date and not _DATE_RE.match(date): + return None, "review date is not YYYY-MM-DD" + return { + "spec_id": sid, + "status": status, + "status_label": meta["label"], + "status_tone": meta["tone"], + "date": (date or None) if meta["dated"] else None, + "_updated": issue.get("updated_at") or "", + }, None + + +def _fetch(repo): + headers = {"Accept": "application/vnd.github+json", + "User-Agent": "mdip-registry-build"} + token = os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = "Bearer " + token + issues = [] + for page in range(1, MAX_PAGES + 1): + req = urllib.request.Request(API % (repo, LABEL, page), headers=headers) + with urllib.request.urlopen(req, timeout=15) as resp: + batch = json.load(resp) + issues.extend(batch) + if len(batch) < 100: + break + return issues + + +def _read_issues(repo): + path = os.environ.get("MDIP_REVIEWS_FILE") + if path: + with open(path) as fh: + return json.load(fh) + return _fetch(repo) + + +def load(repo, known_ids): + """{spec id: review record} for every specification with a review issue.""" + if os.environ.get("MDIP_REVIEWS", "").lower() in ("off", "0", "false", "no"): + return {} + try: + issues = _read_issues(repo) + except Exception as exc: # network, rate limit, bad JSON: never fail the build + log.warning("maintainer review: issues not read, no status shown (%s)", exc) + return {} + + reviews = {} + for issue in issues: + if "pull_request" in issue: + continue + record, reason = review_from_issue(issue) + where = "issue #%s" % issue.get("number", "?") + if record is None: + log.warning("maintainer review: %s skipped: %s", where, reason) + continue + if record["spec_id"] not in known_ids: + log.warning("maintainer review: %s names unknown specification %r", + where, record["spec_id"]) + continue + # Several issues for one specification: the most recently updated wins. + previous = reviews.get(record["spec_id"]) + if previous is None or record["_updated"] > previous["_updated"]: + reviews[record["spec_id"]] = record + for record in reviews.values(): + record.pop("_updated", None) + return reviews + + +def api_record(review): + """The review as the JSON API publishes it, or None.""" + if not review: + return None + return {"status": {"id": review["status"], "label": review["status_label"]}, + "date": review["date"]} diff --git a/scripts/seed_maintainer_reviews.py b/scripts/seed_maintainer_reviews.py new file mode 100644 index 0000000..d6f4be9 --- /dev/null +++ b/scripts/seed_maintainer_reviews.py @@ -0,0 +1,221 @@ +"""Set up one `maintainer-review` issue per specification from the tracking sheet. + +One-off helper. Only three columns of the private sheet are read +(Specification, Review Status, Review Date): contact details, evidence links +and comments are never read, so they cannot end up in an issue. + +Each issue gets the `maintainer-review` label, one `review: ...` status label, +and a body holding only the specification id and the review date. + + python3 scripts/seed_maintainer_reviews.py SHEET.csv + Print what each issue would carry. Changes nothing. + + python3 scripts/seed_maintainer_reviews.py SHEET.csv --json issues.json + Also write the issues in the GitHub API's shape, for a local preview: + MDIP_REVIEWS_FILE=issues.json make serve + + python3 scripts/seed_maintainer_reviews.py SHEET.csv --apply + Open the missing issues on GitHub with `gh`, and bring the existing + ones (body and status label) in line with the sheet. +""" + +from __future__ import annotations + +import argparse +import csv +import json +import os +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +import maintainer_review as mr # noqa: E402 +import spec_data # noqa: E402 + +COLUMNS = ("Specification", "Review Status", "Review Date") + +SHEET_STATUS = { + "asked": "requested", + "review in progress": "in_progress", + "no answer": "no_answer", + "reviewed": "reviewed", +} + +# Sheet name -> registry id. None: not published (no file in +# data/specifications/ yet, or a row superseded by another one). +SHEET_IDS = { + "GTFS Schedule": "gtfs-schedule", + "GTFS Realtime": "gtfs-realtime", + "GBFS (General Bikeshare Feed)": "gbfs", + "MDS (Mobility Data Specification)": None, # superseded by the row below + "MDS - Mobility Data Specification": "mds", + "NeTEx Profile - Regional Rail": None, + "CDS (Curb Data Specification)": "cds", + "CDS-M (City Data Standard - Mobility)": "cds-m", + "TODS (Transit Operational Data)": "tods", + "TIDES (Transit Integrated Exchange)": "tides", + "Transmodel": None, + "NeTEx (Network & Timetable Exchange)": "netex", + "SIRI (Real Time Information)": "siri", + "OpRa (Operating Raw Data)": "opra", + "OJP (Open Journey Planner)": "ojp", + "ITxPT Architecture": "itxpt", + "GTNS (General Travel Network)": "gtns", + "GMNS (General Modeling Network)": "gmns", + "VDV / VDV-KA": "vdv", + "TransXChange (TXC)": "transxchange", + "GOFS (General On-Demand Feed)": "gofs", + "GTFS Flex": "gtfs-flex", + "TOMP API": "tomp-api", + "ISO 7874": "iso-7874", + "NEMTAC 1000": "nemtac-1000", + "Overture Maps Transportation": "overture-maps", + "OpenStreetMap (OSM) Pedestrian": "osm", + "OSM BNA Guide": "osm-bna-guide", + "OpenSidewalks Exchange": "opensidewalks", + "Connect-OS": "connect-os", + "OSM US Ped WG": "osm-us-pedestrian-wg", + "GATIS": "gatis", + "IMDF": "imdf", + "shared-row": "shared-row", + "Statewide Trails Standard (MA)": "statewide-trails-data-standard-ma", + "ISO GDF 5.1": "iso-20524-2", + "SAE J2735": "sae-j2735", + "NCHRP 08-177": "nchrp-08-177", + "NCHRP 07-31": "nchrp-07-31", + "MAT": "mat", + "OpenTrails": "opentrails", + "Cyclables Data Schema (FR)": "schema-amenagements-cyclables", + "Bike Parking Schema (FR)": "schema-stationnement-cyclable", + "APDS": "apds", + "IXSI": "ixsi", + "CurbLR": "curblr", + "ASAM OpenDRIVE": "asam-opendrive", +} + + +def read_sheet(path): + """The sheet's rows, reduced to COLUMNS. The header row is found by name.""" + with open(path, newline="", encoding="utf-8-sig") as fh: + rows = list(csv.reader(fh)) + for n, row in enumerate(rows): + cells = [c.strip() for c in row] + if "Specification" in cells and "Review Status" in cells: + index = {c: cells.index(c) for c in COLUMNS if c in cells} + break + else: + sys.exit("No header row with 'Specification' and 'Review Status' in %s" % path) + out = [] + for row in rows[n + 1:]: + rec = {c: (row[i].strip() if i < len(row) else "") for c, i in index.items()} + if rec.get("Specification"): + out.append(rec) + return out + + +def plan(sheet, specs): + """(issues, notes) after checking every row against data/.""" + by_id = {s["id"]: s for s in specs} + seen, issues, notes = {}, [], [] + for rec in sheet: + name = rec["Specification"] + if name not in SHEET_IDS: + notes.append("%s: not mapped to a registry id, skipped" % name) + continue + sid = SHEET_IDS[name] + if sid is None: + notes.append("%s: not published, skipped" % name) + continue + seen.setdefault(sid, []).append(rec) + for sid, recs in seen.items(): + if len(recs) > 1: + notes.append("%s: %d rows in the sheet, skipped until one is kept" % (sid, len(recs))) + continue + rec = recs[0] + status = SHEET_STATUS.get(rec["Review Status"].lower()) + if not status: + notes.append("%s: unknown review status, skipped" % sid) + continue + issues.append({ + "id": sid, + "title": "Maintainer review: %s" % by_id[sid]["short_name"], + "body": mr.make_body(sid, rec.get("Review Date")), + "labels": [mr.LABEL, mr.REVIEW_STATUS[status]["github_label"]], + }) + for sid in sorted(set(by_id) - set(seen)): + notes.append("%s: not in the sheet, no status shown" % sid) + return issues, notes + + +def gh(*args): + return subprocess.run(["gh", *args, "--repo", spec_data.GITHUB_REPO], + check=True, capture_output=True, text=True).stdout + + +def ensure_labels(): + have = {l["name"] for l in json.loads(gh("label", "list", "--limit", "200", "--json", "name"))} + colours = {"reviewed": "0E8A16"} + for key, meta in mr.REVIEW_STATUS.items(): + if meta["github_label"] not in have: + gh("label", "create", meta["github_label"], "--color", colours.get(key, "FBCA04")) + print("label created: %s" % meta["github_label"]) + + +def apply(issues): + ensure_labels() + existing = {} + for i in json.loads(gh("issue", "list", "--label", mr.LABEL, "--state", "all", + "--limit", "500", "--json", "number,body,labels")): + existing[mr.parse_body(i["body"]).get("specification id", "")] = i + for want in issues: + have = existing.get(want["id"]) + if have is None: + args = ["issue", "create", "--title", want["title"], "--body", want["body"]] + for l in want["labels"]: + args += ["--label", l] + print("created %s: %s" % (want["id"], gh(*args).strip())) + continue + num = str(have["number"]) + current = {l["name"] for l in have["labels"]} + args = ["issue", "edit", num, "--body", want["body"]] + for l in sorted(current - set(want["labels"])): + if l.startswith("review: "): + args += ["--remove-label", l] + for l in want["labels"]: + if l not in current: + args += ["--add-label", l] + gh(*args) + print("updated %s: #%s" % (want["id"], num)) + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("sheet") + ap.add_argument("--json", help="write the issues as GitHub API JSON for a local preview") + ap.add_argument("--apply", action="store_true", help="create or update the issues on GitHub") + args = ap.parse_args() + + data = spec_data.build_dataset() + issues, notes = plan(read_sheet(args.sheet), data["specs"]) + + for i in sorted(issues, key=lambda i: i["id"]): + rec, _ = mr.review_from_issue({"body": i["body"], "labels": i["labels"]}) + print("%-36s %-40s %s" % (i["id"], rec["status_label"], rec["date"] or "")) + print("\n%d issues." % len(issues)) + for n in notes: + print("note: " + n) + + if args.json: + with open(args.json, "w") as fh: + json.dump([{"number": n + 1, "title": i["title"], "body": i["body"], "updated_at": "", + "labels": [{"name": l} for l in i["labels"]]} + for n, i in enumerate(issues)], fh, indent=1) + print("wrote %s" % args.json) + + if args.apply: + apply(issues) + + +if __name__ == "__main__": + main() diff --git a/scripts/spec_api.py b/scripts/spec_api.py index 692b5ad..bc8ad1a 100644 --- a/scripts/spec_api.py +++ b/scripts/spec_api.py @@ -25,6 +25,7 @@ import io import json +import maintainer_review import spec_data API_DIR = "api" @@ -74,6 +75,7 @@ def full_record(spec, site_url): "rationale": c["rationale"] or None, "evidence_url": c["evidence_url"]} for c in spec["criteria"]], "principles_met": spec["criteria_met"], + "maintainer_review": maintainer_review.api_record(spec.get("review")), "page_url": _page(site_url, spec), "source_url": _source(spec["source_path"]), } @@ -102,7 +104,8 @@ def export_csv(records): cols = ["id", "short_name", "full_name", "status", "modes", "maintainer_ids", "maintainer_names", "licences", "homepage_url", "first_release_year", "lifecycle", "adoption_tier", "adopters_count", "adopters_count_updated", - "adopters_registry_url", "confirmed_by_maintainer"] + "adopters_registry_url", "confirmed_by_maintainer", + "maintainer_review_status", "maintainer_review_date"] principle_ids = [p["id"] for p in records[0]["principles"]] if records else [] for pid in principle_ids: cols += [pid + "_verdict", pid + "_rationale", pid + "_evidence_url"] @@ -131,6 +134,9 @@ def export_csv(records): "description": r["description"], "page_url": r["page_url"], "source_url": r["source_url"], } + mr = r["maintainer_review"] or {} + row["maintainer_review_status"] = (mr.get("status") or {}).get("id") + row["maintainer_review_date"] = mr.get("date") for p in r["principles"]: row[p["id"] + "_verdict"] = p["verdict"] row[p["id"] + "_rationale"] = p["rationale"] @@ -183,6 +189,10 @@ def obj(props, required=()): "adopters_registry_url": _nullable(url), "confirmed_by_maintainer": _nullable({"type": "boolean"}), }), + "MaintainerReview": obj({ + "status": obj({"id": _enum(maintainer_review.REVIEW_STATUS), "label": s}), + "date": _nullable(dict(s, format="date")), + }, ("status",)), "CatalogueRecord": obj({ "id": s, "short_name": s, "full_name": _nullable(s), "summary": _nullable(s), "status": _enum(V.STATUS), "modes": arr(_enum(V.MODES)), @@ -205,6 +215,8 @@ def obj(props, required=()): "adoption": ref("Adoption"), "principles": arr(ref("PrincipleVerdict")), "principles_met": i, + # allOf, because OpenAPI 3.0 ignores keys set next to a $ref. + "maintainer_review": dict(allOf=[ref("MaintainerReview")], nullable=True), "page_url": url, "source_url": dict(url, description="The entry's file on GitHub."), }, ("id", "short_name", "status", "principles")),