CI report

.github/scripts/ci_report.py

@@ -0,0 +1,187 @@
+"""Rebuilds the one CI report comment of the pull request whose head commit a finished workflow run tested."""
+
+import io
+import json
+import os
+import re
+import subprocess
+import sys
+import zipfile
+from datetime import datetime, timezone
+
+MARKER = "<!-- countly-ci-report -->"
+FRAGMENT_PREFIX = "ci-report-"
+FRAGMENT_FILE = "report.md"
+MAX_FRAGMENT_BYTES = 65536
+MAX_SUMMARY_CHARS = 200
+MAX_DETAILS_CHARS = 12000
+MAX_BODY_CHARS = 60000
+MAX_CELL_SUMMARIES = 3
+# a summary that reports a failure, which opens its section when the run failed
+FAILING = re.compile(
+    r"\b([1-9][0-9]* (failed|new findings?|of [0-9]+ scenarios differ)|has a blocking vulnerability)\b")
+ICONS = {
+    "success": "✅ passed",
+    "failure": "❌ failed",
+    "cancelled": "⚪ cancelled",
+    "timed_out": "⌛ timed out",
+    "skipped": "⏭️ skipped",
+    "neutral": "➖ neutral",
+    "action_required": "⚠️ action required",
+    "stale": "⚪ stale",
+    "startup_failure": "❌ did not start",
+}
+ORDER = {"failure": 0, "startup_failure": 0, "timed_out": 0, "action_required": 1, "running": 2, "cancelled": 3}
+
+
+def gh(*args, raw=False):
+    """Runs gh with the given arguments and returns its output, parsed as JSON unless raw."""
+    out = subprocess.run(["gh", *args], check=True, capture_output=True).stdout
+    return out if raw else json.loads(out or b"null")
+
+
+def paged(path):
+    """Returns every page of a list endpoint, merged, for endpoints that answer with a list or with one keyed list."""
+    pages = gh("api", "--paginate", "--slurp", path)
+    items = []
+    for page in pages:
+        if isinstance(page, list):
+            items.extend(page)
+        else:
+            items.extend(next(value for value in page.values() if isinstance(value, list)))
+    return items
+
+
+def pull_requests(repo, sha):
+    """Returns the numbers of the open pull requests whose head is the given commit."""
+    return [pr["number"] for pr in paged(f"repos/{repo}/pulls?state=open&per_page=100") if pr["head"]["sha"] == sha]
+
+
+def latest_runs(repo, sha, own_name):
+    """Returns the newest pull request run of each workflow for the commit, leaving out this report's own workflow."""
+    runs = {}
+    for run in paged(f"repos/{repo}/actions/runs?head_sha={sha}&event=pull_request&per_page=100"):
+        if run["name"] == own_name:
+            continue
+        kept = runs.get(run["name"])
+        if kept is None or (run["created_at"], run["id"]) > (kept["created_at"], kept["id"]):
+            runs[run["name"]] = run
+    return list(runs.values())
+
+
+def defused(text):
+    """Returns untrusted Markdown that cannot mention anyone, close the report's sections or fake its marker."""
+    text = text.replace(MARKER, "")
+    text = re.sub(r"@(?=[A-Za-z0-9])", "@&#8203;", text)
+    # a fragment's own collapsible sections become bold headings, since they would close the report's section
+    text = re.sub(r"<details[^>]*>\s*<summary[^>]*>(.*?)</summary>", r"**\1**", text, flags=re.IGNORECASE | re.DOTALL)
+    text = re.sub(r"</details\s*>", "", text, flags=re.IGNORECASE)
+    return re.sub(r"<(/?)(details|summary)", r"&lt;\1\2", text, flags=re.IGNORECASE)
+
+
+def fragments(repo, run):
+    """Returns the report fragments the run uploaded, each as a dict with key, title, summary and details."""
+    found = []
+    for artifact in paged(f"repos/{repo}/actions/runs/{run['id']}/artifacts?per_page=100"):
+        if not artifact["name"].startswith(FRAGMENT_PREFIX) or artifact.get("expired"):
+            continue
+        zipped = gh("api", f"repos/{repo}/actions/artifacts/{artifact['id']}/zip", raw=True)
+        archive = zipfile.ZipFile(io.BytesIO(zipped))
+        if FRAGMENT_FILE not in archive.namelist():
+            continue
+        key = artifact["name"][len(FRAGMENT_PREFIX):]
+        data = archive.read(FRAGMENT_FILE)[:MAX_FRAGMENT_BYTES].decode("utf-8", "replace")
+        lines = defused(data).replace("\r\n", "\n").split("\n")
+        title = lines[0].strip() or key
+        summary = (lines[1].strip() if len(lines) > 1 else "")[:MAX_SUMMARY_CHARS]
+        details = "\n".join(lines[2:]).strip()
+        if len(details) > MAX_DETAILS_CHARS:
+            details = details[:MAX_DETAILS_CHARS] + "\n\n…cut here; the run's job summary has the rest."
+        found.append({"key": key, "title": title, "summary": summary, "details": details})
+    return sorted(found, key=lambda fragment: fragment["key"])
+
+
+def failing_jobs(repo, run):
+    """Returns the names of the run's jobs that did not succeed or get skipped."""
+    jobs = paged(f"repos/{repo}/actions/runs/{run['id']}/jobs?per_page=100")
+    return [job["name"] for job in jobs if job.get("conclusion") not in (None, "success", "skipped", "neutral")]
+
+
+def state(run):
+    """Returns the run's conclusion, or "running" while it has none."""
+    return run["conclusion"] if run["status"] == "completed" and run["conclusion"] else "running"
+
+
+def label(run, fragment):
+    """Returns the fragment's title, or nothing when the title only repeats its workflow's name."""
+    return "" if fragment["title"].casefold() == run["name"].casefold() else fragment["title"]
+
+
+def joined(*parts):
+    """Returns the non-empty parts joined by a colon."""
+    return ": ".join(part for part in parts if part)
+
+
+def body(repo, sha, runs):
+    """Returns the comment: a table with one row per workflow, then a collapsible section per fragment."""
+    rows, sections = [], []
+    for run in sorted(runs, key=lambda run: (ORDER.get(state(run), 4), run["name"].lower())):
+        result = state(run)
+        found = fragments(repo, run) if result != "running" else []
+        parts = [joined(label(run, fragment), fragment["summary"]) for fragment in found if fragment["summary"]]
+        if len(parts) > MAX_CELL_SUMMARIES:
+            parts = parts[:MAX_CELL_SUMMARIES] + [f"{len(parts) - MAX_CELL_SUMMARIES} more below"]
+        failed = failing_jobs(repo, run) if result in ("failure", "timed_out") else []
+        if failed:
+            parts.insert(0, "Failed: " + ", ".join(failed))
+        summary = "; ".join(parts)
+        icon = "⏳ running" if result == "running" else ICONS.get(result, result)
+        cell = summary.replace("|", "\\|")
+        rows.append(f"| [{run['name']}]({run['html_url']}) | {icon} | {cell} |")
+        for fragment in found:
+            opened = " open" if result in ("failure", "timed_out") and FAILING.search(fragment["summary"]) else ""
+            heading = joined(run["name"], label(run, fragment))
+            if fragment["summary"]:
+                heading += f", {fragment['summary']}"
+            details = fragment["details"] or "No details."
+            sections.append(f"<details{opened}>\n<summary><b>{heading}</b></summary>\n\n{details}\n\n</details>\n")
+    when = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
+    head = [MARKER, "### CI report", "", "| Check | Result | Summary |", "| --- | --- | --- |", *rows, ""]
+    tail = [f"<sub>For {sha[:7]}, updated {when}. Each check's run page has its full log.</sub>"]
+    text = "\n".join(head + sections + tail)
+    while len(text) > MAX_BODY_CHARS and sections:
+        sections.pop()
+        text = "\n".join(head + sections + ["…more sections did not fit in one comment.", ""] + tail)
+    return text
+
+
+def publish(repo, number, text):
+    """Updates the pull request's report comment, or posts it when there is none yet."""
+    comments = paged(f"repos/{repo}/issues/{number}/comments?per_page=100")
+    mine = [comment for comment in comments
+            if comment["user"]["login"] == "github-actions[bot]" and comment["body"].startswith(MARKER)]
+    if mine:
+        gh("api", "--method", "PATCH", f"repos/{repo}/issues/comments/{mine[-1]['id']}", "-f", f"body={text}")
+    else:
+        gh("api", "--method", "POST", f"repos/{repo}/issues/{number}/comments", "-f", f"body={text}")
+
+
+def main():
+    """Rebuilds the report for the commit in HEAD_SHA, or prints it when DRY_RUN is set."""
+    repo, sha = os.environ["REPO"], os.environ["HEAD_SHA"]
+    numbers = [int(os.environ["PR"])] if os.environ.get("PR") else pull_requests(repo, sha)
+    if not numbers:
+        print(f"No open pull request has {sha} as its head, so there is nothing to report on.")
+        return 0
+    text = body(repo, sha, latest_runs(repo, sha, os.environ.get("OWN_WORKFLOW", "CI report")))
+    if os.environ.get("DRY_RUN"):
+        print(text)
+        return 0
+    for number in numbers:
+        publish(repo, number, text)
+        print(f"Updated the CI report of #{number}.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/scripts/public_api_report.sh

@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
-# Writes the report of a pull request's public API changes to the job summary, and to the step output "report", which
-# the workflow's comment job posts on the pull request. The comment lists at most 200 lines per section.
+# Writes the report of a pull request's public API changes to the job summary, to the step output "report", which
+# the workflow's comment job posts on the pull request, and to $RUNNER_TEMP/ci-report-public-api/report.md, the
+# fragment the CI report comment shows. The comment and the fragment list at most 200 lines per section.
 #   $1 a file with one removed or changed declaration per line
 #   $2 a file with one declaration per line that the pull request adds to the public API
 set -euo pipefail
@@ -44,3 +45,15 @@ delimiter="EOF_$(od -An -N8 -tx1 /dev/urandom | tr -d ' \n')"
   report 200
   echo "$delimiter"
 } >> "$GITHUB_OUTPUT"
+
+removed=$(grep -c . "$breaking" || true)
+new=$(grep -c . "$added" || true)
+summary='No public API change'
+if [ "$removed" -gt 0 ] || [ "$new" -gt 0 ]; then
+  summary="$removed removed or changed, $new added"
+fi
+mkdir -p "$RUNNER_TEMP/ci-report-public-api"
+{
+  printf 'Public API\n%s\n' "$summary"
+  report 200 | tail -n +3
+} > "$RUNNER_TEMP/ci-report-public-api/report.md"

.github/workflows/ci-report.yml

@@ -0,0 +1,47 @@
+name: CI report
+
+# Keeps one comment on each pull request with the result of every workflow that ran for its head commit, a table
+# first, then one collapsible section per report fragment (an artifact named ci-report-<key> holding report.md: a
+# title line, a summary line, then the details). It runs from the default branch after each listed workflow, so a
+# fork's pull request gets its report too: nothing from the pull request is checked out or run, and the fragments
+# are read as untrusted text.
+on:  # zizmor: ignore[dangerous-triggers] the job reads the pull request's artifacts as text and never runs its code
+  workflow_run:
+    workflows:
+      - "CodeQL"
+      - "Package Managers"
+      - "Public API changelog"
+      - "Public API changes"
+      - "Run Tests"
+      - "Static analysis"
+    types: [completed]
+
+permissions: {}
+
+concurrency:
+  group: ci-report-${{ github.event.workflow_run.head_sha }}
+  cancel-in-progress: false
+
+jobs:
+  report:
+    name: Update the CI report
+    if: github.event.workflow_run.event == 'pull_request'
+    runs-on: ubuntu-24.04
+    permissions:
+      actions: read
+      contents: read
+      pull-requests: write
+    steps:
+      - name: Check out the report script
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+          sparse-checkout: .github/scripts/ci_report.py
+          sparse-checkout-cone-mode: false
+
+      - name: Rebuild the report comment
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+          HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
+        run: python3 .github/scripts/ci_report.py

.github/workflows/public_api.yml

@@ -46,6 +46,14 @@ jobs:
         id: report
         run: bash .github/scripts/public_api_report.sh "$RUNNER_TEMP/breaking.txt" "$RUNNER_TEMP/added.txt"
 
+      - name: Upload the CI report fragment
+        if: ${{ !cancelled() && steps.report.outcome == 'success' }}
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: ci-report-public-api
+          path: ${{ runner.temp }}/ci-report-public-api
+          if-no-files-found: ignore
+
       - name: Require a CHANGELOG entry for breaking changes
         run: |
           if [ ! -s "$RUNNER_TEMP/breaking.txt" ]; then

.github/workflows/static_analysis.yml

@@ -64,4 +64,29 @@ jobs:
           if [ -f "$RUNNER_TEMP/clang-tidy-base.sarif" ]; then
             base=(--base "$RUNNER_TEMP/clang-tidy-base.sarif" --base-root "$RUNNER_TEMP/base")
           fi
-          python3 .github/scripts/sarif_summary.py clang-tidy "$RUNNER_TEMP/clang-tidy.sarif" "${base[@]}" | tee -a "$GITHUB_STEP_SUMMARY"
+          python3 .github/scripts/sarif_summary.py clang-tidy "$RUNNER_TEMP/clang-tidy.sarif" "${base[@]}" | tee -a "$GITHUB_STEP_SUMMARY" "$RUNNER_TEMP/static-analysis.md"
+
+      - name: Write the CI report fragment
+        if: ${{ !cancelled() }}
+        run: |
+          [ -s "$RUNNER_TEMP/static-analysis.md" ] || exit 0
+          new=$(sed -n 's/^### New compared with the base branch: \([0-9][0-9]*\)$/\1/p' "$RUNNER_TEMP/static-analysis.md")
+          case "$new" in
+            '') summary=$(sed -n '1s/^### //p' "$RUNNER_TEMP/static-analysis.md") ;;
+            0) summary="No new findings" ;;
+            1) summary="1 new finding" ;;
+            *) summary="$new new findings" ;;
+          esac
+          mkdir -p "$RUNNER_TEMP/ci-report-static-analysis"
+          {
+            printf 'Static analysis\n%s\n' "$summary"
+            cat "$RUNNER_TEMP/static-analysis.md"
+          } > "$RUNNER_TEMP/ci-report-static-analysis/report.md"
+
+      - name: Upload the CI report fragment
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: ci-report-static-analysis
+          path: ${{ runner.temp }}/ci-report-static-analysis
+          if-no-files-found: ignore

.github/workflows/tests.yml

@@ -351,4 +351,24 @@ jobs:
             echo "| Metric | Covered | Count |"
             echo "|---|---|---|"
             awk '/^(lines|functions|branches):/ { gsub(/[():]/, ""); printf "| %s | %s | %s/%s |\n", $1, $2, $3, $6 }' "$RUNNER_TEMP/coverage.txt"
-          } >> "$GITHUB_STEP_SUMMARY"
+          } > "$RUNNER_TEMP/coverage.md"
+          cat "$RUNNER_TEMP/coverage.md" >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Write the CI report fragment (coverage)
+        if: ${{ !cancelled() }}
+        run: |
+          [ -s "$RUNNER_TEMP/coverage.md" ] || exit 0
+          summary=$(awk '$2 == "lines" { print "Line coverage " $4 }' "$RUNNER_TEMP/coverage.md")
+          mkdir -p "$RUNNER_TEMP/ci-report-coverage"
+          {
+            printf 'Coverage\n%s\n' "${summary:-See details}"
+            cat "$RUNNER_TEMP/coverage.md"
+          } > "$RUNNER_TEMP/ci-report-coverage/report.md"
+
+      - name: Upload the CI report fragment (coverage)
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
+        with:
+          name: ci-report-coverage
+          path: ${{ runner.temp }}/ci-report-coverage
+          if-no-files-found: ignore