From a8a11abd71c82c290b021823db31df51b809d4be Mon Sep 17 00:00:00 2001 From: Frederic Simard Date: Tue, 20 Oct 2026 17:50:05 -0400 Subject: [PATCH] ci: add systems.csv structure and new-URL checks on pull requests Adds a pull request workflow that runs when systems.csv, the workflow, or its scripts change. It is adapted from the GBFS repository's systems.csv checks (MobilityData/gbfs#894), keeping only the structure validation and the new-URL check. The sort job is not included, and systems.json is not checked because it is generated from systems.csv after merge. scripts/validate-systems-csv.js checks that the header matches the GOFS columns (Country Code, Name, Location, Access Information, URL, Supported Versions), that every row has exactly 6 correctly quoted fields, and that Country Code, Name, Location and URL are not empty. Access Information is optional. An empty Supported Versions is reported as a warning only. Errors are emitted as GitHub Actions annotations so they appear inline on the PR. scripts/check-new-urls.js checks only URLs that the PR adds compared to the base branch, so a PR never fails for a dead link it did not introduce. It checks the URL and Access Information columns. The "forthcoming" placeholder in URL is skipped, and Access Information is only checked when it starts with http:// or https://. Every new URL must return 2xx after following redirects, except a feed URL on a row whose Access Information is filled in: that feed needs a login, so 401 or 403 also passes because the server answered and only refused the missing credentials. An unreachable URL always fails. The SSRF guard, 200-URL cap, timeouts, retry and concurrency limits from the original script are kept unchanged. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/systems_csv_pr_checks.yml | 53 ++++ scripts/check-new-urls.js | 312 ++++++++++++++++++++ scripts/validate-systems-csv.js | 197 ++++++++++++ 3 files changed, 562 insertions(+) create mode 100644 .github/workflows/systems_csv_pr_checks.yml create mode 100755 scripts/check-new-urls.js create mode 100755 scripts/validate-systems-csv.js diff --git a/.github/workflows/systems_csv_pr_checks.yml b/.github/workflows/systems_csv_pr_checks.yml new file mode 100644 index 0000000..1a24738 --- /dev/null +++ b/.github/workflows/systems_csv_pr_checks.yml @@ -0,0 +1,53 @@ +name: systems.csv PR checks + +# Runs whenever a pull request adds or changes systems.csv, this workflow, or the scripts it runs. +# 1. validate-structure - fail on malformed rows / missing required values. +# 2. check-new-urls - fail if any newly added URL does not return HTTP 200-299. +# systems.json is not checked: it is generated from systems.csv after merge. +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - systems.csv + - .github/workflows/systems_csv_pr_checks.yml + - scripts/validate-systems-csv.js + - scripts/check-new-urls.js + +# Cancel superseded runs when new commits are pushed to the same PR. +concurrency: + group: systems-csv-${{ github.event.pull_request.number }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + validate-structure: + name: Validate structure + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7.0.1 + - uses: actions/setup-node@v7.0.0 + with: + node-version: 24 + - name: Validate systems.csv structure + run: node scripts/validate-systems-csv.js systems.csv + + check-new-urls: + name: Check new URLs return 2xx + runs-on: ubuntu-latest + # Only spend network requests once the file is known to be well-formed. + needs: validate-structure + steps: + - uses: actions/checkout@v7.0.1 + with: + fetch-depth: 0 + - uses: actions/setup-node@v7.0.0 + with: + node-version: 24 + - name: Extract base systems.csv + run: | + git show "${{ github.event.pull_request.base.sha }}:systems.csv" > /tmp/systems.base.csv 2>/dev/null \ + || : > /tmp/systems.base.csv + - name: Check newly added URLs + run: node scripts/check-new-urls.js /tmp/systems.base.csv systems.csv diff --git a/scripts/check-new-urls.js b/scripts/check-new-urls.js new file mode 100755 index 0000000..fd36c97 --- /dev/null +++ b/scripts/check-new-urls.js @@ -0,0 +1,312 @@ +#!/usr/bin/env node +'use strict'; +// +// Check that URLs *newly added* in this PR return an HTTP status in the 200-299 range. +// +// "Newly added" = a URL value present in the head systems.csv that was not present +// anywhere in the base systems.csv. This deliberately ignores pre-existing URLs so a +// PR is never failed for a dead link it did not introduce. +// +// Two columns are checked: "URL" (the GOFS feed) and "Access Information" (where to get access to the feed). Redirects are followed; the FINAL status must be 2xx. +// +// A feed that needs a login is one whose row has "Access Information" filled in. Its "URL" may also return 401 or 403, since the server is reachable and only refuses the missing credentials. Every other URL must return 2xx, and an unreachable URL always fails. +// +// The "forthcoming" placeholder in "URL" is skipped. "Access Information" may hold free text, so it is only checked when the value starts with http:// or https://. +// +// URLs are compared as exact strings (after trimming whitespace). Any change to an existing URL, even adding a trailing slash or switching http to https, makes it count as newly added, so it gets checked. +// +// Limits and safeguards: +// - Hosts that resolve to loopback, private, link-local, CGNAT or ULA addresses are refused, and every redirect hop is re-checked (SSRF guard, see below). +// - Only http and https URLs are accepted. +// - At most 200 new URLs per PR; above that the check fails and asks for a smaller PR. +// - Each request uses GET, times out after 20 seconds, and follows up to 5 redirects. +// - A network error or timeout is retried once; an HTTP error status is not retried. +// - Up to 6 URLs are checked in parallel. +// +// Usage: node scripts/check-new-urls.js [baseCsv] [headCsv] +// baseCsv : systems.csv from the PR base; if omitted, missing or empty, every URL in headCsv is treated as new +// headCsv : systems.csv from the PR head (default: systems.csv) +// +// Exit codes: +// 0 no new URLs, or every new URL returned 2xx +// 1 at least one new URL failed, or the 200-URL limit was exceeded +// 2 unexpected failure (for example headCsv cannot be read) +// +const fs = require('fs'); +const net = require('net'); +const dns = require('dns').promises; +const http = require('http'); +const https = require('https'); +const { URL } = require('url'); + +const baseFile = process.argv[2] || ''; +const headFile = process.argv[3] || 'systems.csv'; + +const FEED_COLUMN = 'URL'; +const ACCESS_COLUMN = 'Access Information'; +const URL_COLUMNS = [FEED_COLUMN, ACCESS_COLUMN]; +// Statuses that count as reachable for a feed URL that needs a login. +const AUTH_STATUSES = new Set([401, 403]); +// Columns that may hold free text: only values that look like http(s) URLs are checked. +const URL_IF_LINK_COLUMNS = new Set(['Access Information']); +// Placeholder values that are not URLs and are never checked (compared case-insensitively). +const PLACEHOLDERS = new Set(['forthcoming']); +const TIMEOUT_MS = 20000; +const MAX_REDIRECTS = 5; +const CONCURRENCY = 6; +// This job runs on untrusted fork PRs, so cap how many outbound checks a single +// PR can trigger. A legitimate contribution adds a handful of systems at a time. +const MAX_NEW_URLS = 200; +const USER_AGENT = + 'Mozilla/5.0 (compatible; gofs-ci/1.0; +https://github.com/MobilityData/GOFS)'; + +// --- SSRF guard ------------------------------------------------------------ +// URLs come from untrusted PR authors and are fetched by the CI runner, so +// refuse to connect to loopback / private / link-local / CGNAT / ULA addresses +// (this blocks probing of the runner's own network and cloud metadata endpoints +// such as 169.254.169.254). Hostnames are resolved and every resolved address +// is checked, and each redirect hop is re-validated. +function isBlockedIPv4(ip) { + const p = ip.split('.').map(Number); + if (p.length !== 4 || p.some((n) => Number.isNaN(n) || n < 0 || n > 255)) return true; + const [a, b] = p; + if (a === 0) return true; // 0.0.0.0/8 + if (a === 10) return true; // 10.0.0.0/8 + if (a === 127) return true; // loopback + if (a === 169 && b === 254) return true; // link-local (incl. metadata) + if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12 + if (a === 192 && b === 168) return true; // 192.168.0.0/16 + if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT 100.64.0.0/10 + return false; +} +function isBlockedIPv6(ip) { + const s = ip.toLowerCase().split('%')[0]; // strip zone id + if (s === '::1' || s === '::') return true; // loopback / unspecified + const mapped = s.match(/::ffff:(\d+\.\d+\.\d+\.\d+)$/); // IPv4-mapped + if (mapped) return isBlockedIPv4(mapped[1]); + if (/^fe[89ab]/.test(s)) return true; // fe80::/10 link-local + if (/^f[cd]/.test(s)) return true; // fc00::/7 ULA + return false; +} +function isBlockedIp(ip) { + if (net.isIPv4(ip)) return isBlockedIPv4(ip); + if (net.isIPv6(ip)) return isBlockedIPv6(ip); + return true; // not a recognizable IP -> treat as unsafe +} +async function hostIsSafe(hostname) { + const h = hostname.toLowerCase().replace(/^\[|\]$/g, ''); // strip IPv6 brackets + if (h === 'localhost' || h.endsWith('.localhost')) return false; + if (net.isIP(h)) return !isBlockedIp(h); + let addrs; + try { + addrs = await dns.lookup(h, { all: true }); + } catch (e) { + return false; // unresolvable -> unsafe (also fails the 2xx check) + } + if (!addrs.length) return false; + return addrs.every((a) => !isBlockedIp(a.address)); +} + +// --- CSV parsing (same shape as validate-systems-csv.js) ------------------- +function parseCSV(text) { + const records = []; + let field = ''; + let fields = []; + let inQuotes = false; + let i = 0; + let hasContent = false; + + const pushField = () => { fields.push(field); field = ''; }; + const pushRecord = () => { records.push(fields); fields = []; hasContent = false; }; + + while (i < text.length) { + const c = text[i]; + if (inQuotes) { + if (c === '"') { + if (text[i + 1] === '"') { field += '"'; i += 2; continue; } + inQuotes = false; i++; continue; + } + field += c; i++; continue; + } + if (c === '"') { inQuotes = true; hasContent = true; i++; continue; } + if (c === ',') { pushField(); i++; continue; } + if (c === '\r') { i++; continue; } + if (c === '\n') { pushField(); pushRecord(); i++; continue; } + field += c; hasContent = true; i++; + } + if (hasContent || field.length > 0 || fields.length > 0) { pushField(); pushRecord(); } + return records; +} + +// Return a Map of URL value (from URL_COLUMNS) -> authRequired, found in a CSV string. +// authRequired is true when the URL is a feed URL ("URL" column) on a row whose "Access Information" is filled in, meaning the feed needs a login. +function urlsFrom(text) { + const urls = new Map(); + if (!text || !text.trim()) return urls; + const records = parseCSV(text); + if (records.length === 0) return urls; + const header = records[0]; + const accessIdx = header.indexOf(ACCESS_COLUMN); + const cols = URL_COLUMNS + .map((name) => ({ name, i: header.indexOf(name) })) + .filter((c) => c.i >= 0); + for (let r = 1; r < records.length; r++) { + const row = records[r]; + const needsLogin = accessIdx >= 0 && (row[accessIdx] || '').trim() !== ''; + for (const { name, i } of cols) { + const v = (row[i] || '').trim(); + if (!v) continue; + if (PLACEHOLDERS.has(v.toLowerCase())) continue; + if (URL_IF_LINK_COLUMNS.has(name) && !/^https?:\/\//i.test(v)) continue; + const authRequired = name === FEED_COLUMN && needsLogin; + urls.set(v, urls.get(v) || authRequired); + } + } + return urls; +} + +// --- HTTP check ------------------------------------------------------------ +// Resolve one URL, following redirects, resolving to { url, status, ok, error }. +// Async because the SSRF host check resolves DNS before any connection is made. +async function checkOnce(rawUrl, redirectsLeft) { + let target; + try { + target = new URL(rawUrl); + } catch (e) { + return { url: rawUrl, status: null, ok: false, error: `invalid URL: ${e.message}` }; + } + if (target.protocol !== 'http:' && target.protocol !== 'https:') { + return { url: rawUrl, status: null, ok: false, error: `unsupported protocol: ${target.protocol}` }; + } + if (!(await hostIsSafe(target.hostname))) { + return { + url: rawUrl, + status: null, + ok: false, + error: `blocked host (local/private address not allowed): ${target.hostname}`, + }; + } + + return new Promise((resolve) => { + const lib = target.protocol === 'https:' ? https : http; + const req = lib.request( + target, + { + method: 'GET', + headers: { 'User-Agent': USER_AGENT, Accept: '*/*' }, + timeout: TIMEOUT_MS, + }, + (res) => { + const status = res.statusCode; + // Follow redirects to a final status. + if (status >= 300 && status < 400 && res.headers.location && redirectsLeft > 0) { + res.resume(); // discard body + let next; + try { + next = new URL(res.headers.location, target).toString(); + } catch (e) { + resolve({ url: rawUrl, status, ok: false, error: `bad redirect target: ${res.headers.location}` }); + return; + } + resolve(checkOnce(next, redirectsLeft - 1)); + return; + } + res.resume(); // drain so the socket can be freed + resolve({ url: rawUrl, status, ok: status >= 200 && status <= 299, error: null }); + } + ); + + req.on('timeout', () => { req.destroy(new Error(`timeout after ${TIMEOUT_MS}ms`)); }); + req.on('error', (e) => { + resolve({ url: rawUrl, status: null, ok: false, error: e.message }); + }); + req.end(); + }); +} + +// Check a URL with one retry on transient failure (network error / timeout). +async function checkUrl(rawUrl) { + let result = await checkOnce(rawUrl, MAX_REDIRECTS); + if (!result.ok && result.status === null) { + result = await checkOnce(rawUrl, MAX_REDIRECTS); + } + return result; +} + +async function runPool(items, worker, size) { + const results = new Array(items.length); + let next = 0; + async function drain() { + while (next < items.length) { + const cur = next++; + results[cur] = await worker(items[cur]); + } + } + await Promise.all(Array.from({ length: Math.min(size, items.length) }, drain)); + return results; +} + +async function main() { + const headText = fs.readFileSync(headFile, 'utf8'); + let baseText = ''; + if (baseFile && fs.existsSync(baseFile)) { + baseText = fs.readFileSync(baseFile, 'utf8'); + } + + const baseUrls = urlsFrom(baseText); + const headUrls = urlsFrom(headText); + const newUrls = [...headUrls.keys()].filter((u) => !baseUrls.has(u)).sort(); + + if (newUrls.length === 0) { + console.log('No newly added URLs to check.'); + process.exit(0); + } + + if (newUrls.length > MAX_NEW_URLS) { + console.log( + `::error::This PR adds ${newUrls.length} new URLs, exceeding the limit of ${MAX_NEW_URLS}. ` + + `Please split it into smaller pull requests.` + ); + process.exit(1); + } + + console.log(`Checking ${newUrls.length} newly added URL(s) for HTTP 200-299 (401/403 allowed for feeds that need a login)...\n`); + const results = await runPool(newUrls, checkUrl, CONCURRENCY); + + // A feed that needs a login passes on 401/403: the server answered, it only refused the missing credentials. + for (const r of results) { + if (!r.ok && headUrls.get(r.url) && AUTH_STATUSES.has(r.status)) { + r.ok = true; + r.authOnly = true; + } + } + + const failed = []; + for (const r of results) { + if (r.ok) { + console.log(` ok ${r.status} ${r.url}${r.authOnly ? ' (login required)' : ''}`); + } else { + failed.push(r); + const detail = r.status !== null ? `HTTP ${r.status}` : r.error; + console.log(` FAIL ${detail} ${r.url}`); + } + } + + if (failed.length > 0) { + console.log(''); + for (const r of failed) { + const detail = r.status !== null ? `returned HTTP ${r.status}` : `error: ${r.error}`; + console.log(`::error::New URL ${detail}: ${r.url}`); + } + console.log(`\n${failed.length} of ${newUrls.length} new URL(s) failed the check.`); + process.exit(1); + } + + console.log(`\nAll ${newUrls.length} new URL(s) passed.`); + process.exit(0); +} + +main().catch((e) => { + console.error(`::error::check-new-urls.js failed: ${e.stack || e.message}`); + process.exit(2); +}); diff --git a/scripts/validate-systems-csv.js b/scripts/validate-systems-csv.js new file mode 100755 index 0000000..ef46321 --- /dev/null +++ b/scripts/validate-systems-csv.js @@ -0,0 +1,197 @@ +#!/usr/bin/env node +'use strict'; +// +// Validate the structural integrity of systems.csv. +// +// Checks: +// 1. The header row exactly matches the expected GOFS systems schema (order + names). +// 2. Every data row parses to exactly 6 correctly-quoted fields (catches missing +// values, extra commas, and broken/unterminated quoting). +// 3. Required columns must be non-empty -> ERROR (fails the build). +// Country Code, Name, Location, URL +// 4. "Supported Versions" empty -> WARNING (surfaced to author/reviewers, non-fatal). +// 5. "Access Information" is optional and may be empty. +// +// A value containing only whitespace counts as empty. Fully blank lines (such as a trailing newline) are ignored. +// +// Emits GitHub Actions annotations (::error / ::warning) so issues appear inline on the PR. +// +// Usage: node scripts/validate-systems-csv.js [path-to-csv] (default: systems.csv) +// +// Exit codes: +// 0 no errors (warnings may still be reported) +// 1 one or more errors, or the file is empty +// 2 file cannot be read +// +const fs = require('fs'); + +const FILE = process.argv[2] || 'systems.csv'; + +const EXPECTED_HEADER = [ + 'Country Code', + 'Name', + 'Location', + 'Access Information', + 'URL', + 'Supported Versions', +]; + +// Columns that may legitimately be empty (no error, no warning). +const OPTIONAL = new Set(['Access Information']); + +// Columns whose emptiness is only a warning. +const WARN_IF_EMPTY = new Set(['Supported Versions']); + +// Escape a GitHub Actions workflow-command message so newlines, carriage +// returns, and percent signs survive intact instead of truncating the +// annotation at the first newline. +function ghEscapeData(s) { + return String(s).replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A'); +} + +// Parse CSV into records, tracking the 1-based source line where each record starts. +// Returns { records: [{ fields, line }], error: {line, message} | null }. +function parseCSV(text) { + const records = []; + let field = ''; + let fields = []; + let inQuotes = false; + let line = 1; // current line number in the source + let recordStartLine = 1; + let started = false; // whether the current record has any content + let i = 0; + + const pushField = () => { fields.push(field); field = ''; }; + const pushRecord = () => { records.push({ fields, line: recordStartLine }); fields = []; started = false; }; + + while (i < text.length) { + const c = text[i]; + + if (!started && !inQuotes) { + recordStartLine = line; + started = true; + } + + if (inQuotes) { + if (c === '"') { + if (text[i + 1] === '"') { field += '"'; i += 2; continue; } + inQuotes = false; i++; continue; + } + if (c === '\n') { field += '\n'; line++; i++; continue; } + field += c; i++; continue; + } + + if (c === '"') { inQuotes = true; i++; continue; } + if (c === ',') { pushField(); i++; continue; } + if (c === '\r') { i++; continue; } + if (c === '\n') { + pushField(); + pushRecord(); + line++; i++; + continue; + } + field += c; i++; + } + + if (inQuotes) { + return { records, error: { line: recordStartLine, message: 'Unterminated quoted field (broken CSV quoting).' } }; + } + + // Flush trailing record if the file did not end with a newline and had content. + if (started || field.length > 0 || fields.length > 0) { + pushField(); + pushRecord(); + } + + return { records, error: null }; +} + +function isBlankRecord(fields) { + return fields.length === 1 && fields[0].trim() === ''; +} + +function main() { + let text; + try { + text = fs.readFileSync(FILE, 'utf8'); + } catch (e) { + console.error(`::error file=${FILE}::Cannot read ${FILE}: ${e.message}`); + process.exit(2); + } + + const { records, error } = parseCSV(text); + const errors = []; + const warnings = []; + + if (error) { + errors.push({ line: error.line, message: error.message }); + } + + if (records.length === 0) { + console.error(`::error file=${FILE}::${FILE} is empty.`); + process.exit(1); + } + + // 1. Header check. + const header = records[0].fields; + const headerOk = + header.length === EXPECTED_HEADER.length && + EXPECTED_HEADER.every((h, idx) => header[idx] === h); + if (!headerOk) { + errors.push({ + line: 1, + message: + `Header does not match the expected schema.\n` + + ` expected: ${EXPECTED_HEADER.join(',')}\n` + + ` found: ${header.join(',')}`, + }); + } + + const N = EXPECTED_HEADER.length; + + // 2 + 3 + 4. Per-row checks. + for (let r = 1; r < records.length; r++) { + const { fields, line } = records[r]; + + // Ignore fully blank lines (e.g. a trailing newline at end of file). + if (isBlankRecord(fields)) continue; + + if (fields.length !== N) { + errors.push({ + line, + message: `Expected ${N} fields but found ${fields.length}. Check for missing values, stray commas, or unquoted commas.`, + }); + // Field-count is wrong; skip per-column emptiness to avoid noise. + continue; + } + + for (let c = 0; c < N; c++) { + const name = EXPECTED_HEADER[c]; + const empty = fields[c].trim() === ''; + if (!empty) continue; + if (OPTIONAL.has(name)) continue; + if (WARN_IF_EMPTY.has(name)) { + warnings.push({ line, message: `Column "${name}" is empty.` }); + } else { + errors.push({ line, message: `Required column "${name}" is empty.` }); + } + } + } + + for (const w of warnings) { + console.log(`::warning file=${FILE},line=${w.line}::${ghEscapeData(w.message)}`); + } + for (const e of errors) { + console.log(`::error file=${FILE},line=${e.line}::${ghEscapeData(e.message)}`); + } + + const dataRows = records.length - 1; + console.log( + `\nsystems.csv structure check: ${dataRows} data rows, ` + + `${errors.length} error(s), ${warnings.length} warning(s).` + ); + + process.exit(errors.length > 0 ? 1 : 0); +} + +main();