Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions .github/workflows/systems_csv_pr_checks.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
name: systems.csv PR checks

# Runs whenever a pull request adds or changes systems.csv, this workflow, or the scripts it runs.
# 1. validate-structure - fail on malformed rows / missing required values.
# 2. check-new-urls - fail if any newly added URL does not return HTTP 200-299.
# systems.json is not checked: it is generated from systems.csv after merge.
on:
pull_request:
types: [opened, synchronize, reopened]
paths:
- systems.csv
- .github/workflows/systems_csv_pr_checks.yml
- scripts/validate-systems-csv.js
- scripts/check-new-urls.js

# Cancel superseded runs when new commits are pushed to the same PR.
concurrency:
group: systems-csv-${{ github.event.pull_request.number }}
cancel-in-progress: true

permissions:
contents: read

jobs:
validate-structure:
name: Validate structure
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7.0.1
- uses: actions/setup-node@v7.0.0
with:
node-version: 24
- name: Validate systems.csv structure
run: node scripts/validate-systems-csv.js systems.csv

check-new-urls:
name: Check new URLs return 2xx
runs-on: ubuntu-latest
# Only spend network requests once the file is known to be well-formed.
needs: validate-structure
steps:
- uses: actions/checkout@v7.0.1
with:
fetch-depth: 0
- uses: actions/setup-node@v7.0.0
with:
node-version: 24
- name: Extract base systems.csv
run: |
git show "${{ github.event.pull_request.base.sha }}:systems.csv" > /tmp/systems.base.csv 2>/dev/null \
|| : > /tmp/systems.base.csv
- name: Check newly added URLs
run: node scripts/check-new-urls.js /tmp/systems.base.csv systems.csv
312 changes: 312 additions & 0 deletions scripts/check-new-urls.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,312 @@
#!/usr/bin/env node
'use strict';
//
// Check that URLs *newly added* in this PR return an HTTP status in the 200-299 range.
//
// "Newly added" = a URL value present in the head systems.csv that was not present
// anywhere in the base systems.csv. This deliberately ignores pre-existing URLs so a
// PR is never failed for a dead link it did not introduce.
//
// Two columns are checked: "URL" (the GOFS feed) and "Access Information" (where to get access to the feed). Redirects are followed; the FINAL status must be 2xx.
//
// A feed that needs a login is one whose row has "Access Information" filled in. Its "URL" may also return 401 or 403, since the server is reachable and only refuses the missing credentials. Every other URL must return 2xx, and an unreachable URL always fails.
//
// The "forthcoming" placeholder in "URL" is skipped. "Access Information" may hold free text, so it is only checked when the value starts with http:// or https://.
//
// URLs are compared as exact strings (after trimming whitespace). Any change to an existing URL, even adding a trailing slash or switching http to https, makes it count as newly added, so it gets checked.
//
// Limits and safeguards:
// - Hosts that resolve to loopback, private, link-local, CGNAT or ULA addresses are refused, and every redirect hop is re-checked (SSRF guard, see below).
// - Only http and https URLs are accepted.
// - At most 200 new URLs per PR; above that the check fails and asks for a smaller PR.
// - Each request uses GET, times out after 20 seconds, and follows up to 5 redirects.
// - A network error or timeout is retried once; an HTTP error status is not retried.
// - Up to 6 URLs are checked in parallel.
//
// Usage: node scripts/check-new-urls.js [baseCsv] [headCsv]
// baseCsv : systems.csv from the PR base; if omitted, missing or empty, every URL in headCsv is treated as new
// headCsv : systems.csv from the PR head (default: systems.csv)
//
// Exit codes:
// 0 no new URLs, or every new URL returned 2xx
// 1 at least one new URL failed, or the 200-URL limit was exceeded
// 2 unexpected failure (for example headCsv cannot be read)
//
const fs = require('fs');
const net = require('net');
const dns = require('dns').promises;
const http = require('http');
const https = require('https');
const { URL } = require('url');

const baseFile = process.argv[2] || '';
const headFile = process.argv[3] || 'systems.csv';

const FEED_COLUMN = 'URL';
const ACCESS_COLUMN = 'Access Information';
const URL_COLUMNS = [FEED_COLUMN, ACCESS_COLUMN];
// Statuses that count as reachable for a feed URL that needs a login.
const AUTH_STATUSES = new Set([401, 403]);
// Columns that may hold free text: only values that look like http(s) URLs are checked.
const URL_IF_LINK_COLUMNS = new Set(['Access Information']);
// Placeholder values that are not URLs and are never checked (compared case-insensitively).
const PLACEHOLDERS = new Set(['forthcoming']);
const TIMEOUT_MS = 20000;
const MAX_REDIRECTS = 5;
const CONCURRENCY = 6;
// This job runs on untrusted fork PRs, so cap how many outbound checks a single
// PR can trigger. A legitimate contribution adds a handful of systems at a time.
const MAX_NEW_URLS = 200;
const USER_AGENT =
'Mozilla/5.0 (compatible; gofs-ci/1.0; +https://github.com/MobilityData/GOFS)';

// --- SSRF guard ------------------------------------------------------------
// URLs come from untrusted PR authors and are fetched by the CI runner, so
// refuse to connect to loopback / private / link-local / CGNAT / ULA addresses
// (this blocks probing of the runner's own network and cloud metadata endpoints
// such as 169.254.169.254). Hostnames are resolved and every resolved address
// is checked, and each redirect hop is re-validated.
function isBlockedIPv4(ip) {
const p = ip.split('.').map(Number);
if (p.length !== 4 || p.some((n) => Number.isNaN(n) || n < 0 || n > 255)) return true;
const [a, b] = p;
if (a === 0) return true; // 0.0.0.0/8
if (a === 10) return true; // 10.0.0.0/8
if (a === 127) return true; // loopback
if (a === 169 && b === 254) return true; // link-local (incl. metadata)
if (a === 172 && b >= 16 && b <= 31) return true; // 172.16.0.0/12
if (a === 192 && b === 168) return true; // 192.168.0.0/16
if (a === 100 && b >= 64 && b <= 127) return true; // CGNAT 100.64.0.0/10
return false;
}
function isBlockedIPv6(ip) {
const s = ip.toLowerCase().split('%')[0]; // strip zone id
if (s === '::1' || s === '::') return true; // loopback / unspecified
const mapped = s.match(/::ffff:(\d+\.\d+\.\d+\.\d+)$/); // IPv4-mapped
if (mapped) return isBlockedIPv4(mapped[1]);
if (/^fe[89ab]/.test(s)) return true; // fe80::/10 link-local
if (/^f[cd]/.test(s)) return true; // fc00::/7 ULA
return false;
}
function isBlockedIp(ip) {
if (net.isIPv4(ip)) return isBlockedIPv4(ip);
if (net.isIPv6(ip)) return isBlockedIPv6(ip);
return true; // not a recognizable IP -> treat as unsafe
}
async function hostIsSafe(hostname) {
const h = hostname.toLowerCase().replace(/^\[|\]$/g, ''); // strip IPv6 brackets
if (h === 'localhost' || h.endsWith('.localhost')) return false;
if (net.isIP(h)) return !isBlockedIp(h);
let addrs;
try {
addrs = await dns.lookup(h, { all: true });
} catch (e) {
return false; // unresolvable -> unsafe (also fails the 2xx check)
}
if (!addrs.length) return false;
return addrs.every((a) => !isBlockedIp(a.address));
}

// --- CSV parsing (same shape as validate-systems-csv.js) -------------------
function parseCSV(text) {
const records = [];
let field = '';
let fields = [];
let inQuotes = false;
let i = 0;
let hasContent = false;

const pushField = () => { fields.push(field); field = ''; };
const pushRecord = () => { records.push(fields); fields = []; hasContent = false; };

while (i < text.length) {
const c = text[i];
if (inQuotes) {
if (c === '"') {
if (text[i + 1] === '"') { field += '"'; i += 2; continue; }
inQuotes = false; i++; continue;
}
field += c; i++; continue;
}
if (c === '"') { inQuotes = true; hasContent = true; i++; continue; }
if (c === ',') { pushField(); i++; continue; }
if (c === '\r') { i++; continue; }
if (c === '\n') { pushField(); pushRecord(); i++; continue; }
field += c; hasContent = true; i++;
}
if (hasContent || field.length > 0 || fields.length > 0) { pushField(); pushRecord(); }
return records;
}

// Return a Map of URL value (from URL_COLUMNS) -> authRequired, found in a CSV string.
// authRequired is true when the URL is a feed URL ("URL" column) on a row whose "Access Information" is filled in, meaning the feed needs a login.
function urlsFrom(text) {
const urls = new Map();
if (!text || !text.trim()) return urls;
const records = parseCSV(text);
if (records.length === 0) return urls;
const header = records[0];
const accessIdx = header.indexOf(ACCESS_COLUMN);
const cols = URL_COLUMNS
.map((name) => ({ name, i: header.indexOf(name) }))
.filter((c) => c.i >= 0);
for (let r = 1; r < records.length; r++) {
const row = records[r];
const needsLogin = accessIdx >= 0 && (row[accessIdx] || '').trim() !== '';
for (const { name, i } of cols) {
const v = (row[i] || '').trim();
if (!v) continue;
if (PLACEHOLDERS.has(v.toLowerCase())) continue;
if (URL_IF_LINK_COLUMNS.has(name) && !/^https?:\/\//i.test(v)) continue;
const authRequired = name === FEED_COLUMN && needsLogin;
urls.set(v, urls.get(v) || authRequired);
}
}
return urls;
}

// --- HTTP check ------------------------------------------------------------
// Resolve one URL, following redirects, resolving to { url, status, ok, error }.
// Async because the SSRF host check resolves DNS before any connection is made.
async function checkOnce(rawUrl, redirectsLeft) {
let target;
try {
target = new URL(rawUrl);
} catch (e) {
return { url: rawUrl, status: null, ok: false, error: `invalid URL: ${e.message}` };
}
if (target.protocol !== 'http:' && target.protocol !== 'https:') {
return { url: rawUrl, status: null, ok: false, error: `unsupported protocol: ${target.protocol}` };
}
if (!(await hostIsSafe(target.hostname))) {
return {
url: rawUrl,
status: null,
ok: false,
error: `blocked host (local/private address not allowed): ${target.hostname}`,
};
}

return new Promise((resolve) => {
const lib = target.protocol === 'https:' ? https : http;
const req = lib.request(
target,
{
method: 'GET',
headers: { 'User-Agent': USER_AGENT, Accept: '*/*' },
timeout: TIMEOUT_MS,
},
(res) => {
const status = res.statusCode;
// Follow redirects to a final status.
if (status >= 300 && status < 400 && res.headers.location && redirectsLeft > 0) {
res.resume(); // discard body
let next;
try {
next = new URL(res.headers.location, target).toString();
} catch (e) {
resolve({ url: rawUrl, status, ok: false, error: `bad redirect target: ${res.headers.location}` });
return;
}
resolve(checkOnce(next, redirectsLeft - 1));
return;
}
res.resume(); // drain so the socket can be freed
resolve({ url: rawUrl, status, ok: status >= 200 && status <= 299, error: null });
}
);

req.on('timeout', () => { req.destroy(new Error(`timeout after ${TIMEOUT_MS}ms`)); });
req.on('error', (e) => {
resolve({ url: rawUrl, status: null, ok: false, error: e.message });
});
req.end();
});
}

// Check a URL with one retry on transient failure (network error / timeout).
async function checkUrl(rawUrl) {
let result = await checkOnce(rawUrl, MAX_REDIRECTS);
if (!result.ok && result.status === null) {
result = await checkOnce(rawUrl, MAX_REDIRECTS);
}
return result;
}

async function runPool(items, worker, size) {
const results = new Array(items.length);
let next = 0;
async function drain() {
while (next < items.length) {
const cur = next++;
results[cur] = await worker(items[cur]);
}
}
await Promise.all(Array.from({ length: Math.min(size, items.length) }, drain));
return results;
}

async function main() {
const headText = fs.readFileSync(headFile, 'utf8');
let baseText = '';
if (baseFile && fs.existsSync(baseFile)) {
baseText = fs.readFileSync(baseFile, 'utf8');
}

const baseUrls = urlsFrom(baseText);
const headUrls = urlsFrom(headText);
const newUrls = [...headUrls.keys()].filter((u) => !baseUrls.has(u)).sort();

if (newUrls.length === 0) {
console.log('No newly added URLs to check.');
process.exit(0);
}

if (newUrls.length > MAX_NEW_URLS) {
console.log(
`::error::This PR adds ${newUrls.length} new URLs, exceeding the limit of ${MAX_NEW_URLS}. ` +
`Please split it into smaller pull requests.`
);
process.exit(1);
}

console.log(`Checking ${newUrls.length} newly added URL(s) for HTTP 200-299 (401/403 allowed for feeds that need a login)...\n`);
const results = await runPool(newUrls, checkUrl, CONCURRENCY);

// A feed that needs a login passes on 401/403: the server answered, it only refused the missing credentials.
for (const r of results) {
if (!r.ok && headUrls.get(r.url) && AUTH_STATUSES.has(r.status)) {
r.ok = true;
r.authOnly = true;
}
}

const failed = [];
for (const r of results) {
if (r.ok) {
console.log(` ok ${r.status} ${r.url}${r.authOnly ? ' (login required)' : ''}`);
} else {
failed.push(r);
const detail = r.status !== null ? `HTTP ${r.status}` : r.error;
console.log(` FAIL ${detail} ${r.url}`);
}
}

if (failed.length > 0) {
console.log('');
for (const r of failed) {
const detail = r.status !== null ? `returned HTTP ${r.status}` : `error: ${r.error}`;
console.log(`::error::New URL ${detail}: ${r.url}`);
}
console.log(`\n${failed.length} of ${newUrls.length} new URL(s) failed the check.`);
process.exit(1);
}

console.log(`\nAll ${newUrls.length} new URL(s) passed.`);
process.exit(0);
}

main().catch((e) => {
console.error(`::error::check-new-urls.js failed: ${e.stack || e.message}`);
process.exit(2);
});
Loading
Loading