From 7cf0842507de2fb4c84a0e7678eccb5590456197 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 22 Oct 2026 13:40:45 +0200 Subject: [PATCH] feat: Canonical-only sitemap, contributor docs optional in llms.txt --- antora-playbook.yml | 6 +++++- lib/llms-txt.js | 37 ++++++++++++++++++++++++++---------- lib/sitemap-canonical.js | 27 ++++++++++++++++++++++++++ local-antora-playbook.yml | 1 + only-dev-antora-playbook.yml | 1 + truly-local-playbook.yml | 1 + 6 files changed, 62 insertions(+), 11 deletions(-) create mode 100644 lib/sitemap-canonical.js diff --git a/antora-playbook.yml b/antora-playbook.yml index 8fc6cefdc..83a46d30c 100644 --- a/antora-playbook.yml +++ b/antora-playbook.yml @@ -3,7 +3,10 @@ site: title: Stackable Documentation url: https://docs.stackable.tech start_page: home::index.adoc - robots: allow + robots: | + User-agent: * + Allow: / + Sitemap: https://docs.stackable.tech/sitemap.xml keys: enable_tracking: true # this key is used in the documentation-ui # URL config settings. @@ -30,6 +33,7 @@ antora: - require: '@sntke/antora-mermaid-extension' - ./lib/stackable-operator-helpers.js - ./lib/llms-txt.js + - ./lib/sitemap-canonical.js content: sources: - url: . diff --git a/lib/llms-txt.js b/lib/llms-txt.js index 75bbd4e10..ac3f9b8c8 100644 --- a/lib/llms-txt.js +++ b/lib/llms-txt.js @@ -10,6 +10,11 @@ // Useful links: // Extensions: https://docs.antora.org/antora/latest/extend/extensions/ // Types of events: https://docs.antora.org/antora/latest/extend/generator-events-reference/ + +// Pages on how Stackable builds SDP go under "Optional", the llms.txt convention +// for links an agent can skip when context is short (https://llmstxt.org/). +const OPTIONAL_MODULES = new Set(['contributor', 'internal']) + module.exports.register = function () { this.once('beforePublish', ({ playbook, contentCatalog, siteCatalog }) => { const siteUrl = (playbook.site.url || '').replace(/\/$/, '') @@ -26,6 +31,7 @@ module.exports.register = function () { 'Every entry below links the published page and, in parentheses, its AsciiDoc', 'source on GitHub. To suggest a change to a page, edit that source file.', ] + const optional = [] for (const component of contentCatalog.getComponents()) { const version = component.latest @@ -36,24 +42,24 @@ module.exports.register = function () { const byModule = new Map() for (const page of pages) { const module = page.src.module + if (OPTIONAL_MODULES.has(module)) { + optional.push(page) + continue + } if (!byModule.has(module)) byModule.set(module, []) byModule.get(module).push(page) } for (const [module, modulePages] of byModule) { lines.push('', `## ${component.title}: ${module === 'ROOT' ? 'general' : module}`, '') - for (const page of modulePages) { - const title = (page.asciidoc && page.asciidoc.doctitle ? page.asciidoc.doctitle : page.src.stem) - .replace(/<[^>]+>/g, '') - const description = page.asciidoc && page.asciidoc.attributes['description'] - const source = sourceUrl(page.src) - let entry = `- [${title}](${siteUrl}${page.pub.url})` - if (description) entry += `: ${description.replace(/\s+/g, ' ').trim()}` - if (source) entry += ` (source: ${source})` - lines.push(entry) - } + for (const page of modulePages) lines.push(entryFor(page, siteUrl)) } } + if (optional.length) { + lines.push('', '## Optional', '') + for (const page of optional) lines.push(entryFor(page, siteUrl)) + } + siteCatalog.addFile({ contents: Buffer.from(lines.join('\n') + '\n', 'utf8'), out: { path: 'llms.txt' }, @@ -61,6 +67,17 @@ module.exports.register = function () { }) } +function entryFor (page, siteUrl) { + const title = (page.asciidoc && page.asciidoc.doctitle ? page.asciidoc.doctitle : page.src.stem) + .replace(/<[^>]+>/g, '') + const description = page.asciidoc && page.asciidoc.attributes['description'] + const source = sourceUrl(page.src) + let entry = `- [${title}](${siteUrl}${page.pub.url})` + if (description) entry += `: ${description.replace(/\s+/g, ' ').trim()}` + if (source) entry += ` (source: ${source})` + return entry +} + function sourceUrl (src) { const origin = src.origin if (!origin || !origin.url || !origin.url.startsWith('https://github.com/')) return undefined diff --git a/lib/sitemap-canonical.js b/lib/sitemap-canonical.js new file mode 100644 index 000000000..6e7938a92 --- /dev/null +++ b/lib/sitemap-canonical.js @@ -0,0 +1,27 @@ +// Lists only pages that are their own canonical: all of stable, plus older pages +// that no release since still has. Every sitemap URL is a suggested canonical, +// and Google asks that a sitemap not contradict rel=canonical: +// https://developers.google.com/search/docs/crawling-indexing/consolidate-duplicate-urls +// Prereleases stay out. +// +// Reads pub.canonicalUrl, which Antora only sets while composing pages, so this +// must run at mapSite or later. +module.exports.register = function () { + this.once('contextStarted', () => { + const { mapSite } = this.getFunctions() + this.replaceFunctions({ + mapSite (playbook, pages) { + const siteUrl = (playbook.site.url || '').replace(/\/$/, '') + const { contentCatalog } = this.getVariables() + return mapSite(playbook, pages.filter((page) => isOwnCanonical(page, siteUrl, contentCatalog))) + }, + }) + }) +} + +function isOwnCanonical (page, siteUrl, contentCatalog) { + const version = contentCatalog.getComponentVersion(page.src.component, page.src.version) + if (version && version.prerelease) return false + const canonical = page.pub.canonicalUrl + return canonical === undefined || canonical === siteUrl + page.pub.url +} diff --git a/local-antora-playbook.yml b/local-antora-playbook.yml index 1469b3fd9..211806ca7 100644 --- a/local-antora-playbook.yml +++ b/local-antora-playbook.yml @@ -15,6 +15,7 @@ antora: - require: '@sntke/antora-mermaid-extension' - ./lib/stackable-operator-helpers.js - ./lib/llms-txt.js + - ./lib/sitemap-canonical.js content: sources: - url: ./ diff --git a/only-dev-antora-playbook.yml b/only-dev-antora-playbook.yml index 14f5e6813..135269a87 100644 --- a/only-dev-antora-playbook.yml +++ b/only-dev-antora-playbook.yml @@ -15,6 +15,7 @@ antora: - require: '@sntke/antora-mermaid-extension' - ./lib/stackable-operator-helpers.js - ./lib/llms-txt.js + - ./lib/sitemap-canonical.js content: sources: - url: ./ diff --git a/truly-local-playbook.yml b/truly-local-playbook.yml index 220fd3512..6e0081122 100644 --- a/truly-local-playbook.yml +++ b/truly-local-playbook.yml @@ -15,6 +15,7 @@ antora: - require: '@sntke/antora-mermaid-extension' - ./lib/stackable-operator-helpers.js - ./lib/llms-txt.js + - ./lib/sitemap-canonical.js content: sources: - url: ./