feat: Canonical-only sitemap, contributor docs optional in llms.txt

antora-playbook.yml

@@ -3,7 +3,10 @@ site:
   title: Stackable Documentation
   url: https://docs.stackable.tech
   start_page: home::index.adoc
-  robots: allow
+  robots: |
+    User-agent: *
+    Allow: /
+    Sitemap: https://docs.stackable.tech/sitemap.xml
   keys:
     enable_tracking: true # this key is used in the documentation-ui
 # URL config settings.
@@ -30,6 +33,7 @@ antora:
     - require: '@sntke/antora-mermaid-extension'
     - ./lib/stackable-operator-helpers.js
     - ./lib/llms-txt.js
+    - ./lib/sitemap-canonical.js
 content:
   sources:
     - url: .

lib/llms-txt.js

@@ -10,6 +10,11 @@
 // Useful links:
 // Extensions: https://docs.antora.org/antora/latest/extend/extensions/
 // Types of events: https://docs.antora.org/antora/latest/extend/generator-events-reference/
+
+// Pages on how Stackable builds SDP go under "Optional", the llms.txt convention
+// for links an agent can skip when context is short (https://llmstxt.org/).
+const OPTIONAL_MODULES = new Set(['contributor', 'internal'])
+
 module.exports.register = function () {
   this.once('beforePublish', ({ playbook, contentCatalog, siteCatalog }) => {
     const siteUrl = (playbook.site.url || '').replace(/\/$/, '')
@@ -26,6 +31,7 @@ module.exports.register = function () {
       'Every entry below links the published page and, in parentheses, its AsciiDoc',
       'source on GitHub. To suggest a change to a page, edit that source file.',
     ]
+    const optional = []
 
     for (const component of contentCatalog.getComponents()) {
       const version = component.latest
@@ -36,31 +42,42 @@ module.exports.register = function () {
       const byModule = new Map()
       for (const page of pages) {
         const module = page.src.module
+        if (OPTIONAL_MODULES.has(module)) {
+          optional.push(page)
+          continue
+        }
         if (!byModule.has(module)) byModule.set(module, [])
         byModule.get(module).push(page)
       }
       for (const [module, modulePages] of byModule) {
         lines.push('', `## ${component.title}: ${module === 'ROOT' ? 'general' : module}`, '')
-        for (const page of modulePages) {
-          const title = (page.asciidoc && page.asciidoc.doctitle ? page.asciidoc.doctitle : page.src.stem)
-            .replace(/<[^>]+>/g, '')
-          const description = page.asciidoc && page.asciidoc.attributes['description']
-          const source = sourceUrl(page.src)
-          let entry = `- [${title}](${siteUrl}${page.pub.url})`
-          if (description) entry += `: ${description.replace(/\s+/g, ' ').trim()}`
-          if (source) entry += ` (source: ${source})`
-          lines.push(entry)
-        }
+        for (const page of modulePages) lines.push(entryFor(page, siteUrl))
       }
     }
 
+    if (optional.length) {
+      lines.push('', '## Optional', '')
+      for (const page of optional) lines.push(entryFor(page, siteUrl))
+    }
+
     siteCatalog.addFile({
       contents: Buffer.from(lines.join('\n') + '\n', 'utf8'),
       out: { path: 'llms.txt' },
     })
   })
 }
 
+function entryFor (page, siteUrl) {
+  const title = (page.asciidoc && page.asciidoc.doctitle ? page.asciidoc.doctitle : page.src.stem)
+    .replace(/<[^>]+>/g, '')
+  const description = page.asciidoc && page.asciidoc.attributes['description']
+  const source = sourceUrl(page.src)
+  let entry = `- [${title}](${siteUrl}${page.pub.url})`
+  if (description) entry += `: ${description.replace(/\s+/g, ' ').trim()}`
+  if (source) entry += ` (source: ${source})`
+  return entry
+}
+
 function sourceUrl (src) {
   const origin = src.origin
   if (!origin || !origin.url || !origin.url.startsWith('https://github.com/')) return undefined

lib/sitemap-canonical.js

@@ -0,0 +1,27 @@
+// Lists only pages that are their own canonical: all of stable, plus older pages
+// that no release since still has. Every sitemap URL is a suggested canonical,
+// and Google asks that a sitemap not contradict rel=canonical:
+// https://developers.google.com/search/docs/crawling-indexing/consolidate-duplicate-urls
+// Prereleases stay out.
+//
+// Reads pub.canonicalUrl, which Antora only sets while composing pages, so this
+// must run at mapSite or later.
+module.exports.register = function () {
+  this.once('contextStarted', () => {
+    const { mapSite } = this.getFunctions()
+    this.replaceFunctions({
+      mapSite (playbook, pages) {
+        const siteUrl = (playbook.site.url || '').replace(/\/$/, '')
+        const { contentCatalog } = this.getVariables()
+        return mapSite(playbook, pages.filter((page) => isOwnCanonical(page, siteUrl, contentCatalog)))
+      },
+    })
+  })
+}
+
+function isOwnCanonical (page, siteUrl, contentCatalog) {
+  const version = contentCatalog.getComponentVersion(page.src.component, page.src.version)
+  if (version && version.prerelease) return false
+  const canonical = page.pub.canonicalUrl
+  return canonical === undefined || canonical === siteUrl + page.pub.url
+}

local-antora-playbook.yml

@@ -15,6 +15,7 @@ antora:
     - require: '@sntke/antora-mermaid-extension'
     - ./lib/stackable-operator-helpers.js
     - ./lib/llms-txt.js
+    - ./lib/sitemap-canonical.js
 content:
   sources:
     - url: ./

only-dev-antora-playbook.yml

@@ -15,6 +15,7 @@ antora:
     - require: '@sntke/antora-mermaid-extension'
     - ./lib/stackable-operator-helpers.js
     - ./lib/llms-txt.js
+    - ./lib/sitemap-canonical.js
 content:
   sources:
     - url: ./

truly-local-playbook.yml

@@ -15,6 +15,7 @@ antora:
     - require: '@sntke/antora-mermaid-extension'
     - ./lib/stackable-operator-helpers.js
     - ./lib/llms-txt.js
+    - ./lib/sitemap-canonical.js
 content:
   sources:
     - url: ./