test(features): run feature tests against environment services, starting with Keycloak

.github/workflows/e2e-suites.yml

@@ -46,6 +46,11 @@ jobs:
             family: conformance
             package: sandbox
             runner: ubuntu-24.04
+          - environment: ubuntu-docker-rootful-keycloak
+            installer: deb
+            family: features
+            package: keycloak
+            runner: ubuntu-24.04
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 90
     steps:
@@ -76,7 +81,7 @@ jobs:
           # would rebuild the changed layers on every run. Key on the playbooks
           # with no restore-keys fallback: restoring an older entry would carry
           # its stale disks into the new one, which then only grows.
-          key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/images.nix') }}
+          key: tmachine-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('tests/tmachine/ansible/**', 'tests/tmachine/images.nix', 'scripts/keycloak-realm.json') }}
 
       - name: Download integration inputs
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1

TESTING.md

@@ -190,6 +190,55 @@ an explicit matrix and runs the runner once per row. Branch E2E calls it when
 the `test:e2e` label is set. It is advisory, so a failure doesn't block
 merging. To run a suite in another environment on CI, add a row.
 
+### Adding a feature suite
+
+A feature suite tests one OpenShell feature and lives in the
+`tests/suites/features` workspace. `tests/suites/features/keycloak` is the
+example to copy.
+
+1. Add a crate under `tests/suites/features/<name>` that depends on
+   `openshell-testkit`, and add it to `members` in
+   `tests/suites/features/Cargo.toml`.
+2. Write the tests in `tests/*.rs` as `#[testkit::test]` functions.
+3. Re-lock the workspace: `cargo metadata --manifest-path
+   tests/suites/features/Cargo.toml`, then
+   `uv run tasks/scripts/check_cargo_lockfile_alignment.py --fix`.
+4. Add a row for the suite to the matrix in `.github/workflows/e2e-suites.yml`.
+
+If the feature only needs the gateway, run the suite in an existing
+environment such as `ubuntu-docker-rootful`, and skip the rest of this
+section.
+
+**Suites that need a service.** A feature that needs something besides the
+gateway, such as an OIDC issuer, runs in a tmachine environment that provides
+it. The environment and the suite share a contract: a set of
+`OPENSHELL_TEST_<SERVICE>_*` environment variables that say how to reach the
+service.
+
+- **Environment.** Add an environment to `tests/tmachine/config.nix` whose
+  setup playbook starts the service (with a restart policy, since setup is
+  cached as a disk) and writes the variables as `KEY=VALUE` lines to
+  `/etc/openshell-test/env.d/<service>.env`. Pass files the playbook needs
+  through the environment's setup `inputs`, and add them to the tmachine disk
+  cache key in `e2e-suites.yml`. The runner passes the variables to the tests
+  and rejects names without the `OPENSHELL_TEST_` prefix.
+- **Fixture.** In the suite's `src/lib.rs`, define the variable names as
+  constants and a `<Service>Context::new(cx)` that reads them, waits for the
+  service, and wraps the testkit `Context`. Each test calls it on its first
+  line. Nextest runs each test in its own process, so the fixture builds its
+  clients there from the variables. A missing variable fails the test; it
+  never skips it.
+- **Documentation.** Document the variables in the fixture's module docs, so
+  anyone can run the suite against their own deployment by setting them.
+
+Run the Keycloak suite with:
+
+```shell
+nix run .#openshell-test-runner -- \
+  --environment ubuntu-docker-rootful-keycloak --installer deb \
+  --family features --package keycloak
+```
+
 ### Python E2E (`e2e/python/`)
 
 `mise run e2e:python` builds `openshell/e2e-python:dev` from

tasks/scripts/check_cargo_lockfile_alignment.py

@@ -37,6 +37,7 @@
 ALIGNED_WORKSPACES = (
     "tests/testkit",
     "tests/suites/conformance",
+    "tests/suites/features",
 )
 
 

tests/runner/src/execute.rs

@@ -7,7 +7,9 @@
 //!
 //! Tests find the gateway the way the CLI does, so the runner points them at
 //! the one the installer registered with `OPENSHELL_GATEWAY` and at the
-//! installed CLI with `OPENSHELL_BIN`.
+//! installed CLI with `OPENSHELL_BIN`. Anything else the environment provides,
+//! such as Keycloak, the tests learn from the variables it publishes (see
+//! [`crate::test_env`]).
 
 use std::fmt;
 use std::path::Path;
@@ -19,6 +21,7 @@ use tmachine::Session;
 use tmachine::config::Config;
 
 use crate::bundle::{Bundle, PROFILE, READY_BINARY};
+use crate::test_env;
 
 const GUEST_DIR: &str = "/var/tmp/openshell-tests";
 /// How long the gateway has to become healthy after the environment is up.
@@ -201,10 +204,15 @@ async fn run_tests(session: &Session, run: &Run<'_>) -> Result<Status> {
         !openshell_bin.is_empty(),
         "the installer did not put openshell on the guest's PATH"
     );
-    let env = [
+    let mut env = vec![
         format!("OPENSHELL_GATEWAY={}", run.gateway),
         format!("OPENSHELL_BIN={openshell_bin}"),
     ];
+    env.extend(
+        test_env::read(session)
+            .await
+            .context("read the variables the environment publishes for tests")?,
+    );
 
     let guest = GUEST_DIR;
     checked(

tests/runner/src/main.rs

@@ -17,6 +17,7 @@ use crate::execute::Run;
 
 mod bundle;
 mod execute;
+mod test_env;
 
 /// Build a test family for the guest, boot a tmachine environment with
 /// OpenShell installed, run the tests in it, and check the results.

tests/runner/src/test_env.rs

@@ -0,0 +1,123 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Variables an environment publishes for its tests.
+//!
+//! An environment that runs something besides `OpenShell`, such as Keycloak,
+//! says how tests reach it in `/etc/openshell-test/env.d/*.env`: one
+//! `OPENSHELL_TEST_*=value` per line. The tmachine environment's setup
+//! writes the file; the runner passes its variables to the tests, and suite
+//! fixtures read them. Tests don't depend on tmachine: anyone running a suite
+//! against their own deployment sets the same variables.
+
+use anyhow::{Context, Result, bail, ensure};
+use tmachine::Session;
+
+/// Where environments publish the variables tests read.
+pub const ENV_DIR: &str = "/etc/openshell-test/env.d";
+/// The prefix every published variable must have, so an environment can't
+/// override how tests find the gateway or the CLI.
+pub const ENV_PREFIX: &str = "OPENSHELL_TEST_";
+
+/// Read the variables the booted environment publishes, as `KEY=VALUE`
+/// strings. An environment that publishes none has no `ENV_DIR`.
+pub async fn read(session: &Session) -> Result<Vec<String>> {
+    let script = format!(
+        "for f in {ENV_DIR}/*.env; do [ -e \"$f\" ] || continue; printf '\\0%s\\n' \"$f\"; cat \"$f\"; done"
+    );
+    let output = session.exec(&["sh", "-c", &script]).await?;
+    ensure!(
+        output.status.success(),
+        "read {ENV_DIR}: {}",
+        String::from_utf8_lossy(&output.stderr).trim()
+    );
+    let contents =
+        String::from_utf8(output.stdout).with_context(|| format!("{ENV_DIR} is not UTF-8"))?;
+    parse_files(&contents)
+}
+
+/// Parse the files `read` prints: each is a NUL, its path and a newline,
+/// then its contents.
+fn parse_files(output: &str) -> Result<Vec<String>> {
+    let mut vars = Vec::new();
+    for file in output.split('\0').filter(|file| !file.is_empty()) {
+        let (path, body) = file.split_once('\n').unwrap_or((file, ""));
+        vars.extend(parse_env(path, body)?);
+    }
+    Ok(vars)
+}
+
+/// Parse a published env file: `KEY=VALUE` lines, blank lines and `#`
+/// comments. Values are taken literally; there is no quoting. Errors name
+/// the key, never the value, since values can be credentials.
+pub fn parse_env(path: &str, contents: &str) -> Result<Vec<String>> {
+    let mut vars = Vec::new();
+    for (number, line) in contents.lines().enumerate() {
+        let line = line.trim_end();
+        if line.trim().is_empty() || line.trim_start().starts_with('#') {
+            continue;
+        }
+        let Some((key, _value)) = line.split_once('=') else {
+            bail!("{path}:{}: expected KEY=VALUE", number + 1);
+        };
+        ensure!(
+            key.starts_with(ENV_PREFIX)
+                && key.len() > ENV_PREFIX.len()
+                && key
+                    .bytes()
+                    .all(|b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_'),
+            "{path}:{}: {key:?} is not an {ENV_PREFIX}* variable name",
+            number + 1
+        );
+        vars.push(line.to_string());
+    }
+    Ok(vars)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn parses_published_variables() {
+        let vars = parse_env(
+            "keycloak.env",
+            "# Published by keycloak.yaml\n\nOPENSHELL_TEST_KEYCLOAK_ISSUER=http://127.0.0.1:8180/realms/openshell\nOPENSHELL_TEST_KEYCLOAK_PASSWORD=a=b\n",
+        )
+        .unwrap();
+        assert_eq!(
+            vars,
+            [
+                "OPENSHELL_TEST_KEYCLOAK_ISSUER=http://127.0.0.1:8180/realms/openshell",
+                "OPENSHELL_TEST_KEYCLOAK_PASSWORD=a=b",
+            ]
+        );
+    }
+
+    #[test]
+    fn reads_every_published_file() {
+        assert_eq!(parse_files("").unwrap(), Vec::<String>::new());
+        let vars = parse_files(
+            "\0/etc/openshell-test/env.d/a.env\nOPENSHELL_TEST_A=1\n\0/etc/openshell-test/env.d/b.env\nOPENSHELL_TEST_B=2",
+        )
+        .unwrap();
+        assert_eq!(vars, ["OPENSHELL_TEST_A=1", "OPENSHELL_TEST_B=2"]);
+
+        let err = parse_files("\0/etc/openshell-test/env.d/b.env\nPATH=/x\n").unwrap_err();
+        assert!(err.to_string().contains("b.env:1"), "{err}");
+    }
+
+    #[test]
+    fn refuses_variables_outside_the_prefix() {
+        let err = parse_env("x.env", "OPENSHELL_GATEWAY=other\n").unwrap_err();
+        assert!(err.to_string().contains("OPENSHELL_GATEWAY"), "{err}");
+    }
+
+    #[test]
+    fn errors_do_not_echo_values() {
+        let err = parse_env("x.env", "openshell_test_secret=hunter2\n").unwrap_err();
+        assert!(!err.to_string().contains("hunter2"), "{err}");
+        let err = parse_env("x.env", "hunter2\n").unwrap_err();
+        assert!(!err.to_string().contains("hunter2"), "{err}");
+    }
+}