diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f3d5cb..ee2b544 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,7 +18,7 @@ jobs: uses: actions/checkout@v4 with: repository: inline-chat/inline - ref: 4d39dab03a39fc661a8a00d366e4debb8c0e969c + ref: 9ef08fff242d92db9a9a92c78620ff749ef7117b path: inline-public - name: Install system dependencies diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 225aa1f..c72eff3 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,7 +41,7 @@ jobs: uses: actions/checkout@v4 with: repository: inline-chat/inline - ref: 4d39dab03a39fc661a8a00d366e4debb8c0e969c + ref: 9ef08fff242d92db9a9a92c78620ff749ef7117b path: inline-public - name: Set build time diff --git a/Cargo.lock b/Cargo.lock index 83c9b68..c9dbe92 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,23 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures 0.2.17", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -73,6 +90,12 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "axum" version = "0.8.9" @@ -201,6 +224,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "colorchoice" version = "1.0.5" @@ -241,6 +274,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32fast" +version = "1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04" +dependencies = [ + "cfg-if", +] + [[package]] name = "crypto-common" version = "0.1.7" @@ -393,6 +435,17 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "flate2" +version = "1.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + [[package]] name = "foldhash" version = "0.2.0" @@ -550,6 +603,33 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hegeltest" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b95008b427c3bd008deeafe667169eae3f69543035f2ce57e9b9bba45876c0" +dependencies = [ + "hegeltest-macros", + "parking_lot", +] + +[[package]] +name = "hegeltest-macros" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ccc83b3daf472bc2d84b675ac403fdfcc76df02044a2ab9facc1294baeba017" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "hex-literal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" + [[package]] name = "hostname" version = "0.4.2" @@ -640,7 +720,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots", + "webpki-roots 1.0.8", ] [[package]] @@ -784,6 +864,7 @@ name = "inline-client" version = "0.6.3" dependencies = [ "futures-util", + "getrandom 0.3.4", "inline-sdk", "log", "prost", @@ -796,18 +877,28 @@ dependencies = [ [[package]] name = "inline-protocol" -version = "0.6.2" +version = "0.7.22-alpha.3" dependencies = [ + "aes", + "cipher", + "hex-literal", + "num-bigint", "prost", "prost-build", "serde", + "sha1", + "sha2", + "subtle", ] [[package]] name = "inline-sdk" -version = "0.6.2" +version = "0.7.22-alpha.3" dependencies = [ + "base64", + "flate2", "futures-util", + "getrandom 0.3.4", "hostname", "inline-protocol", "log", @@ -815,10 +906,21 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "thiserror 1.0.69", "tokio", "tokio-tungstenite", "url", + "uuid", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", ] [[package]] @@ -896,6 +998,7 @@ version = "0.38.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6c19a05435c21ac299d71b6a9c13db3e3f47c520517d58990a462a1397a61db" dependencies = [ + "cc", "pkg-config", "vcpkg", ] @@ -912,6 +1015,15 @@ version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + [[package]] name = "log" version = "0.4.33" @@ -936,6 +1048,7 @@ version = "0.2.2" dependencies = [ "axum", "env_logger", + "hegeltest", "inline-client", "log", "rusqlite", @@ -968,6 +1081,16 @@ dependencies = [ "unicase", ] +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.1" @@ -1002,6 +1125,34 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -1020,6 +1171,29 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -1285,6 +1459,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.13.0", +] + [[package]] name = "regex" version = "1.12.4" @@ -1353,7 +1536,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "webpki-roots", + "webpki-roots 1.0.8", ] [[package]] @@ -1482,6 +1665,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "security-framework" version = "3.7.0" @@ -1582,12 +1771,45 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1_smol" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbfa15b3dddfee50a0fff136974b3e1bde555604ba463834a7eb7deb6417705d" + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "shlex" version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + [[package]] name = "slab" version = "0.4.12" @@ -1678,7 +1900,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.3", "once_cell", "rustix", "windows-sys 0.61.2", @@ -1759,6 +1981,7 @@ dependencies = [ "libc", "mio", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.61.2", @@ -1799,6 +2022,7 @@ dependencies = [ "tokio", "tokio-rustls", "tungstenite", + "webpki-roots 0.26.11", ] [[package]] @@ -1952,6 +2176,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.29.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a06f08a856b009bc7bbaf7d3e79808595d032dd19074c0c2ef26e49e03e6bdc" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "sha1_smol", + "wasm-bindgen", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -2076,6 +2312,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.8", +] + [[package]] name = "webpki-roots" version = "1.0.8" @@ -2288,6 +2533,12 @@ dependencies = [ "syn", ] +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + [[package]] name = "zmij" version = "1.0.21" diff --git a/Dockerfile b/Dockerfile index 5604fe3..888df79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,7 @@ ARG VERSION=dev ARG COMMIT=unknown ARG BUILD_TIME=unknown -FROM ${DOCKER_HUB}/golang:1.25-alpine AS go-builder +FROM ${DOCKER_HUB}/golang:1.26-alpine AS go-builder ARG VERSION ARG COMMIT ARG BUILD_TIME @@ -81,6 +81,6 @@ VOLUME ["/data"] EXPOSE 29343 HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ - CMD curl -fsS http://127.0.0.1:29342/health >/dev/null || exit 1 + CMD curl -fsS -H "X-Inline-Protocol-Version: 5" http://127.0.0.1:29342/health >/dev/null || exit 1 CMD ["/docker-run.sh"] diff --git a/README.md b/README.md index a52f029..c769b73 100644 --- a/README.md +++ b/README.md @@ -12,43 +12,58 @@ chats into Matrix rooms, and send messages between Matrix/Beeper and Inline. - [x] Session resume after bridge restart - [x] Startup chat sync - [x] Recent history sync and bridgev2 backfill -- [x] Text messages +- [x] Text messages, supported formatting, entities and account-scoped mentions - [x] Normal message replies - [x] Message edits - [x] Message deletes/redactions - [x] Reactions -- [x] Read receipts and marked-unread state +- [x] Verified source read-prefix projection with double puppeting (limits below) - [x] Permanent per-room mute/unmute - [x] Typing notifications - [x] Images, videos, files, audio, and voice/audio files - [x] Inline member list sync for bridged rooms - [x] Matrix ghosts with Inline display names and avatars when available - [x] Configurable bridge bot and network profile metadata -- [x] DM creation with a numeric Inline user ID -- [x] Basic group/thread creation from Matrix -- [x] Add/remove Inline thread participants from Matrix membership changes +- [x] DM creation using accessible Inline IDs or canonical usernames +- [x] Bounded accessible-user search and explicit private/public channel creation +- [x] Caption edits that retain existing media and reject file replacements +- [x] Separate reply-child rooms with source parent context +- [x] Add Inline thread participants from Matrix invitations +- [ ] Remove Inline thread access from Matrix kicks or invite revocations - [x] Rename Inline threads from Matrix - [x] Explicit Inline thread deletion from Beeper/Matrix -- [x] Management commands for status and reconnect +- [x] Account-scoped workspace Spaces and in-place presentation migration +- [x] Management commands for status, settings, and reconnect - [ ] New Inline account signup or invite-code onboarding - [ ] Matrix-native thread UI for Inline reply-thread chats -- [ ] Contact search for DM creation -- [ ] Formatted text, rich entities, and Matrix mention mapping - [ ] Pins and Inline link-preview projection - [ ] Room avatars - [ ] Room topic/description editing -- [ ] Presence and richer workspace/space representation +- [ ] Presence - [ ] Timed mute expiration (permanent mute is supported) - [ ] Calls +Channels are ordinary Matrix rooms, while DMs retain DM identity. Workspace +navigation requires source-backed access evidence. Automatic channel mute and +manual-unread writes remain disabled until their origin can be distinguished +from user changes; the bridge reports degraded notification policy and continues +delivery. Mute channels manually for a quiet inbox. Source read state uses a +verified private receipt/fully-read prefix and never crosses an unmapped event; +genuine private user receipts are not forwarded by the pinned framework. +See [operations](docs/operations.md#notification-and-read-state-limits) for these +limits and the explicit preview/apply/rollback migration commands. +The operations guide also describes [rich messages and media](docs/operations.md#rich-messages-and-media), +[accessible-user lookup](docs/operations.md#accessible-people-and-channel-creation), +and [separate child context](docs/operations.md#separate-reply-child-context). + ## Requirements - An existing Inline account - A Matrix homeserver or Beeper bridge deployment -- Docker and Docker Compose, or Go 1.25 plus Rust 1.96 for native builds +- Docker and Docker Compose, or Go 1.26.1 plus Rust 1.96 for native builds - A published container image from `ghcr.io/inline-chat/matrix-inline` -Source builds require a local checkout of `inline-chat/inline-public` next to this repo +Source builds require a local checkout of `inline-chat/inline` named `inline-public` next to this repo because the bridge builds against the Rust Inline client crates: ```text @@ -289,6 +304,7 @@ libolm headers. - [Deployment](docs/deployment.md) - [Operations](docs/operations.md) +- [Adapter protocol and compatibility](docs/adapter-protocol.md) - [Smoke test](docs/smoke-test.md) - [Local E2E harness](docs/local-e2e.md) - [systemd units](deploy/systemd) diff --git a/crates/adapter/Cargo.toml b/crates/adapter/Cargo.toml index 59afcd3..09c1a70 100644 --- a/crates/adapter/Cargo.toml +++ b/crates/adapter/Cargo.toml @@ -21,6 +21,7 @@ thiserror.workspace = true tokio.workspace = true [dev-dependencies] +hegel = { package = "hegeltest", version = "0.52" } tower.workspace = true [[bin]] diff --git a/crates/adapter/src/event_store.rs b/crates/adapter/src/event_store.rs index 8e66fe7..f356009 100644 --- a/crates/adapter/src/event_store.rs +++ b/crates/adapter/src/event_store.rs @@ -405,7 +405,6 @@ impl AdapterEventStore { let latest = latest_sequence(&connection, &namespace)?; if expected_generation .map(str::trim) - .filter(|expected| !expected.is_empty()) .is_some_and(|expected| expected != generation) { return Err(AdapterEventStoreError::GenerationChanged { @@ -600,6 +599,43 @@ mod tests { use super::*; + #[hegel::test(test_cases = 1000)] + fn acknowledgement_requires_the_supplied_generation(tc: hegel::TestCase) { + let namespace = format!("fixture-{}", tc.draw(hegel::generators::integers::())); + let suffix = tc.draw(hegel::generators::text()); + let store = AdapterEventStore::open_in_memory(namespace.clone()).unwrap(); + let first = store + .append(ClientEvent::ChatUpserted { + chat_id: InlineId::new(7), + }) + .unwrap(); + for invalid in [ + String::new(), + " \t ".to_owned(), + format!("{}-other-{suffix}", first.generation), + ] { + match store.acknowledge_for_generation(&namespace, Some(&invalid), 1) { + Err(AdapterEventStoreError::GenerationChanged { + generation, + latest_sequence, + }) => { + assert_eq!(generation, first.generation); + assert_eq!(latest_sequence, 1); + } + other => panic!("wrong/missing generation must not acknowledge: {other:?}"), + } + assert_eq!(store.replay(&namespace, 0).unwrap(), vec![first.clone()]); + } + store + .acknowledge_for_generation(&namespace, Some(&first.generation), 0) + .unwrap(); + assert_eq!(store.replay(&namespace, 0).unwrap(), vec![first.clone()]); + store + .acknowledge_for_generation(&namespace, Some(&first.generation), 1) + .unwrap(); + assert!(store.replay(&namespace, 1).unwrap().is_empty()); + } + #[test] fn lossless_events_are_sequenced_replayed_and_acknowledged() { let store = AdapterEventStore::open_in_memory("team").unwrap(); diff --git a/crates/adapter/src/http.rs b/crates/adapter/src/http.rs index 96a3024..0710f53 100644 --- a/crates/adapter/src/http.rs +++ b/crates/adapter/src/http.rs @@ -15,10 +15,11 @@ use std::{ use axum::{ Json, Router, extract::{ - DefaultBodyLimit, FromRequestParts, Multipart, Query, State, - ws::{Message, WebSocket, WebSocketUpgrade}, + DefaultBodyLimit, FromRequestParts, Multipart, Query, Request, State, + ws::{CloseFrame, Message, WebSocket, WebSocketUpgrade}, }, http::{HeaderMap, StatusCode, request::Parts}, + middleware::{self, Next}, response::{IntoResponse, Response}, routing::{get, post}, }; @@ -44,6 +45,7 @@ use crate::protocol::{ }; const SIDECAR_UPLOAD_BODY_LIMIT_BYTES: usize = 101 * 1024 * 1024; +const PROTOCOL_VERSION_HEADER: &str = "x-inline-protocol-version"; const SESSION_NAMESPACE_HEADER: &str = "x-inline-session-namespace"; /// Async factory used by production adapters to lazily create one native @@ -345,9 +347,11 @@ struct AdapterRequestClient(InlineClient); #[derive(Clone, Debug)] struct AdapterRequestNamespace(String); -async fn request_namespace(parts: &Parts, state: &AdapterHttpState) -> Result { - let namespace = parts - .headers +async fn request_namespace( + headers: &HeaderMap, + state: &AdapterHttpState, +) -> Result { + let namespace = headers .get(SESSION_NAMESPACE_HEADER) .and_then(|value| value.to_str().ok()) .map(str::trim) @@ -373,7 +377,7 @@ impl FromRequestParts for AdapterRequestClient { parts: &mut Parts, state: &AdapterHttpState, ) -> Result { - let namespace = request_namespace(parts, state).await?; + let namespace = request_namespace(&parts.headers, state).await?; state .client_for_namespace(&namespace) .await @@ -389,7 +393,7 @@ impl FromRequestParts for AdapterRequestNamespace { parts: &mut Parts, state: &AdapterHttpState, ) -> Result { - let namespace = request_namespace(parts, state).await?; + let namespace = request_namespace(&parts.headers, state).await?; state .client_for_namespace(&namespace) .await @@ -398,6 +402,31 @@ impl FromRequestParts for AdapterRequestNamespace { } } +// This gate runs before extractors: even health can lazily construct a client. +// Reject duplicate headers instead of accepting an ambiguous negotiated version. +fn compatible_protocol(headers: &HeaderMap) -> bool { + let mut versions = headers.get_all(PROTOCOL_VERSION_HEADER).iter(); + matches!((versions.next(), versions.next()), (Some(value), None) + if value.as_bytes() == PROTOCOL_VERSION.to_string().as_bytes()) +} + +async fn require_protocol(request: Request, next: Next) -> Response { + if !compatible_protocol(request.headers()) { + return ( + StatusCode::CONFLICT, + Json(serde_json::json!({ + "error": "ProtocolMismatch", "expected_protocol_version": PROTOCOL_VERSION + })), + ) + .into_response(); + } + next.run(request).await +} + +async fn compatibility() -> Json { + Json(ProtocolInfo::current()) +} + /// Builds the localhost sidecar router consumed by the Go bridge. pub fn sidecar_router(client: InlineClient) -> Router { sidecar_router_with_state(AdapterHttpState::new(client)) @@ -449,6 +478,8 @@ pub fn sidecar_router_with_state(state: AdapterHttpState) -> Router { .route("/rpc/typing", post(rpc_typing)) .route("/rpc/events/ack", post(rpc_events_ack)) .route("/rpc/upload", post(rpc_upload)) + .route_layer(middleware::from_fn(require_protocol)) + .route("/compatibility", get(compatibility)) .layer(DefaultBodyLimit::max(SIDECAR_UPLOAD_BODY_LIMIT_BYTES)) .with_state(state) } @@ -541,10 +572,23 @@ async fn status( } async fn rpc( - AdapterRequestClient(client): AdapterRequestClient, State(state): State, + headers: HeaderMap, Json(request): Json, ) -> Response { + // The generic envelope has its own version. Validate it before resolving + // the namespace, just as the router validates headers before extraction. + if request.protocol_version != PROTOCOL_VERSION { + return protocol_mismatch_response(request.id, request.protocol_version); + } + let namespace = match request_namespace(&headers, &state).await { + Ok(namespace) => namespace, + Err(response) => return response, + }; + let client = match state.client_for_namespace(&namespace).await { + Ok(client) => client, + Err(error) => return (StatusCode::BAD_REQUEST, error).into_response(), + }; dispatch_request(state, client, request).await } @@ -871,6 +915,17 @@ async fn dispatch_command( dispatch_request(state, client, request).await } +fn protocol_mismatch_response(request_id: SidecarRequestId, received: u16) -> Response { + Json(SidecarResponse::error( + request_id, + SidecarError::new( + ClientErrorCategory::ProtocolMismatch, + format!("unsupported adapter protocol version {received}; expected {PROTOCOL_VERSION}"), + ), + )) + .into_response() +} + async fn dispatch_request( _state: AdapterHttpState, client: InlineClient, @@ -878,22 +933,7 @@ async fn dispatch_request( ) -> Response { let request_id = request.id.clone(); if request.protocol_version != PROTOCOL_VERSION { - log::warn!( - "adapter protocol mismatch: got {}, expected {}", - request.protocol_version, - PROTOCOL_VERSION - ); - return Json(SidecarResponse::error( - request_id, - SidecarError::new( - ClientErrorCategory::ProtocolMismatch, - format!( - "unsupported adapter protocol version {}; expected {}", - request.protocol_version, PROTOCOL_VERSION - ), - ), - )) - .into_response(); + return protocol_mismatch_response(request_id, request.protocol_version); } log::debug!("handling adapter command: {}", request.command.kind()); @@ -1088,7 +1128,6 @@ struct EventsQuery { #[derive(Debug, Deserialize, Serialize)] struct EventsAckRequest { session_namespace: String, - #[serde(default)] generation: String, sequence: u64, } @@ -1127,6 +1166,13 @@ async fn rpc_events_ack( ) .into_response(); } + if request.generation.trim().is_empty() { + return ( + StatusCode::BAD_REQUEST, + "event acknowledgement requires a generation", + ) + .into_response(); + } match state.event_store.acknowledge_for_generation( &namespace, Some(&request.generation), @@ -1193,9 +1239,24 @@ async fn events_ws( .into_response(); } let live = state.subscribe_events(); + // Capture identity even when replay is empty. Passing this identity to + // replay also rejects a reset between capture and the replay read. + let generation = match state.event_store.generation(&namespace) { + Ok(generation) => generation, + Err(error) => { + log::error!("failed to load adapter event generation: {error}"); + return StatusCode::INTERNAL_SERVER_ERROR.into_response(); + } + }; + let expected_generation = query + .generation + .as_deref() + .map(str::trim) + .filter(|expected| !expected.is_empty()) + .unwrap_or(&generation); let replay = match state.event_store.replay_for_generation( &namespace, - query.generation.as_deref(), + Some(expected_generation), query.after_sequence, ) { Ok(replay) => replay, @@ -1227,7 +1288,15 @@ async fn events_ws( } }; ws.on_upgrade(move |socket| { - event_socket(state, socket, namespace, query.after_sequence, replay, live) + event_socket( + state, + socket, + namespace, + generation, + query.after_sequence, + replay, + live, + ) }) .into_response() } @@ -1273,11 +1342,16 @@ async fn event_socket( state: AdapterHttpState, mut socket: WebSocket, namespace: String, + generation: String, mut last_sent: u64, replay: Vec, mut events: broadcast::Receiver, ) { for envelope in replay { + if envelope.generation != generation { + close_event_socket_for_reset(&mut socket).await; + return; + } if send_event_envelope(&mut socket, &envelope).await.is_err() { return; } @@ -1294,10 +1368,15 @@ async fn event_socket( log::warn!( "adapter websocket lagged by {skipped} live events; replaying after {last_sent}" ); - let replay = match state.event_store.replay(&namespace, last_sent) { + let replay = match state.event_store.replay_for_generation( + &namespace, + Some(&generation), + last_sent, + ) { Ok(replay) => replay, Err(error) => { log::error!("adapter websocket replay failed after lag: {error}"); + close_event_socket_for_reset(&mut socket).await; break; } }; @@ -1315,6 +1394,12 @@ async fn event_socket( if envelope.session_namespace != namespace { continue; } + // Compare identity before its sequence: low sequences in a new + // generation are a reset, never duplicates of this socket's cursor. + if envelope.generation != generation { + close_event_socket_for_reset(&mut socket).await; + break; + } if envelope .sequence .is_some_and(|sequence| sequence <= last_sent) @@ -1330,6 +1415,15 @@ async fn event_socket( } } +async fn close_event_socket_for_reset(socket: &mut WebSocket) { + let _ = socket + .send(Message::Close(Some(CloseFrame { + code: 1012, + reason: "event replay changed; reconnect and reconcile".into(), + }))) + .await; +} + async fn send_event_envelope( socket: &mut WebSocket, envelope: &SidecarEventEnvelope, @@ -1345,6 +1439,8 @@ async fn send_event_envelope( #[cfg(test)] mod tests { + use std::io::{Read as _, Write as _}; + use std::net::TcpStream; use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering}; use axum::{ @@ -1367,6 +1463,410 @@ mod tests { serde_json::from_slice(&bytes).unwrap() } + // A small wire fixture reads server frames only; it does not implement or + // reproduce Axum's upgrade machinery. No client account/network is needed. + fn fixture_socket(addr: SocketAddr, uri: &str) -> TcpStream { + let mut socket = TcpStream::connect(addr).unwrap(); + socket + .set_read_timeout(Some(Duration::from_secs(3))) + .unwrap(); + write!(socket, "GET {uri} HTTP/1.1\r\nHost: {addr}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\nX-Inline-Protocol-Version: 5\r\nX-Inline-Session-Namespace: fixture\r\n\r\n").unwrap(); + let mut headers = Vec::new(); + while !headers.ends_with(b"\r\n\r\n") { + let mut byte = [0]; + socket.read_exact(&mut byte).unwrap(); + headers.push(byte[0]); + assert!(headers.len() < 8192); + } + assert!( + headers.starts_with(b"HTTP/1.1 101"), + "{}", + String::from_utf8_lossy(&headers) + ); + socket + } + + fn fixture_frame(socket: &mut TcpStream) -> (u8, Vec) { + let mut header = [0; 2]; + socket + .read_exact(&mut header) + .expect("expected an event or reset close, not silence/EOF"); + assert_eq!(header[1] & 0x80, 0, "server frames are unmasked"); + let size = match header[1] & 0x7f { + 126 => { + let mut bytes = [0; 2]; + socket.read_exact(&mut bytes).unwrap(); + u16::from_be_bytes(bytes) as usize + } + 127 => { + let mut bytes = [0; 8]; + socket.read_exact(&mut bytes).unwrap(); + usize::try_from(u64::from_be_bytes(bytes)).unwrap() + } + size => size as usize, + }; + assert!(size < 64 * 1024, "fixture frame unexpectedly large"); + let mut payload = vec![0; size]; + socket.read_exact(&mut payload).unwrap(); + (header[0] & 0x0f, payload) + } + + fn assert_reset_close(socket: &mut TcpStream) { + let (opcode, payload) = fixture_frame(socket); + assert_eq!( + opcode, + 8, + "expected reset close, received {}", + String::from_utf8_lossy(&payload) + ); + assert!(payload.len() >= 2); + assert_eq!(u16::from_be_bytes([payload[0], payload[1]]), 1012); + } + + fn fixture_event() -> inline_client::ClientEvent { + inline_client::ClientEvent::ChatUpserted { + chat_id: InlineId::new(7), + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn websocket_generation_reset_after_empty_replay_closes_before_filtering() { + let client = InlineClient::builder().build().spawn(); + let state = AdapterHttpState::with_event_store( + client.clone(), + AdapterEventStore::open_in_memory("fixture").unwrap(), + ); + let generation = state.event_store.generation("fixture").unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let app = sidecar_router_with_state(state.clone()); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let mut socket = fixture_socket(addr, &format!("/ws/events?generation={generation}")); + state.event_store.remove_namespace("fixture").unwrap(); + let envelope = state + .event_store + .append_for_namespace("fixture", fixture_event()) + .unwrap(); + assert_ne!(envelope.generation, generation); + state.events.send(envelope).unwrap(); + assert_reset_close(&mut socket); + + // A reconnect negotiates the current generation and can replay it. + let current = state.event_store.generation("fixture").unwrap(); + let mut next = fixture_socket(addr, &format!("/ws/events?generation={current}")); + let (opcode, data) = fixture_frame(&mut next); + assert_eq!(opcode, 1); + let replay: SidecarEventEnvelope = serde_json::from_slice(&data).unwrap(); + assert_eq!(replay.generation, current); + assert_eq!(replay.sequence, Some(1)); + server.abort(); + client.shutdown().await.unwrap(); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn websocket_reset_does_not_suppress_new_low_sequence_regression() { + let client = InlineClient::builder().build().spawn(); + let state = AdapterHttpState::with_event_store( + client.clone(), + AdapterEventStore::open_in_memory("fixture").unwrap(), + ); + let first = state.event_store.append(fixture_event()).unwrap(); + let second = state.event_store.append(fixture_event()).unwrap(); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let app = sidecar_router_with_state(state.clone()); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let mut socket = fixture_socket( + addr, + &format!( + "/ws/events?generation={}&after_sequence=2", + first.generation + ), + ); + // Same-generation duplicates and unrelated namespaces must not reset or + // leak; the following same-generation sequence must be the next frame. + state.events.send(second).unwrap(); + let unrelated = state + .event_store + .append_for_namespace("other", fixture_event()) + .unwrap(); + state.events.send(unrelated).unwrap(); + let third = state.event_store.append(fixture_event()).unwrap(); + state.events.send(third.clone()).unwrap(); + let (opcode, data) = fixture_frame(&mut socket); + assert_eq!(opcode, 1); + let received: SidecarEventEnvelope = serde_json::from_slice(&data).unwrap(); + assert_eq!(received, third); + state.event_store.remove_namespace("fixture").unwrap(); + let next = state.event_store.append(fixture_event()).unwrap(); + assert_eq!(next.sequence, Some(1)); + state.events.send(next).unwrap(); + assert_reset_close(&mut socket); + server.abort(); + client.shutdown().await.unwrap(); + } + + #[hegel::test(test_cases = 1000)] + fn websocket_lag_recovery_stays_in_generation(tc: hegel::TestCase) { + let watermark = tc.draw(hegel::generators::integers::()); + let reset = tc.draw(hegel::generators::booleans()); + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + runtime.block_on(async { + let client = InlineClient::builder().build().spawn(); + let state = AdapterHttpState::with_event_store( + client.clone(), + AdapterEventStore::open_in_memory("fixture").unwrap(), + ); + let original_generation = state.event_store.generation("fixture").unwrap(); + let live = state.subscribe_events(); + // Deliberately lag the actual broadcast receiver before the socket + // starts consuming, avoiding scheduler/sleep-based lag fixtures. + let transient = inline_client::ClientEvent::Typing { + chat_id: InlineId::new(7), + user_id: InlineId::new(8), + is_typing: true, + }; + for _ in 0..257 { + let event = state + .event_store + .append_for_namespace("other", transient.clone()) + .unwrap(); + state.events.send(event).unwrap(); + } + if reset { + state.event_store.remove_namespace("fixture").unwrap(); + } + let next = state.event_store.append(fixture_event()).unwrap(); + let state_for_socket = state.clone(); + let expected = next.clone(); + let generation_for_socket = original_generation.clone(); + let live = Arc::new(StdMutex::new(Some(live))); + let app = Router::new().route( + "/fixture", + get(move |ws: WebSocketUpgrade| { + let state = state_for_socket.clone(); + let live = live.clone(); + let generation = generation_for_socket.clone(); + async move { + let live = live.lock().unwrap().take().unwrap(); + ws.on_upgrade(move |socket| { + event_socket( + state, + socket, + "fixture".to_owned(), + generation, + if reset { watermark } else { 0 }, + Vec::new(), + live, + ) + }) + } + }), + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let mut socket = fixture_socket(addr, "/fixture"); + if reset { + assert_ne!(next.generation, original_generation); + assert_reset_close(&mut socket); + } else { + let (opcode, data) = fixture_frame(&mut socket); + assert_eq!(opcode, 1); + let received: SidecarEventEnvelope = serde_json::from_slice(&data).unwrap(); + assert_eq!(received, expected); + } + server.abort(); + client.shutdown().await.unwrap(); + }); + } + + #[tokio::test] + async fn invalid_wire_request_id_rejects_before_factory() { + let calls = Arc::new(AtomicUsize::new(0)); + let counted = calls.clone(); + let factory: AdapterClientFactory = Arc::new(move |_| { + counted.fetch_add(1, AtomicOrdering::SeqCst); + Box::pin(async { Err("factory must not run".to_owned()) }) + }); + let state = AdapterHttpState::with_client_factory( + AdapterEventStore::open_in_memory("fixture").unwrap(), + factory, + ); + let app = sidecar_router_with_state(state); + for id in [ + "".to_owned(), + " \t ".to_owned(), + "x\0y".to_owned(), + "a".repeat(129), + "é".repeat(65), + ] { + let request = + serde_json::json!({"protocol_version":5,"id":id,"command":{"type":"status"}}); + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/rpc") + .header(PROTOCOL_VERSION_HEADER, "5") + .header(SESSION_NAMESPACE_HEADER, "fixture") + .header("content-type", "application/json") + .body(Body::from(request.to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::UNPROCESSABLE_ENTITY, + "{id:?}" + ); + } + assert_eq!(calls.load(AtomicOrdering::SeqCst), 0); + } + + #[tokio::test] + async fn generic_rpc_body_version_rejects_before_account_factory() { + let calls = Arc::new(AtomicUsize::new(0)); + let counted = calls.clone(); + let factory: AdapterClientFactory = Arc::new(move |_| { + counted.fetch_add(1, AtomicOrdering::SeqCst); + Box::pin(async { Err("factory must not run".to_owned()) }) + }); + let state = AdapterHttpState::with_client_factory( + AdapterEventStore::open_in_memory("fixture").unwrap(), + factory, + ); + let request = serde_json::json!({"protocol_version":4,"id":"fixture-rpc","command":{"type":"status"}}); + let response = sidecar_router_with_state(state) + .oneshot( + Request::builder() + .method("POST") + .uri("/rpc") + .header(PROTOCOL_VERSION_HEADER, "5") + .header(SESSION_NAMESPACE_HEADER, "fixture") + .header("content-type", "application/json") + .body(Body::from(request.to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(calls.load(AtomicOrdering::SeqCst), 0); + let response: SidecarResponse = json_response(response).await; + assert!(matches!(response.outcome, SidecarOutcome::Error(_))); + } + + #[hegel::test] + fn protocol_version_requires_exact_supported_value(tc: hegel::TestCase) { + use hegel::generators as gs; + let version = tc.draw(gs::integers::()); + let mut headers = HeaderMap::new(); + headers.insert( + PROTOCOL_VERSION_HEADER, + version.to_string().parse().unwrap(), + ); + assert_eq!(compatible_protocol(&headers), version == 5); + headers.append(PROTOCOL_VERSION_HEADER, "5".parse().unwrap()); + assert!( + !compatible_protocol(&headers), + "duplicate version headers are ambiguous" + ); + } + + #[tokio::test] + async fn protocol_gate_precedes_every_account_factory() { + let calls = Arc::new(AtomicUsize::new(0)); + let counted = calls.clone(); + let factory: AdapterClientFactory = Arc::new(move |_| { + counted.fetch_add(1, AtomicOrdering::SeqCst); + Box::pin(async { Err("factory must not run".to_owned()) }) + }); + let store = AdapterEventStore::open_in_memory("fixture").unwrap(); + let app = sidecar_router_with_state(AdapterHttpState::with_client_factory(store, factory)); + let paths = [ + "/health", + "/status", + "/ws/events", + "/rpc", + "/rpc/auth/start", + "/rpc/auth/verify", + "/rpc/resume", + "/rpc/connect", + "/rpc/logout", + "/rpc/dialogs", + "/rpc/state/dialogs", + "/rpc/state/account", + "/rpc/state/chat", + "/rpc/state/chat/history", + "/rpc/history", + "/rpc/chat/participants", + "/rpc/chat/participants/add", + "/rpc/chat/participants/remove", + "/rpc/chat/info", + "/rpc/chat/delete", + "/rpc/chat/create-dm", + "/rpc/chat/create-thread", + "/rpc/chat/create-reply-thread", + "/rpc/send", + "/rpc/send_text", + "/rpc/edit", + "/rpc/delete", + "/rpc/react", + "/rpc/read", + "/rpc/marked-unread", + "/rpc/dialog/notifications", + "/rpc/typing", + "/rpc/events/ack", + "/rpc/upload", + ]; + for path in paths { + for version in [None, Some("4"), Some("6"), Some("garbage")] { + let method = if path.starts_with("/rpc") && path != "/rpc/state/account" { + "POST" + } else { + "GET" + }; + let mut request = Request::builder() + .method(method) + .uri(path) + .header(SESSION_NAMESPACE_HEADER, "fixture"); + if let Some(version) = version { + request = request.header("x-inline-protocol-version", version); + } + let response = app + .clone() + .oneshot(request.body(Body::from("not json")).unwrap()) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::CONFLICT, + "{path} {version:?}" + ); + } + } + assert_eq!(calls.load(AtomicOrdering::SeqCst), 0); + let response = app + .oneshot( + Request::builder() + .uri("/compatibility") + .header(SESSION_NAMESPACE_HEADER, "ignored") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let info: ProtocolInfo = json_response(response).await; + assert_eq!(info.protocol_version, 5); + assert_eq!(calls.load(AtomicOrdering::SeqCst), 0); + } + #[tokio::test] async fn health_endpoint_reports_protocol_and_status() { let client = InlineClient::builder() @@ -1378,6 +1878,7 @@ mod tests { let response = app .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .uri("/health") .body(Body::empty()) .unwrap(), @@ -1446,7 +1947,7 @@ mod tests { let app = sidecar_router_with_state(state.clone()); let body = serde_json::to_vec(&EventsAckRequest { session_namespace: "test".to_owned(), - generation: String::new(), + generation: state.event_store().generation("test").unwrap(), sequence: 1, }) .unwrap(); @@ -1454,6 +1955,7 @@ mod tests { let response = app .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .method("POST") .uri("/rpc/events/ack") .header("content-type", "application/json") @@ -1488,6 +1990,7 @@ mod tests { let response = app .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .method("POST") .uri("/rpc/events/ack") .header(SESSION_NAMESPACE_HEADER, "test") @@ -1524,6 +2027,7 @@ mod tests { let response = app .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .uri("/status") .body(Body::empty()) .unwrap(), @@ -1579,6 +2083,7 @@ mod tests { .clone() .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .uri("/status") .header(SESSION_NAMESPACE_HEADER, namespace) .body(Body::empty()) @@ -1600,6 +2105,7 @@ mod tests { let missing = app .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .uri("/status") .body(Body::empty()) .unwrap(), @@ -1626,6 +2132,7 @@ mod tests { let response = app .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .uri("/status") .header(SESSION_NAMESPACE_HEADER, "account-1") .body(Body::empty()) @@ -1669,6 +2176,7 @@ mod tests { .clone() .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .uri("/status") .header(SESSION_NAMESPACE_HEADER, "account-1") .body(Body::empty()) @@ -1682,6 +2190,7 @@ mod tests { .clone() .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .method("POST") .uri("/rpc/logout") .header(SESSION_NAMESPACE_HEADER, "account-1") @@ -1695,6 +2204,7 @@ mod tests { let recreated = app .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .uri("/status") .header(SESSION_NAMESPACE_HEADER, "account-1") .body(Body::empty()) @@ -1717,6 +2227,7 @@ mod tests { let response = app .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .method("POST") .uri("/rpc/resume") .body(Body::empty()) @@ -1748,6 +2259,7 @@ mod tests { app.clone() .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .method("POST") .uri("/rpc/connect") .header("content-type", "application/json") @@ -1767,6 +2279,7 @@ mod tests { let response = app .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .method("POST") .uri("/rpc/send") .header("content-type", "application/json") @@ -1811,6 +2324,7 @@ mod tests { app.clone() .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .method("POST") .uri("/rpc/connect") .header("content-type", "application/json") @@ -1827,6 +2341,7 @@ mod tests { let response = app .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .method("POST") .uri("/rpc/chat/participants") .header("content-type", "application/json") @@ -1861,6 +2376,7 @@ mod tests { app.clone() .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .method("POST") .uri("/rpc/connect") .header("content-type", "application/json") @@ -1926,6 +2442,7 @@ mod tests { .clone() .oneshot( Request::builder() + .header(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION.to_string()) .method("POST") .uri(uri) .header("content-type", "application/json") diff --git a/crates/adapter/src/protocol.rs b/crates/adapter/src/protocol.rs index 3c52570..581ba70 100644 --- a/crates/adapter/src/protocol.rs +++ b/crates/adapter/src/protocol.rs @@ -16,7 +16,7 @@ use inline_client::{ use serde::{Deserialize, Serialize}; /// Current bridge adapter command/event protocol version. -pub const PROTOCOL_VERSION: u16 = 4; +pub const PROTOCOL_VERSION: u16 = 5; /// Version metadata returned by health/status endpoints. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -81,10 +81,17 @@ impl SidecarStatus { } /// Stable request ID supplied by the Go bridge or generated by HTTP wrappers. -#[derive(Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Clone, PartialEq, Eq, Hash, Serialize)] #[serde(transparent)] pub struct SidecarRequestId(String); +impl<'de> Deserialize<'de> for SidecarRequestId { + fn deserialize>(deserializer: D) -> Result { + let raw = String::deserialize(deserializer)?; + Self::try_new(raw).map_err(serde::de::Error::custom) + } +} + impl SidecarRequestId { /// Maximum request ID length in bytes. pub const MAX_LEN: usize = 128; @@ -498,6 +505,46 @@ mod tests { use super::*; + #[hegel::test(test_cases = 2000)] + fn wire_request_id_agrees_with_constructor(tc: hegel::TestCase) { + let raw = tc.draw(hegel::generators::text()); + let expected = SidecarRequestId::try_new(&raw); + let wire = serde_json::to_string(&raw).unwrap(); + let decoded = serde_json::from_str::(&wire); + match expected { + Ok(expected) => assert_eq!(decoded.unwrap(), expected), + Err(_) => assert!(decoded.is_err(), "invalid wire ID accepted: {raw:?}"), + } + } + + #[test] + fn wire_request_id_utf8_and_invalid_regressions() { + for raw in [ + "".to_owned(), + " ".to_owned(), + "x\0y".to_owned(), + "x\ny".to_owned(), + "a".repeat(129), + "é".repeat(65), + ] { + let wire = serde_json::to_string(&raw).unwrap(); + assert!( + serde_json::from_str::(&wire).is_err(), + "{raw:?}" + ); + } + for raw in [ + "a".repeat(128), + "é".repeat(64), + format!(" \t{}\n", "é".repeat(64)), + ] { + let decoded: SidecarRequestId = + serde_json::from_str(&serde_json::to_string(&raw).unwrap()).unwrap(); + assert_eq!(decoded.as_str(), raw.trim()); + assert_eq!(decoded.as_str().len(), 128); + } + } + #[test] fn request_round_trips_as_tagged_json() { let req = SidecarRequest::current( @@ -510,6 +557,12 @@ mod tests { external_id: Some(ExternalId::try_new("matrix-event", "$event").unwrap()), random_id: Some(RandomId::new(99)), reply_to_message_id: None, + ..SendTextRequest::new( + PeerRef::Chat { + chat_id: InlineId::new(7), + }, + "", + ) }), ); diff --git a/deploy/systemd/README.md b/deploy/systemd/README.md index 1d0ad34..68ae828 100644 --- a/deploy/systemd/README.md +++ b/deploy/systemd/README.md @@ -86,5 +86,5 @@ sudo systemctl enable --now matrix-inline.service systemctl status matrix-inline-adapter.service systemctl status matrix-inline.service journalctl -u matrix-inline-adapter.service -u matrix-inline.service -f -curl -fsS http://127.0.0.1:29342/health +curl -fsS -H "X-Inline-Protocol-Version: 5" http://127.0.0.1:29342/health ``` diff --git a/docker-run.sh b/docker-run.sh index 56254c6..037b65f 100755 --- a/docker-run.sh +++ b/docker-run.sh @@ -1,6 +1,25 @@ #!/bin/bash set -Eeuo pipefail +function wait_for_sidecar { + for _ in $(seq 1 60); do + if curl -fsS -H "X-Inline-Protocol-Version: 5" "http://${INLINE_SIDECAR_BIND}/health" >/dev/null 2>&1; then + break + fi + if ! kill -0 "${sidecar_pid}" 2>/dev/null; then + wait "${sidecar_pid}" || true + return 1 + fi + sleep 1 + done + curl -fsS -H "X-Inline-Protocol-Version: 5" "http://${INLINE_SIDECAR_BIND}/health" >/dev/null 2>&1 +} + +# The readiness helper can be sourced without config, store or service effects. +if [[ "${BASH_SOURCE[0]}" != "${0}" ]]; then + return 0 +fi + BINARY_NAME=/usr/bin/matrix-inline SIDECAR_NAME=/usr/bin/matrix-inline-adapter DATA_DIR=${DATA_DIR:-/data} @@ -154,17 +173,7 @@ function stop_children { trap stop_children INT TERM -for _ in $(seq 1 60); do - if curl -fsS "http://${INLINE_SIDECAR_BIND}/health" >/dev/null 2>&1; then - break - fi - if ! kill -0 "${sidecar_pid}" 2>/dev/null; then - wait "${sidecar_pid}" - fi - sleep 1 -done - -if ! curl -fsS "http://${INLINE_SIDECAR_BIND}/health" >/dev/null 2>&1; then +if ! wait_for_sidecar; then echo "matrix-inline-adapter did not become healthy" >&2 stop_children exit 1 diff --git a/docs/adapter-protocol.md b/docs/adapter-protocol.md new file mode 100644 index 0000000..f7f16b2 --- /dev/null +++ b/docs/adapter-protocol.md @@ -0,0 +1,39 @@ +# Adapter compatibility + +The bridge and adapter use protocol 5. Upgrade them together with the matching +`inline-client` revision recorded by the build. Keep each account's namespace +and durable stores when upgrading. + +`GET /compatibility` returns protocol metadata without selecting an account, +opening a native client or creating an event-log namespace. Clients must verify +`protocol_version` before calling account routes. The Go bridge probes before +every request and websocket connection, including direct authentication calls. + +Every other route requires `X-Inline-Protocol-Version: 5`, including `/health`, +`/status`, multipart uploads, event acknowledgements and websocket setup. The +adapter rejects missing, unsupported, malformed or duplicate version headers +with HTTP 409 before resolving an account or parsing a request body. The generic +`/rpc` envelope must also have `protocol_version: 5`; it is validated before the +account client is resolved. Alias routes use the required header as their +request version. + +For a manual compatibility check: + +```sh +curl -fsS http://127.0.0.1:29342/compatibility | jq . +curl -fsS -H 'X-Inline-Protocol-Version: 5' http://127.0.0.1:29342/health | jq . +``` + +Protocol 4 adapters do not implement this ingress guarantee. The preflight +rejects an old adapter before an operation when it is already installed. It +cannot prevent an old adapter from ignoring the version header if it replaces +the compatible process between the probe and that operation. Stop bridge +clients before a downgrade; an in-flight downgrade is unsupported. + +Protocol 5 carries a nullable complete structural descriptor for a dialog. +An absent descriptor means unknown metadata; explicit null parent or workspace +IDs in a complete descriptor clear the old relation. Structural completeness +does not imply complete read state, notifications or membership. Cached space, +member and user arrays likewise do not establish current access: consumers +must check access evidence and its completeness before creating navigation or +enabling operations. diff --git a/docs/deployment.md b/docs/deployment.md index 7c587be..b535b3b 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -282,7 +282,7 @@ Check status: systemctl status matrix-inline-adapter.service systemctl status matrix-inline.service journalctl -u matrix-inline-adapter.service -u matrix-inline.service -f -curl -fsS http://127.0.0.1:29342/health +curl -fsS -H "X-Inline-Protocol-Version: 5" http://127.0.0.1:29342/health ``` ## After Deployment diff --git a/docs/operations.md b/docs/operations.md index 3dcecde..d2ffdff 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -28,7 +28,7 @@ Start the adapter before the bridge. Stop the bridge before the adapter. Adapter health: ```sh -curl -fsS http://127.0.0.1:29342/health | jq . +curl -fsS -H "X-Inline-Protocol-Version: 5" http://127.0.0.1:29342/health | jq . ``` Account-specific adapter endpoints require the bridge's private session @@ -93,18 +93,108 @@ Adapter event cursors are `(generation, sequence)` pairs. If the adapter SQLite database is restored behind the Matrix database, the adapter rotates to a new generation and returns a structured reset. The Go bridge then performs authoritative reconciliation and atomically replaces its saved cursor before -resuming replay. Protocol v4 requires the Go bridge and Rust sidecar to be +resuming replay. Protocol v5 requires the Go bridge and Rust sidecar to be upgraded together. +Receipt reconciliation conservatively reinspects timeline order after delivery, +including messages beyond the source read cursor. Inspection is limited to 20 +pages of 500 events and a five-second reconciliation timeout. Long timelines can +delay the serialized room lane or leave read projection pending. The persisted +applied cursor alone cannot justify a shortcut after failed completion saves, +late delivery, deletion or mapping changes; receipt recovery remains enabled. + Dialog reconciliation uses stable chat-ID keyset pagination and persists its next cursor in login metadata. Activity reordering cannot skip rows, and large accounts resume from the last completed page after restart or RPC-budget yield. -Delivered message IDs are also persisted monotonically in portal metadata. +Live message delivery and historical coverage use separate monotonic portal +checkpoints. A live message does not skip an older history page. History advances +only for an explicit source-ID catchup page requested from its current historical +frontier, after every selected message has a real mapping in the same portal and +receiver. Generic forward and backward pages verify mappings without advancing +that frontier; a new empty frontier requests source IDs after zero. Existing message and room IDs are retained. Enable +`backfill.enabled` to import or recover history; disabled backfill leaves history +pending. The pinned framework's timestamp cutoff is supplemented by bounded +source-ID recovery in the existing portal event lane. Permanent media projection failures (invalid schemes, reviewed terminal 4xx, or the configured size ceiling) produce a stable Matrix unavailable notice and allow the lossless cursor to continue. Timeouts, 429, and 5xx remain retryable. +Matrix kicks and invite revocations are unsupported because removing a direct +participant may leave access through a group or public workspace. The bridge +does not claim the user lost access when it cannot verify that result. + +### Rich messages and media + +Bold, italic, code, preformatted text and safe links survive both directions. +Entity ranges use checked UTF-16 offsets. Only typed mentions whose identities +are verified for the current login become user mentions; unknown identities +stay readable and group mentions never become room-wide broadcasts. + +Native message revisions take precedence over edit dates. Older or equal +revisions do not replace accepted content. When the source supplies neither, +content fingerprints suppress identical replays but cannot establish an order +between different revisions. Formatting changes are included in fingerprints. + +Caption edits reuse the saved Matrix media object, including after restart. +Caption-only edits require the original Matrix media type, URI, filename and +attachment metadata. Clients that rewrite or omit that metadata can receive an +explicit unsupported error. Replacing a file through an edit is unsupported. Old messages without enough +saved media identity reject caption edits rather than silently ignoring a file +replacement. + +### Accessible people and channel creation + +User lookup accepts exact positive IDs and canonical usernames from this +login's accessible directory. Search returns a bounded subset, not a global +directory. Cached users alone do not prove access. Bounded or ambiguous username +results return an error rather than selecting an arbitrary person. Email and +phone lookup are unsupported. A missed exact ID in a truncated refresh also +returns an incomplete-result error; a known exact ID remains usable. + +Peers known only through hidden or deleted conversations are excluded from +directory discovery. This is an eligibility limit, not a claim that a hidden +peer is unreachable or that Inline denied permission. The hidden-dialog +`include` setting changes room projection, not this directory policy. An +independent accessible-workspace member or successful live participant read +can still make the same peer discoverable. Restore source discovery before +starting a DM from Matrix when no eligible entry exists. + +`thread` and `private_thread` create ordinary private Inline channels, with +explicit accessible participants. An optional parent must identify an accessible +workspace for the same login. `public_thread` requires that workspace parent +and no explicit participants. A true group DM is unsupported. Current Inline +permission errors remain authoritative; cached membership or role information +does not grant permission to create or manage a channel. + +### Separate reply-child context + +Eligible reply-child conversations keep their own room, message IDs, history +and access boundaries. Their topics retain readable source parent context and, +when the parent message already has a mapping for this login, a parent backlink. +Missing or deleted roots keep the source chat/message reference. Context is +written only inside the child; it does not publish private child details in the +parent. The default hidden-dialog policy still excludes hidden children. + +Source channel descriptions can populate empty or connector-owned Matrix topics; +later user topics are preserved. This read-side projection does not enable topic +editing or generate DM topics. Temporary parent-mapping database errors retain +existing child context; conclusive missing mappings retain readable native IDs. + +Generated topics use a durable pending-intent journal. An authoritative detach +retracts only connector-owned context after Matrix state comparison/readback; +unknown descriptors, later user edits and legacy unowned topics are preserved. +Pending topic writes can retry on a later portal event, at most once per room +every 30 seconds. An idle room has no automatic retry, and failures before an +intent is saved require a later dialog reconciliation. +Topic replacement requires a controlled window: Matrix state comparison is not +an atomic compare-and-swap with concurrent user edits. + +Matrix thread routing and native child creation are unsupported. Thread history +requests fail explicitly, and a parent chat's message ID cannot be sent as a +child-local reply ID. No audience/history equivalence or thread flattening is +assumed. + ### Hidden Inline dialogs The network config defaults `hidden_dialogs` to `exclude`. Dialogs whose Inline @@ -129,6 +219,87 @@ not left or deleted automatically, but new Inline events for the hidden dialog are no longer projected. If the Inline dialog later becomes visible, normal reconciliation resumes and fills its missing history from durable state. +### Channel identity and workspace navigation + +Authoritative Inline structure distinguishes DMs, channels, and separate reply +children. Channels use ordinary Matrix rooms; this changes their presentation in +clients that previously treated all non-DMs as group DMs. A partial or conflicting +descriptor does not invent a room type. Descriptions and workspace names come +from the account's cached source records. + +A workspace becomes a Matrix Space only with source-backed access evidence for +that login. Explicitly removed access or absence from a complete access list detaches +navigation; unknown access holds existing links and pending hierarchy work. A +failed snapshot refresh retains the last known snapshot and does not trigger a +detach. Navigation records never grant mutation permission or delete +independently accessible chat rooms or their history. Access proof is refreshed +in bounded passes, so a newly discovered workspace can appear later. Status +reports unverified access without exposing workspace names or member lists. + +### Notification and read-state limits + +`network.mute_channels_by_default: true` records the operator's desired default +for new channels. Each login can override that desired default: + +```text +inline-channel-default muted +inline-channel-default normal +inline-channel-default default +inline-settings +``` + +Automatic mute, Low Priority, and manual-unread writes are disabled because the +current target cannot reliably distinguish their bridge echoes from later user +choices. `notification_policy_degraded` is reported immediately and message +delivery continues. These settings do not silently quiet channels. For a quiet +inbox, mute channels manually in Beeper/Matrix; putting them in Low Priority and +choosing a DMs view are separate client settings. Inline's native notification +mode is preserved independently. Explicit user mute/unmute remains supported. + +Incoming source read state uses only the verified prefix of the actual Matrix +timeline. An unmapped event or an unread source event is a barrier, including +stickers and polls. Timeline inspection is bounded to 10,000 events; missing +mappings, exhausted bounds, or a missing double puppet leave the intent pending. +With double puppeting, the bridge writes private read receipts and fully-read +markers, verifies their readback, and checks that manual-unread was untouched. +Exact receipt observations and sync positions survive later readback failures in +the journal, bound to the account, room, puppet, target, and current source intent. +A changed intent or conflicting private receipt invalidates that evidence. +It does not emit public source-driven receipts. The pinned mautrix version +forwards genuine public user receipts to Inline but ignores genuine private user +receipts, which is a known outgoing fidelity limit. Unverified manual-unread +projection remains pending rather than being claimed as applied. + +### Existing-room presentation migration + +Migration preserves room IDs, message mappings, history checkpoints, and local +notification/tag state. Select 1–50 explicit positive numeric chat IDs in the +management room; omission of a mode means preview: + +```text +inline-migrate preview 202 203 +inline-migrate apply 202 203 +inline-migrate rollback 202 203 +``` + +Preview performs no presentation writes. Apply journals the before and desired +room kind, workspace links, and bridge metadata before making effects. A repeated +apply or rollback resumes an interrupted operation. Every link write uses an +exact journaled value, so restart can distinguish a partial migration from an +unrelated later edit. Rollback restores saved relationship metadata as well as +both link directions. A differing user value causes a conflict. Rolled-back or +conflicted presentation stays held until explicitly selected again. The hold runs +before the framework receives mutable room metadata, including on resync after a +restart. History checks and post-delivery reconciliation continue while +presentation is held. + +Run apply/rollback in a controlled window with no concurrent administrators or +clients editing room presentation. Matrix has no atomic conditional state write: +per-write read guards detect existing edits, but an edit racing between the read +and write can still be overwritten. This is not an atomic remote compare-and-set +guarantee. Inspect preview and status before and after the operation; resolve +conflicts deliberately rather than repeatedly overriding a user's choice. + ## Backups Stop the bridge before making filesystem-level backups. @@ -256,5 +427,5 @@ ireconnect - Run: ```sh -curl -fsS -X POST http://127.0.0.1:29342/rpc/resume | jq . +curl -fsS -H "X-Inline-Protocol-Version: 5" -X POST http://127.0.0.1:29342/rpc/resume | jq . ``` diff --git a/docs/smoke-test.md b/docs/smoke-test.md index 5c136cc..5d41e33 100644 --- a/docs/smoke-test.md +++ b/docs/smoke-test.md @@ -21,9 +21,9 @@ catch-up and realtime delivery after reconnecting an existing login. For an installed bridge: ```sh -curl -fsS http://127.0.0.1:29342/health | jq . -curl -fsS http://127.0.0.1:29342/status | jq . -curl -fsS -X POST http://127.0.0.1:29342/rpc/resume | jq . +curl -fsS -H "X-Inline-Protocol-Version: 5" http://127.0.0.1:29342/health | jq . +curl -fsS -H "X-Inline-Protocol-Version: 5" http://127.0.0.1:29342/status | jq . +curl -fsS -H "X-Inline-Protocol-Version: 5" -X POST http://127.0.0.1:29342/rpc/resume | jq . ``` Expected adapter states: diff --git a/go.mod b/go.mod index 563ca42..c57bd59 100644 --- a/go.mod +++ b/go.mod @@ -1,24 +1,27 @@ module github.com/inline-chat/matrix-inline -go 1.25.0 +go 1.26.1 require ( github.com/coder/websocket v1.8.14 + github.com/mattn/go-sqlite3 v1.14.44 + github.com/rs/zerolog v1.35.1 go.mau.fi/util v0.9.9 + golang.org/x/net v0.55.0 gopkg.in/yaml.v3 v3.0.1 + hegel.dev/go/hegel v0.9.30 maunium.net/go/mautrix v0.28.0 ) require ( filippo.io/edwards25519 v1.2.0 // indirect github.com/coreos/go-systemd/v22 v22.7.0 // indirect + github.com/ebitengine/purego v0.12.0-alpha.1.0.20261015130536-d456b1bb95e8 // indirect github.com/lib/pq v1.12.3 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-sqlite3 v1.14.44 // indirect github.com/petermattis/goid v0.0.0-20260330135022-df67b199bc81 // indirect github.com/rs/xid v1.6.0 // indirect - github.com/rs/zerolog v1.35.1 // indirect github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect github.com/tidwall/gjson v1.19.0 // indirect github.com/tidwall/match v1.1.1 // indirect @@ -28,9 +31,8 @@ require ( go.mau.fi/zeroconfig v0.2.0 // indirect golang.org/x/crypto v0.52.0 // indirect golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a // indirect - golang.org/x/net v0.55.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.45.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/sys v0.48.0 // indirect golang.org/x/text v0.37.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect maunium.net/go/mauflag v1.0.0 // indirect diff --git a/go.sum b/go.sum index aa43660..d8cc1d3 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ebitengine/purego v0.12.0-alpha.1.0.20261015130536-d456b1bb95e8 h1:l5iZODQBpef/JRTKyr32A+0uzYmtNhglAg42fFgK3+0= +github.com/ebitengine/purego v0.12.0-alpha.1.0.20261015130536-d456b1bb95e8/go.mod h1:DCHPP08djqhNSoTfImcnHYQRZmd0qhakvrozqaEYhGQ= github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -50,11 +52,11 @@ golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a h1:+3jdDGGB8NGb1Zktc737jlt3/ golang.org/x/exp v0.0.0-20260508232706-74f9aab9d74a/go.mod h1:d2fgXJLVs4dYDHUk5lwMIfzRzSrWCfGZb0ZqeLa/Vcw= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= -golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/text v0.37.0 h1:Cqjiwd9eSg8e0QAkyCaQTNHFIIzWtidPahFWR83rTrc= golang.org/x/text v0.37.0/go.mod h1:a5sjxXGs9hsn/AJVwuElvCAo9v8QYLzvavO5z2PiM38= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= @@ -63,6 +65,8 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +hegel.dev/go/hegel v0.9.30 h1:FESJbmmlbDd5FijNZMK+hiH6wqXAKLTWPnmnXbu22d4= +hegel.dev/go/hegel v0.9.30/go.mod h1:32oDGA+F4vXA9fLQ/IoyZGIKir8JWjop3fcb6LNs2t4= maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= maunium.net/go/mautrix v0.28.0 h1:vBakLzf8MAdfED3NzAKiMeKQbc3AQ4EAS03NC+TVMXQ= diff --git a/pkg/connector/backlink_projection_test.go b/pkg/connector/backlink_projection_test.go new file mode 100644 index 0000000..ebf294d --- /dev/null +++ b/pkg/connector/backlink_projection_test.go @@ -0,0 +1,163 @@ +package connector + +import ( + "fmt" + "hegel.dev/go/hegel" + "math" + "strings" + "testing" + + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/id" +) + +func seedTopicParent(t *testing.T, f *topicFixture, chat int64, eventID id.EventID) { + t.Helper() + key := f.ic.portalKeyForChat(chat) + db := f.ic.UserLogin.Bridge.DB + if err := db.Portal.Insert(t.Context(), &database.Portal{PortalKey: key, MXID: id.RoomID("!parent-" + string(eventID) + ":test"), Metadata: &PortalMetadata{}}); err != nil { + t.Fatal(err) + } + if err := db.Message.Insert(t.Context(), &database.Message{ID: makeMessageID(chat, 9), Room: key, MXID: eventID, Metadata: &MessageMetadata{}}); err != nil { + t.Fatal(err) + } +} +func breakTopicMessageReads(t *testing.T, f *topicFixture) { + t.Helper() + db := f.ic.UserLogin.Bridge.DB + if _, err := db.Exec(t.Context(), `ALTER TABLE message RENAME TO message_hold`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(t.Context(), `CREATE VIEW message AS SELECT * FROM message_hold WHERE json_extract('not-json','$')`); err != nil { + t.Fatal(err) + } +} +func restoreTopicMessageReads(t *testing.T, f *topicFixture) { + t.Helper() + db := f.ic.UserLogin.Bridge.DB + if _, err := db.Exec(t.Context(), `DROP VIEW message`); err != nil { + t.Fatal(err) + } + if _, err := db.Exec(t.Context(), `ALTER TABLE message_hold RENAME TO message`); err != nil { + t.Fatal(err) + } +} +func TestSourceTopicBacklinkDBFailurePreservesOwnedText(t *testing.T) { + f := newTopicFixture(t) + seedTopicParent(t, f, 3, "$root3") + f.apply(t, topicChild()) + before := f.topic + if !strings.Contains(before, "matrix.to") { + t.Fatal("backlink was not projected") + } + calls, writes := f.matrixCalls, f.writes + breakTopicMessageReads(t, f) + f.apply(t, topicChild()) + if f.topic != before || f.writes != writes || f.matrixCalls != calls { + t.Fatalf("transient DB error rewrote owned context: topic%q writes%d=>%d Matrixcalls%d=>%d", f.topic, writes, f.writes, calls, f.matrixCalls) + } + restoreTopicMessageReads(t, f) + if _, err := f.ic.UserLogin.Bridge.DB.Exec(t.Context(), `DELETE FROM message WHERE id=$1`, "3/9"); err != nil { + t.Fatal(err) + } + f.apply(t, topicChild()) + if strings.Contains(f.topic, "matrix.to") || !strings.Contains(f.topic, "chat 3") || f.writes != writes+1 { + t.Fatalf("conclusive absence did not retain readable numeric context: %q writes%d", f.topic, f.writes) + } +} +func TestSourceTopicPrecreationDBFailureExposesNoStaleCache(t *testing.T) { + f := newTopicFixture(t) + seedTopicParent(t, f, 3, "$root3") + seedTopicParent(t, f, 8, "$root8") + f.portal.MXID = "" + if err := f.portal.Save(t.Context()); err != nil { + t.Fatal(err) + } + f.apply(t, topicChild()) + old := f.portal.Topic + if !strings.Contains(old, "chat 3") { + t.Fatal("old creation context not cached") + } + breakTopicMessageReads(t, f) + changed := topicChild() + changed.Structure.ParentChatID = foundationInt64(8) + f.apply(t, changed) + stored, err := f.ic.UserLogin.Bridge.DB.Portal.GetByKey(t.Context(), f.portal.PortalKey) + if err != nil { + t.Fatal(err) + } + pending := stored.Metadata.(*PortalMetadata).InlineTopic.Pending + if f.portal.Topic != "" || pending == nil || pending.Desired != old || f.matrixCalls != 0 { + t.Fatalf("lookup failure exposed non-durable context cache%q pending%+v calls%d", f.portal.Topic, pending, f.matrixCalls) + } + restoreTopicMessageReads(t, f) + f.apply(t, changed) + if !strings.Contains(f.portal.Topic, "chat 8") || !strings.Contains(f.portal.Topic, "$root8") || f.matrixCalls != 0 { + t.Fatalf("creation did not recover latest context: %q calls%d", f.portal.Topic, f.matrixCalls) + } +} +func TestSourceTopicChannelDescriptionFollowsOwnership(t *testing.T) { + f := newTopicFixture(t) + description := "source description" + d := topicChannel() + d.Description = &description + f.apply(t, d) + if f.topic != description { + t.Fatalf("source description missing: %q", f.topic) + } + description = "new description" + f.apply(t, d) + if f.topic != description { + t.Fatal("owned description failed to update") + } + writes := f.writes + f.topic = "user choice" + description = "later source" + f.apply(t, d) + if f.topic != "user choice" || f.writes != writes { + t.Fatal("source description overwrote user topic") + } +} + +func TestSourceTopicDirectDescriptionDoesNotSeedTopic(t *testing.T) { + f := newTopicFixture(t) + d := topicChannel() + description := "not a projected DM topic" + d.Description = &description + d.Structure.Kind = "direct" + d.Structure.PeerUserID = foundationInt64(42) + if err := f.ic.reconcileSourceTopic(t.Context(), f.portal, d); err != nil { + t.Fatal(err) + } + if f.topic != "" || f.matrixCalls != 0 { + t.Fatal("direct description unexpectedly became a generated topic") + } +} + +func TestSourceTopicBacklinkLookupFailureProperty(t *testing.T) { + f := newTopicFixture(t) + seedTopicParent(t, f, 3, "$root3") + f.apply(t, topicChild()) + hegel.Test(t, func(ht *hegel.T) { + number := hegel.Draw(ht, hegel.Integers[int64](math.MinInt64, math.MaxInt64)) + failed := hegel.Draw(ht, hegel.Booleans()) + d := topicChild() + description := fmt.Sprintf("source-%d", number) + d.Description = &description + before, calls := f.topic, f.matrixCalls + if failed { + breakTopicMessageReads(t, f) + } + f.apply(t, d) + if failed { + restoreTopicMessageReads(t, f) + } + if failed { + if f.topic != before || f.matrixCalls != calls { + ht.Fatalf("failed mapping read published topic%q=>%q calls%d=>%d", before, f.topic, calls, f.matrixCalls) + } + } else if !strings.HasPrefix(f.topic, description+"\n\n") || !strings.Contains(f.topic, "$root3") { + ht.Fatalf("valid source change lost description/backlink: %q", f.topic) + } + }, hegel.WithTestCases(100), hegel.WithDatabase(t.TempDir())) +} diff --git a/pkg/connector/client.go b/pkg/connector/client.go index 5853447..788f055 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -25,6 +25,7 @@ import ( "maunium.net/go/mautrix/bridgev2/status" "maunium.net/go/mautrix/event" + "github.com/inline-chat/matrix-inline/pkg/msgconv" "github.com/inline-chat/matrix-inline/pkg/sidecar" ) @@ -75,22 +76,30 @@ type InlineClient struct { AccountID string Sidecar *sidecar.Client - mu sync.RWMutex - loggedIn bool - users map[int64]sidecar.UserRecord - dialogs map[int64]sidecar.DialogRecord - details map[int64]struct{} - history map[int64]int64 - historyLoaded map[int64]struct{} - storeNamespace string - lastSidecarSequence uint64 - sidecarEventGeneration string - bridgeStateVersion uint32 - bridgeRecoveryCursor int64 - dialogSyncCursor string - hiddenDialogsDefault hiddenDialogsPolicy - hiddenDialogsOverride hiddenDialogsPolicy - metadataSaveMu sync.Mutex + mu sync.RWMutex + loggedIn bool + users map[int64]sidecar.UserRecord + dialogs map[int64]sidecar.DialogRecord + spaces map[int64]sidecar.SpaceRecord + spaceAccess map[int64]string + spaceAccessComplete bool + workspaceRevision uint64 + workspaceProjectedRevision uint64 + details map[int64]struct{} + history map[int64]int64 + historyLoaded map[int64]struct{} + storeNamespace string + lastSidecarSequence uint64 + sidecarEventGeneration string + bridgeStateVersion uint32 + bridgeRecoveryCursor int64 + dialogSyncCursor string + topicRetryAfter map[int64]time.Time + hiddenDialogsDefault hiddenDialogsPolicy + hiddenDialogsOverride hiddenDialogsPolicy + channelMuteDefault string + channelMuteOverride string + metadataSaveMu sync.Mutex runCancel context.CancelFunc wg sync.WaitGroup @@ -217,6 +226,23 @@ func (ic *InlineClient) GetUserID() networkid.UserID { } func (ic *InlineClient) GetChatInfo(ctx context.Context, portal *bridgev2.Portal) (*bridgev2.ChatInfo, error) { + if err := ic.guardMigrationPresentation(ctx, portal); err != nil { + return nil, err + } + if strings.HasPrefix(string(portal.ID), "space:") { + n, err := strconv.ParseInt(strings.TrimPrefix(string(portal.ID), "space:"), 10, 64) + if err != nil || n <= 0 { + return nil, errors.New("invalid workspace portal") + } + ic.mu.RLock() + space, ok := ic.spaces[n] + access := ic.spaceAccess[n] + ic.mu.RUnlock() + if !ok || access != "accessible" { + return nil, errors.New("workspace access unavailable") + } + return ic.workspaceChatInfo(space), nil + } chatID, err := chatIDFromPortal(portal) if err != nil { return nil, err @@ -252,6 +278,15 @@ func (ic *InlineClient) GetUserInfo(ctx context.Context, ghost *bridgev2.Ghost) func (ic *InlineClient) GetCapabilities(ctx context.Context, portal *bridgev2.Portal) *event.RoomFeatures { features := &event.RoomFeatures{ + ID: "inline-rich-v1", + Formatting: event.FormattingFeatureMap{ + event.FmtBold: event.CapLevelFullySupported, + event.FmtItalic: event.CapLevelFullySupported, + event.FmtInlineCode: event.CapLevelFullySupported, + event.FmtCodeBlock: event.CapLevelFullySupported, + event.FmtInlineLink: event.CapLevelFullySupported, + event.FmtUserLink: event.CapLevelPartialSupport, + }, MaxTextLength: 5000, File: event.FileFeatureMap{ event.MsgImage: { @@ -288,38 +323,32 @@ func (ic *InlineClient) GetCapabilities(ctx context.Context, portal *bridgev2.Po }, }, Reply: event.CapLevelPartialSupport, - Edit: event.CapLevelFullySupported, + Edit: event.CapLevelPartialSupport, Delete: event.CapLevelFullySupported, Reaction: event.CapLevelFullySupported, ReactionCount: 1, - CustomEmojiReactions: true, + CustomEmojiReactions: false, ReadReceipts: true, TypingNotifications: true, MarkAsUnread: true, } - if portal != nil && portal.OtherUserID == "" { - features.State = event.StateFeatureMap{ - event.StateRoomName.Type: {Level: event.CapLevelFullySupported}, - } - features.MemberActions = event.MemberFeatureMap{ - event.MemberActionInvite: event.CapLevelFullySupported, - event.MemberActionKick: event.CapLevelFullySupported, - event.MemberActionRevokeInvite: event.CapLevelFullySupported, - } - features.DeleteChat = true - features.DeleteChatForEveryone = true - } return features } func (ic *InlineClient) ResolveIdentifier(ctx context.Context, identifier string, createChat bool) (*bridgev2.ResolveIdentifierResponse, error) { - userID, err := parseInlineUserID(identifier) + directory, err := ic.loadDirectory(ctx) + if err != nil { + return nil, err + } + user, err := directory.resolve(identifier) if err != nil { return nil, err } + userID := user.Record.UserID + ic.cacheUsers([]sidecar.UserRecord{user.Record}) resp := &bridgev2.ResolveIdentifierResponse{ UserID: makeUserID(strconv.FormatInt(userID, 10)), - UserInfo: ic.userInfoForID(userID), + UserInfo: userInfoFromRecord(user.Record), } if createChat { chat, err := ic.createDMWithUserID(ctx, userID) @@ -335,45 +364,44 @@ func (ic *InlineClient) CreateChatWithGhost(ctx context.Context, ghost *bridgev2 if ghost == nil { return nil, fmt.Errorf("missing Inline ghost") } - userID, err := parseInlineUserID(string(ghost.ID)) + resolved, err := ic.ResolveIdentifier(ctx, string(ghost.ID), true) if err != nil { + if _, valid := directoryPositiveID(string(ghost.ID)); valid && (errors.Is(err, errDirectoryNotFound) || errors.Is(err, errDirectoryIncomplete)) { + return nil, fmt.Errorf("%w: %w", bridgev2.ErrResolveIdentifierTryNext, err) + } return nil, err } - return ic.createDMWithUserID(ctx, userID) + return resolved.Chat, nil } func (ic *InlineClient) CreateGroup(ctx context.Context, params *bridgev2.GroupCreateParams) (*bridgev2.CreateChatResponse, error) { - if params == nil { - return nil, fmt.Errorf("missing Inline group create params") + directory, err := ic.loadDirectory(ctx) + if err != nil { + return nil, err } - var title *string - if params.Name != nil { - title = optionalString(params.Name.Name) + request, participantIDs, err := validateThreadCreation(params, ic.UserLogin.ID, directory) + if err != nil { + return nil, err } - var description *string - if params.Topic != nil { - description = optionalString(params.Topic.Topic) + if ic.Sidecar.SessionNamespace != directory.namespace { + return nil, fmt.Errorf("Inline login changed during creation") } - participants := make([]sidecar.ChatCreateParticipant, 0, len(params.Participants)) - participantIDs := make([]networkid.UserID, 0, len(params.Participants)+1) - for _, participant := range params.Participants { - userID, err := parseInlineUserID(string(participant)) - if err != nil { - return nil, fmt.Errorf("invalid Inline participant %q: %w", participant, err) - } - participants = append(participants, sidecar.ChatCreateParticipant{UserID: userID}) - participantIDs = append(participantIDs, makeUserID(strconv.FormatInt(userID, 10))) - } - chat, err := ic.Sidecar.CreateThread(ctx, sidecar.CreateThreadRequest{ - Title: title, - Description: description, - IsPublic: false, - Participants: participants, - }) + chat, err := ic.Sidecar.CreateThread(ctx, request) if err != nil { return nil, fmt.Errorf("Inline sidecar create thread failed: %w", err) } - return ic.createChatResponse(ctx, chat, ic.chatInfoForCreatedChat(ctx, chat, participantIDs)), nil + for _, participant := range request.Participants { + ic.cacheUsers([]sidecar.UserRecord{directory.users[participant.UserID].Record}) + } + info := ic.chatInfoForCreatedChat(ctx, chat, participantIDs) + kind := database.RoomTypeDefault + info.Type = &kind + info.Members.OtherUserID = "" + info.Members.IsFull = false + if chat != nil { + ic.rememberDialog(sidecar.DialogRecord{ChatID: chat.ChatID, Title: chat.Title, SpaceID: request.SpaceID, Structure: &sidecar.DialogStructure{Kind: "channel", SpaceID: request.SpaceID}}) + } + return ic.createChatResponse(ctx, chat, info), nil } func (ic *InlineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.MatrixMessage) (*bridgev2.MatrixMessageResponse, error) { @@ -385,6 +413,9 @@ func (ic *InlineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.M WithSendNotice(true). WithErrorReason(event.MessageStatusUnsupported) } + if err := validateInlineReplyTarget(msg, chatID); err != nil { + return nil, bridgev2.WrapErrorInStatus(err).WithErrorAsMessage().WithIsCertain(true).WithSendNotice(true).WithErrorReason(event.MessageStatusUnsupported) + } switch effectiveMatrixMessageType(msg.Content) { case event.MsgText: @@ -397,9 +428,11 @@ func (ic *InlineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.M } func (ic *InlineClient) handleMatrixTextMessage(ctx context.Context, msg *bridgev2.MatrixMessage, chatID int64) (*bridgev2.MatrixMessageResponse, error) { + text, entities := msgconv.FromMatrix(msg.Content, ic.outgoingMentionResolver(msg.Content)) request := sidecar.SendTextRequest{ Peer: sidecar.ChatPeer(chatID), - Text: msg.Content.Body, + Text: text, + Entities: entities, ExternalID: matrixExternalID(msg), RandomID: matrixRandomID(msg), } @@ -445,13 +478,20 @@ func (ic *InlineClient) handleMatrixMediaMessage(ctx context.Context, msg *bridg WithErrorReason(event.MessageStatusUnsupported) } + captionContent := *msg.Content + captionContent.Body = msg.Content.GetCaption() + if captionContent.Body == "" { + captionContent.Format, captionContent.FormattedBody = "", "" + } + caption, entities := msgconv.FromMatrix(&captionContent, ic.outgoingMentionResolver(msg.Content)) request := sidecar.UploadRequest{ Peer: sidecar.ChatPeer(chatID), Kind: sidecarUploadKindForMatrix(msg.Content), FileName: optionalString(sanitizeMatrixFileName(msg.Content.GetFileName())), MimeType: optionalString(matrixMediaMimeType(msg.Content)), SizeBytes: uint64Ptr(uint64(len(data))), - Caption: optionalString(msg.Content.GetCaption()), + Caption: optionalString(caption), + Entities: entities, ExternalID: matrixExternalID(msg), RandomID: matrixRandomID(msg), } @@ -528,20 +568,32 @@ func (ic *InlineClient) matrixSendResponse(msg *bridgev2.MatrixMessage, chatID i msg.AddPendingToIgnore(txnID) } + var metadata *MessageMetadata + if msg.Content != nil && msg.Content.MsgType.IsMedia() { + media := *msg.Content + metadata = &MessageMetadata{InlineMedia: &media, MediaHandled: true} + } return &bridgev2.MatrixMessageResponse{ DB: &database.Message{ ID: makeMessageID(chatID, *mutation.MessageID), SenderID: ic.GetUserID(), Timestamp: time.Now(), + Metadata: metadata, }, RemovePending: txnID, }, nil } func (ic *InlineClient) HandleMatrixEdit(ctx context.Context, msg *bridgev2.MatrixEdit) error { - if msg.Content.MsgType != event.MsgText { + if msg == nil || msg.Content == nil || msg.EditTarget == nil || msg.Portal == nil || msg.Portal.Portal == nil || msg.EditTarget.Room != msg.Portal.PortalKey { + return fmt.Errorf("Inline edit target must belong to this conversation and login: %w", bridgev2.ErrUnsupportedMessageType) + } + if msg.Content.MsgType != event.MsgText && !msg.Content.MsgType.IsMedia() { return bridgev2.ErrUnsupportedMessageType } + if msg.Content.MsgType.IsMedia() && !matrixCaptionEditAllowed(msg.EditTarget, msg.Content) { + return fmt.Errorf("Inline media replacement or unverified legacy caption edit is unsupported: %w", bridgev2.ErrUnsupportedMessageType) + } chatID, messageID, err := parseMessageID(msg.EditTarget.ID) if err != nil { return bridgev2.WrapErrorInStatus(fmt.Errorf("Inline edit target is invalid: %w", err)). @@ -550,10 +602,23 @@ func (ic *InlineClient) HandleMatrixEdit(ctx context.Context, msg *bridgev2.Matr WithSendNotice(true). WithErrorReason(event.MessageStatusUnsupported) } + portalChat, portalErr := chatIDFromPortal(msg.Portal) + if portalErr != nil || chatID != portalChat || messageID <= 0 { + return fmt.Errorf("Inline edit target must be a positive message in this conversation: %w", bridgev2.ErrUnsupportedMessageType) + } + content := *msg.Content + if content.MsgType.IsMedia() { + content.Body = content.GetCaption() + if content.Body == "" { + content.Format, content.FormattedBody = "", "" + } + } + text, entities := msgconv.FromMatrix(&content, ic.outgoingMentionResolver(msg.Content)) if err := ic.Sidecar.EditMessage(ctx, sidecar.EditMessageRequest{ ChatID: chatID, MessageID: messageID, - Text: msg.Content.Body, + Text: text, + Entities: entities, ExternalID: matrixEventExternalID(msg.Event), }); err != nil { return bridgev2.WrapErrorInStatus(fmt.Errorf("Inline sidecar edit failed: %w", err)). @@ -591,7 +656,7 @@ func (ic *InlineClient) HandleMatrixMessageRemove(ctx context.Context, msg *brid func (ic *InlineClient) PreHandleMatrixReaction(ctx context.Context, msg *bridgev2.MatrixReaction) (bridgev2.MatrixReactionPreResponse, error) { reaction := strings.TrimSpace(msg.Content.RelatesTo.Key) - if reaction == "" { + if reaction == "" || strings.Contains(reaction, "://") { return bridgev2.MatrixReactionPreResponse{}, bridgev2.WrapErrorInStatus(fmt.Errorf("Inline reactions require a non-empty emoji")). WithErrorAsMessage(). WithIsCertain(true). @@ -673,13 +738,14 @@ func (ic *InlineClient) HandleMatrixReadReceipt(ctx context.Context, msg *bridge if err != nil { return fmt.Errorf("Inline portal ID %q is not a numeric chat ID yet: %w", msg.Portal.ID, err) } - var maxMessageID *int64 - if msg.ExactMessage != nil { - _, parsedMessageID, err := parseMessageID(msg.ExactMessage.ID) - if err == nil { - maxMessageID = &parsedMessageID - } + if msg.ExactMessage == nil { + return errors.New("Inline read receipt has no exact source target") } + targetChat, targetID, err := parseMessageID(msg.ExactMessage.ID) + if err != nil || targetChat != chatID || targetID <= 0 { + return errors.New("Inline read receipt target does not match this chat") + } + maxMessageID := &targetID if err := ic.Sidecar.Read(ctx, sidecar.ReadRequest{ ChatID: chatID, MaxMessageID: maxMessageID, @@ -772,10 +838,10 @@ func (ic *InlineClient) HandleMatrixMembership(ctx context.Context, msg *bridgev UserID: userID, }) case bridgev2.Kick, bridgev2.RevokeInvite: - err = ic.Sidecar.RemoveChatParticipant(ctx, sidecar.RemoveChatParticipantRequest{ - ChatID: chatID, - UserID: userID, - }) + // Removing a direct participant does not prove group or public-space + // access was removed. Do not report a Matrix kick as successful until + // the source can establish the target's effective access afterward. + return nil, fmt.Errorf("%w: Inline cannot verify that all access was removed", bridgev2.ErrMembershipNotSupported) case bridgev2.AcceptInvite, bridgev2.ProfileChange: return &bridgev2.MatrixMembershipResult{}, nil default: @@ -1122,8 +1188,18 @@ func (ic *InlineClient) persistHiddenDialogsOverride(ctx context.Context, policy } func (ic *InlineClient) syncDialogs(ctx context.Context, refreshLive bool) error { + if err := ic.refreshWorkspaceSnapshot(ctx); err != nil { + // A failed observation is not proof that source access was removed. + // Keep the complete prior snapshot and its presentation revision. + if ic.UserLogin != nil && ic.UserLogin.Bridge != nil { + ic.UserLogin.Bridge.Log.Warn().Err(err).Msg("Workspace refresh failed; retaining last known snapshot") + } + } + limit := uint32(startupDialogPageLimit) ic.mu.RLock() + workspaceRevision := ic.workspaceRevision + workspaceChanged := workspaceRevision != ic.workspaceProjectedRevision cursor := ic.dialogSyncCursor ic.mu.RUnlock() seenCursors := make(map[string]struct{}) @@ -1162,7 +1238,7 @@ func (ic *InlineClient) syncDialogs(ctx context.Context, refreshLive bool) error if !ic.shouldProjectDialog(dialog) { continue } - if changed { + if changed || workspaceChanged { ic.queueDialogResync(ctx, dialog) resyncsQueued++ } @@ -1175,6 +1251,9 @@ func (ic *InlineClient) syncDialogs(ctx context.Context, refreshLive bool) error } if page.NextCursor == "" { + ic.mu.Lock() + ic.workspaceProjectedRevision = workspaceRevision + ic.mu.Unlock() if err := ic.persistDialogSyncCursor(ctx, ""); err != nil { return err } @@ -1291,12 +1370,13 @@ func (ic *InlineClient) syncDialogDetails(ctx context.Context, dialog sidecar.Di } ic.UserLogin.Bridge.QueueRemoteEvent(ic.UserLogin, &simplevent.ChatResync{ EventMeta: simplevent.EventMeta{ - Type: bridgev2.RemoteEventChatResync, - PortalKey: ic.portalKeyForChat(dialog.ChatID), - Timestamp: time.Now(), - CreatePortal: true, + Type: bridgev2.RemoteEventChatResync, + PostHandleFunc: ic.projectionPostHandler, + PortalKey: ic.portalKeyForChat(dialog.ChatID), + Timestamp: time.Now(), + CreatePortal: true, }, - ChatInfo: info, + GetChatInfoFunc: ic.migrationGatedInfoProvider(info), }) return nil } @@ -1305,30 +1385,15 @@ func (ic *InlineClient) syncRecentHistory(ctx context.Context, dialog sidecar.Di if !ic.shouldProjectDialog(dialog) || !ic.needsHistoryDelivery(dialog) { return nil } - request := historyRequestForDelivery(dialog, ic.historyCheckpoint(dialog.ChatID), startupHistoryLimit) + checkpoint := ic.historyCheckpoint(dialog.ChatID) + request := historyRequestForDelivery(dialog, checkpoint, startupHistoryLimit) page, err := ic.Sidecar.History(ctx, request) if err != nil { return err } ic.cacheUsers(page.Users) - sort.SliceStable(page.Messages, func(i, j int) bool { - left, right := page.Messages[i], page.Messages[j] - if left.Timestamp != right.Timestamp { - return left.Timestamp < right.Timestamp - } - return left.MessageID < right.MessageID - }) - - for _, msg := range page.Messages { - if err := ctx.Err(); err != nil { - return err - } - if err := eventHandlingResultError("Inline history message", ic.queueInlineMessage(ctx, msg)); err != nil { - return err - } - } - return nil + return ic.queueHistoricalPage(ctx, dialog.ChatID, page.Messages, &checkpoint) } func historyRequestForDelivery(dialog sidecar.DialogRecord, checkpoint int64, limit uint32) sidecar.HistoryRequest { @@ -1336,9 +1401,9 @@ func historyRequestForDelivery(dialog sidecar.DialogRecord, checkpoint int64, li ChatID: dialog.ChatID, Limit: &limit, } - if checkpoint > 0 { - request.AfterMessageID = &checkpoint - } + // An explicit zero cursor selects the oldest source IDs, rather than a + // recent tail page that cannot establish coverage from an empty frontier. + request.AfterMessageID = &checkpoint return request } @@ -1395,22 +1460,11 @@ func (ic *InlineClient) loadHistoryCheckpoint(ctx context.Context, chatID int64) if err != nil { return 0, fmt.Errorf("load Inline portal history checkpoint: %w", err) } - if metadata, ok := portal.Metadata.(*PortalMetadata); ok && metadata != nil && metadata.HistoryCheckpoint > 0 { - checkpoint = metadata.HistoryCheckpoint - ic.rememberHistoryDelivered(chatID, checkpoint) - return checkpoint, nil - } - - messages, err := ic.UserLogin.Bridge.DB.Message.GetLastNInPortal(ctx, ic.portalKeyForChat(chatID), 16) - if err != nil { - return 0, err - } - checkpoint = maxInlineMessageIDForChat(messages, chatID) - if checkpoint > 0 { - if err := ic.persistHistoryCheckpoint(ctx, chatID, checkpoint); err != nil { - return 0, err - } + if metadata, ok := portal.Metadata.(*PortalMetadata); ok && metadata != nil { + checkpoint = metadata.HistoricalCheckpoint } + // A mapped live maximum does not prove coverage of lower historical IDs. + // Legacy rows restart from zero and deduplicate through persisted mappings. ic.mu.Lock() if ic.history == nil { @@ -1509,18 +1563,38 @@ func rateLimitRetryDelay(err error, fallback time.Duration) time.Duration { func (ic *InlineClient) FetchMessages(ctx context.Context, params bridgev2.FetchMessagesParams) (*bridgev2.FetchMessagesResponse, error) { if params.ThreadRoot != "" { - return &bridgev2.FetchMessagesResponse{HasMore: false}, nil + return nil, fmt.Errorf("Inline Matrix thread history is unsupported; use the separate child conversation: %w", bridgev2.ErrUnsupportedMessageType) } - request, err := historyRequestForBackfill(params) + chat, err := chatIDFromPortal(params.Portal) if err != nil { return nil, err } - page, err := ic.Sidecar.History(ctx, request) - if err != nil { - return nil, fmt.Errorf("failed to fetch Inline history: %w", err) + if err = ic.readDesireBeforeDelivery(ctx, chat); err != nil { + return nil, err + } + var page *sidecar.HistoryPage + if bundled, ok := params.BundledData.(historicalPage); ok { + if bundled.ChatID != chat { + return nil, errors.New("historical bundle belongs to another chat") + } + page = &sidecar.HistoryPage{Messages: bundled.Records} + } else { + request, err := historyRequestForBackfill(params) + if err != nil { + return nil, err + } + page, err = ic.Sidecar.History(ctx, request) + if err != nil { + return nil, fmt.Errorf("failed to fetch Inline history: %w", err) + } } ic.cacheUsers(page.Users) + for _, record := range page.Messages { + if record.ChatID != chat || record.MessageID <= 0 { + return nil, errors.New("history record belongs to another chat") + } + } messages, cursor := ic.backfillMessagesFromHistory(ctx, params.Portal, page.Messages) hasMore := page.HasMore && cursor != "" @@ -1529,7 +1603,11 @@ func (ic *InlineClient) FetchMessages(ctx context.Context, params bridgev2.Fetch Cursor: cursor, HasMore: hasMore, Forward: params.Forward, - MarkRead: true, + MarkRead: false, + CompleteCallback: func() { + _ = ic.persistHistoricalMappings(ctx, params.Portal, page.Messages) + ic.reconcileProjection(ctx, params.Portal) + }, }, nil } @@ -1635,7 +1713,7 @@ func (ic *InlineClient) backfillMessagesFromHistory(ctx context.Context, portal continue } } - converted, err := convertInlineMessage(ctx, portal, intent, record) + converted, err := ic.convertInlineMessage(ctx, portal, intent, record) if err != nil { ic.logBackfillConversionError(err, record) continue @@ -1649,6 +1727,7 @@ func (ic *InlineClient) backfillMessagesFromHistory(ctx context.Context, portal ID: messageID, TxnID: transactionIDForMessage(record), Timestamp: inlineTimestamp(record.Timestamp), + StreamOrder: record.MessageID, }) } return backfillMessages, cursor @@ -1889,6 +1968,7 @@ func (ic *InlineClient) reconcileAccountDeletes(ctx context.Context) error { if err != nil { return fmt.Errorf("fetch Inline account reconciliation state: %w", err) } + ic.cacheWorkspaceSnapshot(accountState) for _, chatID := range accountState.DeletedChatIDs { if err := eventHandlingResultError("reconciled chat delete", ic.queueChatDelete(chatID)); err != nil { return err @@ -1933,12 +2013,13 @@ func (ic *InlineClient) reconcileDialogState(ctx context.Context, dialog sidecar } result := ic.UserLogin.Bridge.QueueRemoteEvent(ic.UserLogin, &simplevent.ChatResync{ EventMeta: simplevent.EventMeta{ - Type: bridgev2.RemoteEventChatResync, - PortalKey: ic.portalKeyForChat(dialog.ChatID), - Timestamp: time.Now(), - CreatePortal: true, + Type: bridgev2.RemoteEventChatResync, + PostHandleFunc: ic.projectionPostHandler, + PortalKey: ic.portalKeyForChat(dialog.ChatID), + Timestamp: time.Now(), + CreatePortal: true, }, - ChatInfo: info, + GetChatInfoFunc: ic.migrationGatedInfoProvider(info), }) if err := eventHandlingResultError("reconciled chat resync", result); err != nil { return err @@ -2025,10 +2106,8 @@ func (ic *InlineClient) reconcileCachedHistory(ctx context.Context, chatID int64 if err != nil { return fmt.Errorf("validate cached Inline history for chat %d: %w", chatID, err) } - for _, message := range page.Messages { - if err := eventHandlingResultError("reconciled message upsert", ic.queueInlineMessage(ctx, message)); err != nil { - return err - } + if err := ic.queueHistoricalPage(ctx, chatID, page.Messages, nil); err != nil { + return err } after = pageMax if !page.HasMore { @@ -2125,34 +2204,17 @@ func (ic *InlineClient) queueReadState(state *sidecar.ReadStateRecord) error { if state == nil { return nil } - sender := bridgev2.EventSender{ - Sender: makeUserID(ic.AccountID), - IsFromMe: true, - } - if state.ReadMaxID != nil && *state.ReadMaxID > 0 { - result := ic.UserLogin.Bridge.QueueRemoteEvent(ic.UserLogin, &simplevent.Receipt{ - EventMeta: simplevent.EventMeta{ - Type: bridgev2.RemoteEventReadReceipt, - PortalKey: ic.portalKeyForChat(state.ChatID), - Sender: sender, - Timestamp: time.Now(), - }, - LastTarget: makeMessageID(state.ChatID, *state.ReadMaxID), - }) - if err := eventHandlingResultError("read receipt", result); err != nil { - return err - } + ctx := context.Background() + if err := ic.storeReadDesire(ctx, state); err != nil { + return err } - result := ic.UserLogin.Bridge.QueueRemoteEvent(ic.UserLogin, &simplevent.MarkUnread{ - EventMeta: simplevent.EventMeta{ - Type: bridgev2.RemoteEventMarkUnread, - PortalKey: ic.portalKeyForChat(state.ChatID), - Sender: sender, - Timestamp: time.Now(), - }, - Unread: state.MarkedUnread, + // A ChatResync is a serialized wakeup, never an ordinary framework receipt: + // MarkRead/MarkUnread can clear manual unread with unmarked account data. + result := ic.UserLogin.Bridge.QueueRemoteEvent(ic.UserLogin, &simplevent.ChatResync{ + EventMeta: simplevent.EventMeta{Type: bridgev2.RemoteEventChatResync, PortalKey: ic.portalKeyForChat(state.ChatID), PostHandleFunc: ic.projectionPostHandler}, + GetChatInfoFunc: ic.migrationGatedInfoProvider(&bridgev2.ChatInfo{}), }) - return eventHandlingResultError("mark unread", result) + return eventHandlingResultError("pending read projection", result) } func (ic *InlineClient) handleSequencedSidecarEvent(ctx context.Context, envelope *sidecar.EventEnvelope) error { @@ -2500,15 +2562,19 @@ func (ic *InlineClient) queueDialogResync(ctx context.Context, dialog sidecar.Di if !ic.shouldProjectDialog(dialog) { return bridgev2.EventHandlingResultIgnored } + if err := ic.ensureWorkspace(ctx, dialog); err != nil { + return bridgev2.EventHandlingResult{Error: err} + } chatInfo := ic.chatInfoForDialog(ctx, dialog, dialogName(dialog)) return ic.UserLogin.Bridge.QueueRemoteEvent(ic.UserLogin, &simplevent.ChatResync{ EventMeta: simplevent.EventMeta{ - Type: bridgev2.RemoteEventChatResync, - PortalKey: ic.portalKeyForChat(dialog.ChatID), - Timestamp: time.Now(), - CreatePortal: true, + Type: bridgev2.RemoteEventChatResync, + PostHandleFunc: ic.projectionPostHandler, + PortalKey: ic.portalKeyForChat(dialog.ChatID), + Timestamp: time.Now(), + CreatePortal: true, }, - ChatInfo: chatInfo, + GetChatInfoFunc: ic.migrationGatedInfoProvider(chatInfo), }) } @@ -2543,6 +2609,9 @@ func (ic *InlineClient) queueChatResyncByID(ctx context.Context, chatID int64) b } func (ic *InlineClient) queueSpaceDialogResync(ctx context.Context, spaceID int64) error { + if err := ic.refreshWorkspaceSnapshot(ctx); err != nil { + return err + } ic.mu.RLock() dialogs := make([]sidecar.DialogRecord, 0) for _, dialog := range ic.dialogs { @@ -2601,6 +2670,15 @@ func (ic *InlineClient) rememberDialog(dialog sidecar.DialogRecord) bool { ic.dialogs = make(map[int64]sidecar.DialogRecord) } previous, ok := ic.dialogs[dialog.ChatID] + // A partial observation cannot erase a previously complete structure. A + // conflicting complete observation remains unknown and must be enriched. + if dialog.Structure == nil && ok && previous.Structure != nil { + dialog.Structure = previous.Structure + dialog.PeerUserID = previous.PeerUserID + dialog.SpaceID = previous.SpaceID + dialog.ParentChatID = previous.ParentChatID + dialog.ParentMessageID = previous.ParentMessageID + } ic.dialogs[dialog.ChatID] = dialog changed := !ok || !sameDialogRecord(previous, dialog) if !ok || !int64PtrEqual(previous.PeerUserID, dialog.PeerUserID) { @@ -2616,24 +2694,10 @@ func (ic *InlineClient) cachedDialog(chatID int64) (sidecar.DialogRecord, bool) return dialog, ok } -func sameDialogRecord(left, right sidecar.DialogRecord) bool { - return left.ChatID == right.ChatID && - int64PtrEqual(left.PeerUserID, right.PeerUserID) && - stringPtrEqual(left.Title, right.Title) && - int64PtrEqual(left.LastMessageID, right.LastMessageID) && - int64PtrEqual(left.SyncedThroughMessageID, right.SyncedThroughMessageID) && - boolPtrEqual(left.ChatListHidden, right.ChatListHidden) && - uint32PtrEqual(left.UnreadCount, right.UnreadCount) -} - func boolPtrEqual(left, right *bool) bool { return (left == nil && right == nil) || (left != nil && right != nil && *left == *right) } -func isDMDialog(dialog sidecar.DialogRecord) bool { - return dialog.PeerUserID != nil && *dialog.PeerUserID > 0 -} - func (ic *InlineClient) needsDialogDetailsSync(chatID int64) bool { if chatID <= 0 { return false @@ -2673,6 +2737,14 @@ func (ic *InlineClient) cachedUser(userID int64) (sidecar.UserRecord, bool) { } func (ic *InlineClient) chatInfoForChat(ctx context.Context, chatID int64, name *string) (*bridgev2.ChatInfo, error) { + if d, ok := ic.cachedDialog(chatID); !ok || classifyDialog(d) == roomUnknown { + bounded, cancel := context.WithTimeout(ctx, 5*time.Second) + state, err := ic.Sidecar.ChatState(bounded, sidecar.ChatStateRequest{ChatID: chatID}) + cancel() + if err == nil && state.Dialog != nil { + ic.rememberDialog(*state.Dialog) + } + } if dialog, ok := ic.cachedDialog(chatID); ok && isDMDialog(dialog) { return ic.chatInfoForDialog(ctx, dialog, name), nil } @@ -2681,31 +2753,13 @@ func (ic *InlineClient) chatInfoForChat(ctx context.Context, chatID int64, name return nil, fmt.Errorf("failed to fetch Inline chat participants: %w", err) } ic.cacheUsers(page.Users) - roomType := database.RoomTypeGroupDM - return &bridgev2.ChatInfo{ - Type: &roomType, - Name: name, - Members: ic.chatMemberListFromParticipants(ctx, page), - CanBackfill: true, - }, nil -} - -func (ic *InlineClient) chatInfoForDialog(ctx context.Context, dialog sidecar.DialogRecord, name *string) *bridgev2.ChatInfo { - info := &bridgev2.ChatInfo{ - Name: name, - CanBackfill: true, - } - if dialog.PeerUserID != nil && *dialog.PeerUserID > 0 { - roomType := database.RoomTypeDM - info.Type = &roomType - info.Name = nil - info.Members = ic.chatMemberListForDM(ctx, *dialog.PeerUserID) - } else { - roomType := database.RoomTypeGroupDM - info.Type = &roomType - info.Members = ic.chatMemberListForKnownSelf(ctx) + dialog, _ := ic.cachedDialog(chatID) + info := ic.chatInfoForDialog(ctx, dialog, name) + info.Members = ic.chatMemberListFromParticipants(ctx, page) + if info.Members != nil { + info.Members.OtherUserID = "" } - return info + return info, nil } func (ic *InlineClient) chatMemberListFromParticipants(ctx context.Context, page *sidecar.ChatParticipantsPage) *bridgev2.ChatMemberList { @@ -2890,17 +2944,23 @@ func eventHandlingResultError(kind string, result bridgev2.EventHandlingResult) func (ic *InlineClient) queueInlineMessage(ctx context.Context, msg sidecar.MessageRecord) bridgev2.EventHandlingResult { result := ic.UserLogin.Bridge.QueueRemoteEvent(ic.UserLogin, ic.remoteMessageEvent(msg)) if historyDeliveryAccepted(result) { - if err := ic.persistHistoryCheckpoint(ctx, msg.ChatID, msg.MessageID); err != nil { + row, err := ic.UserLogin.Bridge.DB.Message.GetFirstPartByID(ctx, ic.portalKeyForChat(msg.ChatID).Receiver, makeMessageID(msg.ChatID, msg.MessageID)) + if err != nil || row == nil || row.MXID == "" || row.HasFakeMXID() || row.Room != ic.portalKeyForChat(msg.ChatID) { + return bridgev2.EventHandlingResultFailed.WithError(errors.New("live delivery awaits persisted Matrix mapping")) + } + if err := ic.persistDeliveryCheckpoint(ctx, msg.ChatID, msg.MessageID, false); err != nil { result.Success = false result.Error = err return result } - ic.rememberHistoryDelivered(msg.ChatID, msg.MessageID) } return result } func (ic *InlineClient) persistHistoryCheckpoint(ctx context.Context, chatID, messageID int64) error { + return ic.persistDeliveryCheckpoint(ctx, chatID, messageID, true) +} +func (ic *InlineClient) persistDeliveryCheckpoint(ctx context.Context, chatID, messageID int64, historical bool) error { if chatID <= 0 || messageID <= 0 { return errors.New("invalid Inline history checkpoint") } @@ -2912,13 +2972,18 @@ func (ic *InlineClient) persistHistoryCheckpoint(ctx context.Context, chatID, me if !ok || metadata == nil { return errors.New("Inline portal metadata is unavailable") } - if messageID <= metadata.HistoryCheckpoint { + previous := *metadata + if messageID > metadata.HistoryCheckpoint { + metadata.HistoryCheckpoint = messageID + } + if historical && messageID > metadata.HistoricalCheckpoint { + metadata.HistoricalCheckpoint = messageID + } + if *metadata == previous { return nil } - previous := metadata.HistoryCheckpoint - metadata.HistoryCheckpoint = messageID if err := portal.Save(ctx); err != nil { - metadata.HistoryCheckpoint = previous + *metadata = previous return fmt.Errorf("save Inline history checkpoint: %w", err) } return nil @@ -3033,14 +3098,16 @@ func (ic *InlineClient) remoteMessageEvent(msg sidecar.MessageRecord) *simpleven Sender: makeUserID(strconv.FormatInt(msg.SenderID, 10)), IsFromMe: msg.IsOutgoing, }, - Timestamp: inlineTimestamp(msg.Timestamp), + Timestamp: inlineTimestamp(msg.Timestamp), + StreamOrder: msg.MessageID, + PostHandleFunc: ic.projectionPostHandler, }, Data: msg, ID: makeMessageID(msg.ChatID, msg.MessageID), TransactionID: transactionIDForMessage(msg), - ConvertMessageFunc: convertInlineMessage, - HandleExistingFunc: upsertExistingInlineMessage, - ConvertEditFunc: convertInlineMessageEdit, + ConvertMessageFunc: ic.convertInlineMessage, + HandleExistingFunc: ic.upsertExistingInlineMessage, + ConvertEditFunc: ic.convertInlineMessageEdit, TargetMessage: makeMessageID(msg.ChatID, msg.MessageID), } } @@ -3049,11 +3116,18 @@ func upsertExistingInlineMessage(ctx context.Context, portal *bridgev2.Portal, i if len(existing) == 0 { return bridgev2.UpsertResult{ContinueMessageHandling: true}, nil } + if staleInlineRevision(existing[0], data) { + return bridgev2.UpsertResult{}, nil + } fingerprint := inlineMessageFingerprint(data) if metadata, ok := existing[0].Metadata.(*MessageMetadata); ok && - metadata.InlineFingerprint == fingerprint && + metadata != nil && (data.Content.Type != "media" || metadata.MediaHandled) { - return bridgev2.UpsertResult{}, nil + if metadata.InlineFingerprint == fingerprint || + (metadata.InlineFingerprint != "" && emptyInlineMessageMetadata(data.Metadata) && + metadata.InlineFingerprint == legacyInlineMessageFingerprint(data)) { + return bridgev2.UpsertResult{}, nil + } } return bridgev2.UpsertResult{ SubEvents: []bridgev2.RemoteEvent{&simplevent.Message[sidecar.MessageRecord]{ @@ -3074,10 +3148,15 @@ func upsertExistingInlineMessage(ctx context.Context, portal *bridgev2.Portal, i } func convertInlineMessage(ctx context.Context, portal *bridgev2.Portal, intent bridgev2.MatrixAPI, data sidecar.MessageRecord) (*bridgev2.ConvertedMessage, error) { + return convertInlineMessageWithResolver(ctx, portal, intent, data, nil) +} + +func convertInlineMessageWithResolver(ctx context.Context, portal *bridgev2.Portal, intent bridgev2.MatrixAPI, data sidecar.MessageRecord, resolve msgconv.InlineToMatrix) (*bridgev2.ConvertedMessage, error) { converted, err := convertedContentFromInline(ctx, portal, intent, data.Content) if err != nil { return nil, err } + applyInlineRichContent(converted, data, resolve) if data.ReplyToMessageID != nil { converted.ReplyTo = &networkid.MessageOptionalPartID{ MessageID: makeMessageID(data.ChatID, *data.ReplyToMessageID), @@ -3088,16 +3167,24 @@ func convertInlineMessage(ctx context.Context, portal *bridgev2.Portal, intent b } func convertInlineMessageEdit(ctx context.Context, portal *bridgev2.Portal, intent bridgev2.MatrixAPI, existing []*database.Message, data sidecar.MessageRecord) (*bridgev2.ConvertedEdit, error) { + return convertInlineMessageEditWithResolver(ctx, portal, intent, existing, data, nil) +} + +func convertInlineMessageEditWithResolver(ctx context.Context, portal *bridgev2.Portal, intent bridgev2.MatrixAPI, existing []*database.Message, data sidecar.MessageRecord, resolve msgconv.InlineToMatrix) (*bridgev2.ConvertedEdit, error) { if len(existing) == 0 { return nil, bridgev2.ErrTargetMessageNotFound } - converted, err := convertedContentFromInline(ctx, portal, intent, data.Content) + if staleInlineRevision(existing[0], data) { + return &bridgev2.ConvertedEdit{}, nil + } + converted, err := convertedInlineEditContent(ctx, portal, intent, existing[0], data) if err != nil { return nil, err } if len(converted.Parts) == 0 { return &bridgev2.ConvertedEdit{}, nil } + applyInlineRichContent(converted, data, resolve) setInlineMessageFingerprint(converted, data) return &bridgev2.ConvertedEdit{ ModifiedParts: []*bridgev2.ConvertedEditPart{ @@ -3113,9 +3200,16 @@ func setInlineMessageFingerprint(converted *bridgev2.ConvertedMessage, data side metadata := &MessageMetadata{ InlineFingerprint: inlineMessageFingerprint(data), MediaHandled: data.Content.Type == "media", + InlineRevision: data.Metadata.Revision, + InlineEditTime: data.Metadata.EditTimestamp, } for _, part := range converted.Parts { if part != nil { + if data.Content.Type == "media" && part.Content != nil && part.Content.MsgType.IsMedia() { + media := *part.Content + metadata.InlineMedia = &media + metadata.InlineMediaID = inlineMediaIdentity(data.Content) + } part.DBMetadata = metadata } } @@ -3123,15 +3217,17 @@ func setInlineMessageFingerprint(converted *bridgev2.ConvertedMessage, data side func inlineMessageFingerprint(data sidecar.MessageRecord) string { payload, err := json.Marshal(struct { - SenderID int64 `json:"sender_id"` - IsOutgoing bool `json:"is_outgoing"` - Content sidecar.MessageContent `json:"content"` - ReplyToMessageID *int64 `json:"reply_to_message_id,omitempty"` + SenderID int64 `json:"sender_id"` + IsOutgoing bool `json:"is_outgoing"` + Content sidecar.MessageContent `json:"content"` + ReplyToMessageID *int64 `json:"reply_to_message_id,omitempty"` + Metadata sidecar.MessageMetadata `json:"metadata"` }{ SenderID: data.SenderID, IsOutgoing: data.IsOutgoing, Content: data.Content, ReplyToMessageID: data.ReplyToMessageID, + Metadata: data.Metadata, }) if err != nil { return "" @@ -3140,6 +3236,28 @@ func inlineMessageFingerprint(data sidecar.MessageRecord) string { return fmt.Sprintf("%x", digest[:]) } +// Legacy hashes omitted all metadata. Only semantically empty metadata can use +// that equality proof: a first rich correction or revisionless edit must survive. +func emptyInlineMessageMetadata(m sidecar.MessageMetadata) bool { + return m.Mentioned == nil && m.EditTimestamp == nil && m.Revision == nil && + m.SenderIsBot == nil && !m.IsService && len(m.Entities) == 0 && + len(m.Attachments) == 0 && len(m.Actions) == 0 && m.AgentSession == nil +} + +func legacyInlineMessageFingerprint(data sidecar.MessageRecord) string { + payload, err := json.Marshal(struct { + SenderID int64 `json:"sender_id"` + IsOutgoing bool `json:"is_outgoing"` + Content sidecar.MessageContent `json:"content"` + ReplyToMessageID *int64 `json:"reply_to_message_id,omitempty"` + }{data.SenderID, data.IsOutgoing, data.Content, data.ReplyToMessageID}) + if err != nil { + return "" + } + digest := sha256.Sum256(payload) + return fmt.Sprintf("%x", digest[:]) +} + func convertedContentFromInline(ctx context.Context, portal *bridgev2.Portal, intent bridgev2.MatrixAPI, content sidecar.MessageContent) (*bridgev2.ConvertedMessage, error) { msgType := event.MsgText body := content.Text diff --git a/pkg/connector/client_test.go b/pkg/connector/client_test.go index ca4c2ce..9d91a8f 100644 --- a/pkg/connector/client_test.go +++ b/pkg/connector/client_test.go @@ -220,7 +220,11 @@ func TestBackfillCursorUsesMinimumIDNotFirstChronologicalMessage(t *testing.T) { } func TestFetchMessagesUsesSidecarHistory(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rpc/state/chat" { + json.NewEncoder(w).Encode(sidecar.Response{ProtocolVersion: sidecar.ProtocolVersion, Outcome: sidecar.ResponseOutcome{Status: "ok", Data: json.RawMessage(`{"type":"chat_state","data":{}}`)}}) + return + } if r.URL.Path != "/rpc/history" { t.Fatalf("unexpected path %s", r.URL.Path) } @@ -272,7 +276,7 @@ func TestFetchMessagesUsesSidecarHistory(t *testing.T) { if err != nil { t.Fatalf("FetchMessages() error = %v", err) } - if !resp.HasMore || !resp.MarkRead || resp.Forward { + if !resp.HasMore || resp.MarkRead || resp.Forward { t.Fatalf("response flags = has_more:%v mark_read:%v forward:%v", resp.HasMore, resp.MarkRead, resp.Forward) } if resp.Cursor != networkid.PaginationCursor("7/19") { @@ -287,7 +291,7 @@ func TestFetchMessagesUsesSidecarHistory(t *testing.T) { } func TestGetChatInfoFetchesFullInlineMembers(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/rpc/chat/participants" { t.Fatalf("unexpected path %s", r.URL.Path) } @@ -316,6 +320,7 @@ func TestGetChatInfoFetchesFullInlineMembers(t *testing.T) { AccountID: "1", Sidecar: sidecar.NewClient(server.URL), users: make(map[int64]sidecar.UserRecord), + dialogs: map[int64]sidecar.DialogRecord{7: {ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "channel"}}}, } info, err := ic.GetChatInfo(context.Background(), testPortal("7")) if err != nil { @@ -327,8 +332,8 @@ func TestGetChatInfoFetchesFullInlineMembers(t *testing.T) { if info.Members.TotalMemberCount != 2 || len(info.Members.MemberMap) != 2 { t.Fatalf("member count = %d/%d, want 2", info.Members.TotalMemberCount, len(info.Members.MemberMap)) } - if info.Type == nil || *info.Type != database.RoomTypeGroupDM { - t.Fatalf("Type = %#v, want group DM", info.Type) + if info.Type == nil || *info.Type != database.RoomTypeDefault { + t.Fatalf("Type = %#v, want ordinary channel", info.Type) } if info.Members.OtherUserID != "" { t.Fatalf("OtherUserID = %q, want empty for group chat", info.Members.OtherUserID) @@ -344,7 +349,7 @@ func TestGetChatInfoFetchesFullInlineMembers(t *testing.T) { func TestGetChatInfoUsesCachedDMDialogMembers(t *testing.T) { calledParticipants := false - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { calledParticipants = true t.Fatalf("unexpected sidecar call %s", r.URL.Path) })) @@ -360,6 +365,7 @@ func TestGetChatInfoUsesCachedDMDialogMembers(t *testing.T) { 7: { ChatID: 7, PeerUserID: &peerUserID, + Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: &peerUserID}, Title: stringPtr("Ada Lovelace"), }, }, @@ -401,12 +407,13 @@ func TestChatInfoForDialogIncludesSelfForGroupWithoutParticipantFetch(t *testing } info := ic.chatInfoForDialog(context.Background(), sidecar.DialogRecord{ - ChatID: 7, - Title: &title, + ChatID: 7, + Structure: &sidecar.DialogStructure{Kind: "channel"}, + Title: &title, }, &title) - if info.Type == nil || *info.Type != database.RoomTypeGroupDM { - t.Fatalf("Type = %#v, want group DM", info.Type) + if info.Type == nil || *info.Type != database.RoomTypeDefault { + t.Fatalf("Type = %#v, want ordinary channel", info.Type) } if info.Name == nil || *info.Name != title { t.Fatalf("Name = %#v, want %q", info.Name, title) @@ -430,7 +437,16 @@ func TestChatInfoForDialogIncludesSelfForGroupWithoutParticipantFetch(t *testing } func TestResolveIdentifierCreatesDM(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + peerID := int64(42) + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rpc/dialogs" { + writeConnectorSidecarResult(t, w, "dialogs", sidecar.DialogsPage{Dialogs: []sidecar.DialogRecord{{ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: &peerID}}}, Users: []sidecar.UserRecord{{UserID: 42, DisplayName: stringPtr("Ada Lovelace")}}}) + return + } + if r.URL.Path == "/rpc/state/account" { + writeConnectorSidecarResult(t, w, "account_state", sidecar.AccountStateSnapshot{}) + return + } if r.URL.Path != "/rpc/chat/create-dm" { t.Fatalf("unexpected path %s", r.URL.Path) } @@ -451,7 +467,7 @@ func TestResolveIdentifierCreatesDM(t *testing.T) { ic := &InlineClient{ UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ID: networkid.UserLoginID("1")}}, AccountID: "1", - Sidecar: sidecar.NewClient(server.URL), + Sidecar: sidecar.NewClient(server.URL).WithSessionNamespace("fixture"), users: make(map[int64]sidecar.UserRecord), } ic.cacheUsers([]sidecar.UserRecord{{ @@ -474,7 +490,7 @@ func TestResolveIdentifierCreatesDM(t *testing.T) { } func TestHandleMatrixViewingChatMarksInlineChatRead(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/rpc/read" { t.Fatalf("unexpected path %s", r.URL.Path) } @@ -500,34 +516,16 @@ func TestHandleMatrixViewingChatMarksInlineChatRead(t *testing.T) { } } -func TestChatManagementCapabilitiesAreScopedToInlineThreads(t *testing.T) { +func TestChatManagementCapabilitiesRequireSourcePermissionEvidence(t *testing.T) { ic := &InlineClient{} - group := testPortal("7") - features := ic.GetCapabilities(context.Background(), group) - if !features.MarkAsUnread || !features.DeleteChat || !features.DeleteChatForEveryone { - t.Fatalf("group capabilities = %#v, want marked unread and delete-chat support", features) - } - if name := features.State[event.StateRoomName.Type]; name == nil || name.Level != event.CapLevelFullySupported { - t.Fatalf("room-name capability = %#v, want fully supported", name) - } - for _, action := range []event.MemberAction{ - event.MemberActionInvite, - event.MemberActionKick, - event.MemberActionRevokeInvite, - } { - if features.MemberActions[action] != event.CapLevelFullySupported { - t.Fatalf("member action %q = %v, want fully supported", action, features.MemberActions[action]) + for _, p := range []*bridgev2.Portal{testPortal("7"), testPortal("8"), testPortal("space:1")} { + features := ic.GetCapabilities(t.Context(), p) + if features.DeleteChat || features.DeleteChatForEveryone || len(features.State) > 0 || len(features.MemberActions) > 0 || features.CustomEmojiReactions { + t.Fatalf("unproven mutation/custom emoji capabilities advertised: %+v", features) + } + if !features.MarkAsUnread { + t.Fatal("explicit user marked-unread action missing") } - } - - dm := testPortal("8") - dm.OtherUserID = makeUserID("42") - dmFeatures := ic.GetCapabilities(context.Background(), dm) - if !dmFeatures.MarkAsUnread { - t.Fatal("DM marked-unread capability = false, want true") - } - if dmFeatures.DeleteChat || len(dmFeatures.State) != 0 || len(dmFeatures.MemberActions) != 0 { - t.Fatalf("DM chat-management capabilities = %#v, want no thread-only operations", dmFeatures) } } @@ -545,7 +543,7 @@ func TestDialogDisplayNameKeepsEmojiSeparateFromTitle(t *testing.T) { func TestDialogSyncUsesOneLiveSnapshotThenCachedPages(t *testing.T) { liveCalls := 0 cachedCalls := 0 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/rpc/dialogs": liveCalls++ @@ -577,7 +575,7 @@ func TestDialogSyncUsesOneLiveSnapshotThenCachedPages(t *testing.T) { func TestReplayReconciliationUsesOneLiveSnapshotThenCachedPages(t *testing.T) { liveCalls := 0 cachedCalls := 0 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/rpc/dialogs": liveCalls++ @@ -618,7 +616,7 @@ func TestRateLimitRetryDelayUsesTypedHint(t *testing.T) { } func TestMatrixChatManagementCallsSidecar(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/rpc/marked-unread": var request sidecar.SetMarkedUnreadRequest @@ -652,14 +650,6 @@ func TestMatrixChatManagementCallsSidecar(t *testing.T) { if request.ChatID != 7 || request.UserID != 42 { t.Fatalf("add participant request = %#v, want chat 7 user 42", request) } - case "/rpc/chat/participants/remove": - var request sidecar.RemoveChatParticipantRequest - if err := json.NewDecoder(r.Body).Decode(&request); err != nil { - t.Fatalf("decode remove participant request: %v", err) - } - if request.ChatID != 7 || request.UserID != 42 { - t.Fatalf("remove participant request = %#v, want chat 7 user 42", request) - } case "/rpc/chat/delete": var request sidecar.DeleteChatRequest if err := json.NewDecoder(r.Body).Decode(&request); err != nil { @@ -707,7 +697,7 @@ func TestMatrixChatManagementCallsSidecar(t *testing.T) { } ghost := &bridgev2.Ghost{Ghost: &database.Ghost{ID: makeUserID("42")}} - for _, change := range []bridgev2.MembershipChangeType{bridgev2.Invite, bridgev2.Kick} { + for _, change := range []bridgev2.MembershipChangeType{bridgev2.Invite} { if _, err := ic.HandleMatrixMembership(context.Background(), &bridgev2.MatrixMembershipChange{ MatrixRoomMeta: bridgev2.MatrixRoomMeta[*event.MemberEventContent]{ MatrixEventBase: bridgev2.MatrixEventBase[*event.MemberEventContent]{Portal: portal}, @@ -718,6 +708,18 @@ func TestMatrixChatManagementCallsSidecar(t *testing.T) { t.Fatalf("HandleMatrixMembership(%v) error = %v", change, err) } } + for _, change := range []bridgev2.MembershipChangeType{bridgev2.Kick, bridgev2.RevokeInvite} { + _, err := ic.HandleMatrixMembership(context.Background(), &bridgev2.MatrixMembershipChange{ + MatrixRoomMeta: bridgev2.MatrixRoomMeta[*event.MemberEventContent]{ + MatrixEventBase: bridgev2.MatrixEventBase[*event.MemberEventContent]{Portal: portal}, + }, + Target: ghost, + Type: change, + }) + if !errors.Is(err, bridgev2.ErrMembershipNotSupported) { + t.Fatalf("HandleMatrixMembership(%v) error = %v, want unsupported", change, err) + } + } if err := ic.HandleMatrixDeleteChat(context.Background(), &bridgev2.MatrixDeleteChat{ Portal: portal, }); err != nil { @@ -966,7 +968,7 @@ func TestDialogVisibilityChangeTriggersResyncComparison(t *testing.T) { func TestShouldProjectChatLoadsUnknownHiddenDialogVisibility(t *testing.T) { hidden := true - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/rpc/state/chat" { t.Fatalf("unexpected path %s", r.URL.Path) } @@ -995,7 +997,7 @@ func TestShouldProjectChatLoadsUnknownHiddenDialogVisibility(t *testing.T) { func TestChatUpsertRefreshesVisibleDialogToHiddenBeforeProjection(t *testing.T) { hidden := true - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/rpc/state/chat" { t.Fatalf("unexpected path %s", r.URL.Path) } @@ -1114,9 +1116,9 @@ func TestPrioritizedHistoryDialogsPrefersActiveDMsAndNewest(t *testing.T) { got := ic.prioritizedHistoryDialogs([]sidecar.DialogRecord{ {ChatID: 2, LastMessageID: &last40}, {ChatID: 5, LastMessageID: &last9}, - {ChatID: 1, PeerUserID: &peer100, LastMessageID: &last5}, + {ChatID: 1, Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: &peer100}, PeerUserID: &peer100, LastMessageID: &last5}, {ChatID: 4, LastMessageID: &last22}, - {ChatID: 3, PeerUserID: &peer101, LastMessageID: &last30}, + {ChatID: 3, Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: &peer101}, PeerUserID: &peer101, LastMessageID: &last30}, }) want := []int64{4, 3, 1, 2} @@ -1187,7 +1189,7 @@ func TestIsRateLimitedErrorDetectsInlineFloodResponses(t *testing.T) { func TestSyncDeferredDialogWorkStopsOnRateLimitedHistory(t *testing.T) { var historyCalls int var participantsCalls int - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/rpc/history": historyCalls++ @@ -1219,7 +1221,7 @@ func TestSyncDeferredDialogWorkStopsOnRateLimitedHistory(t *testing.T) { } func TestGetUserInfoUsesCachedInlineUser(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/avatar.jpg" { t.Fatalf("path = %s, want /avatar.jpg", r.URL.Path) } @@ -1265,7 +1267,7 @@ func TestGetUserInfoUsesCachedInlineUser(t *testing.T) { } func TestDownloadInlineMedia(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodGet { t.Fatalf("method = %s, want GET", r.Method) } @@ -1292,7 +1294,7 @@ func TestDownloadInlineMediaRejectsUnsupportedScheme(t *testing.T) { func TestDownloadInlineMediaClassifiesPermanentAndRetryableHTTPFailures(t *testing.T) { status := http.StatusNotFound - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(status) })) defer server.Close() @@ -1363,7 +1365,7 @@ func TestSidecarErrorCertaintyOnlyMarksPermanentFailures(t *testing.T) { func TestDuplicateSidecarReplayOnlyReacknowledgesContiguousCursor(t *testing.T) { acked := make(chan sidecar.EventAckRequest, 1) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var request sidecar.EventAckRequest if err := json.NewDecoder(r.Body).Decode(&request); err != nil { t.Fatalf("decode ack request: %v", err) @@ -1472,3 +1474,130 @@ func mustConnectorJSON(t *testing.T, value any) json.RawMessage { func stringPtr(value string) *string { return &value } + +func newSidecarTestServer(handler http.Handler) *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/compatibility" { + json.NewEncoder(w).Encode(sidecar.ProtocolInfo{ProtocolVersion: sidecar.ProtocolVersion}) + return + } + if r.URL.Path == "/rpc/state/account" { + json.NewEncoder(w).Encode(sidecar.Response{ProtocolVersion: sidecar.ProtocolVersion, Outcome: sidecar.ResponseOutcome{Status: "ok", Data: json.RawMessage(`{"type":"account_state","data":{"deleted_chat_ids":[]}}`)}}) + return + } + handler.ServeHTTP(w, r) + })) +} + +func TestFinalReviewBackwardPageMustNotSkipHistoricalGap(t *testing.T) { + t.Run("backward", func(t *testing.T) { checkGenericPageDoesNotSkipHistory(t, false) }) + t.Run("unrelated-forward-anchor", func(t *testing.T) { checkGenericPageDoesNotSkipHistory(t, true) }) +} + +func checkGenericPageDoesNotSkipHistory(t *testing.T, forward bool) { + t.Helper() + messageID := int64(20) + if forward { + messageID = 22 + } + path := t.TempDir() + "/history.db" + ic := fixtureHistoryClient(t, path) + if err := ic.persistHistoryCheckpoint(t.Context(), 7, 10); err != nil { + t.Fatal(err) + } + key := ic.portalKeyForChat(7) + for _, row := range []*database.Message{ + {ID: "7/10", MXID: "$10", Room: key, Timestamp: time.Unix(200, 0)}, + {ID: "7/21", MXID: "$21", Room: key, Timestamp: time.Unix(100, 0)}, + } { + if err := ic.UserLogin.Bridge.DB.Message.Insert(t.Context(), row); err != nil { + t.Fatal(err) + } + } + anchor, err := ic.UserLogin.Bridge.DB.Message.GetFirstPortalMessage(t.Context(), key) + if err != nil || anchor.ID != "7/21" { + t.Fatalf("actual backward anchor: %+v %v", anchor, err) + } + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/rpc/state/chat": + writeConnectorSidecarResult(t, w, "chat_state", sidecar.ChatStateSnapshot{ChatID: 7}) + case "/rpc/history": + var req sidecar.HistoryRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + } + if (forward && (req.AfterMessageID == nil || *req.AfterMessageID != 21)) || (!forward && (req.BeforeMessageID == nil || *req.BeforeMessageID != 21)) || req.Limit == nil || *req.Limit != 1 { + t.Errorf("unexpected request: %+v", req) + } + writeConnectorSidecarResult(t, w, "history", sidecar.HistoryPage{HasMore: true, Messages: []sidecar.MessageRecord{{ChatID: 7, MessageID: messageID, Timestamp: 50, Content: sidecar.MessageContent{Type: "text", Text: "synthetic"}}}}) + default: + t.Errorf("unexpected route: %s", r.URL.Path) + w.WriteHeader(400) + } + })) + defer server.Close() + ic.Sidecar = sidecar.NewClient(server.URL) + // A plain portal keeps message conversion independent of Matrix I/O. The + // completion callback uses the real SQLite mapping/metadata store above. + p := testPortal("7") + p.Receiver = key.Receiver + resp, err := ic.FetchMessages(t.Context(), bridgev2.FetchMessagesParams{Portal: p, Forward: forward, AnchorMessage: anchor, Count: 1}) + if err != nil { + t.Fatal(err) + } + if len(resp.Messages) != 1 || resp.Messages[0].ID != makeMessageID(7, messageID) || resp.CompleteCallback == nil { + t.Fatalf("backfill response: %+v", resp) + } + if err = ic.UserLogin.Bridge.DB.Message.Insert(t.Context(), &database.Message{ID: makeMessageID(7, messageID), MXID: "$page", Room: key, Timestamp: time.Unix(50, 0)}); err != nil { + t.Fatal(err) + } + resp.CompleteCallback() + restarted := fixtureHistoryClient(t, path) + got, err := restarted.loadHistoryCheckpoint(t.Context(), 7) + if err != nil { + t.Fatal(err) + } + if got != 10 { + t.Fatalf("generic callback advanced frontier to %d while source IDs11..19 have no mappings; expected10", got) + } +} + +func TestGetChatInfoPreservesUnknownMembershipForEmptyParticipants(t *testing.T) { + for _, tc := range []struct { + name string + participants []sidecar.ChatParticipantRecord + }{ + {"empty", nil}, + {"all-invalid", []sidecar.ChatParticipantRecord{{UserID: 0}, {UserID: -1}, {UserID: -9223372036854775808}}}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/rpc/chat/participants" { + t.Errorf("unexpected path %s", r.URL.Path) + w.WriteHeader(400) + return + } + calls++ + writeConnectorSidecarResult(t, w, "chat_participants", sidecar.ChatParticipantsPage{Participants: tc.participants}) + })) + defer server.Close() + ic := &InlineClient{ + UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ID: "1"}}, + AccountID: "1", Sidecar: sidecar.NewClient(server.URL), + dialogs: map[int64]sidecar.DialogRecord{7: {ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "channel"}}}, + } + info, err := ic.GetChatInfo(t.Context(), testPortal("7")) + if err != nil { + t.Fatal(err) + } + if info == nil || info.Members != nil { + t.Fatalf("unknown participant response became authoritative membership: %+v", info) + } + if info.Type == nil || *info.Type != database.RoomTypeDefault || calls != 1 { + t.Fatalf("lost channel info/call: %+v calls%d", info, calls) + } + }) + } +} diff --git a/pkg/connector/commands.go b/pkg/connector/commands.go index e4d55ba..6515dfa 100644 --- a/pkg/connector/commands.go +++ b/pkg/connector/commands.go @@ -62,7 +62,7 @@ func registerInlineCommands(bridge *bridgev2.Bridge) { bridge.Log.Warn().Msg("Bridge command processor does not support custom Inline commands") return } - processor.AddHandlers(commandInlineStatus, commandInlineReconnect, commandInlineSettings, commandInlineHiddenChats) + processor.AddHandlers(commandInlineStatus, commandInlineReconnect, commandInlineSettings, commandInlineHiddenChats, commandInlineMigrate, commandInlineChannelDefault) } func fnInlineStatus(ce *commands.Event) { @@ -152,12 +152,13 @@ func inlineSettingsSummary(client *InlineClient) string { if overrideLabel == "" { overrideLabel = "default" } + muteDefault, muteOverride, muteEffective := client.channelMuteSettings() return fmt.Sprintf( "Inline settings\n\n- Hidden chats: `%s`\n- Account override: `%s`\n- Bridge default: `%s`\n\nHidden chats are Inline dialogs with `showInChatList: false`. Excluding them prevents new Matrix portal creation, history fill, and inbound event projection. Existing Matrix rooms are not deleted.", effective, overrideLabel, defaults, - ) + ) + fmt.Sprintf("\n\nDesired channel mute default: `%s` (bridge `%s`, account `%s`); automatic writes disabled. ", muteEffective, muteDefault, muteOverride) + notificationPolicyDegraded + ". Set Beeper Settings → Accounts → Inline → Show in All Inbox to DMs only; mute and Low Priority are separate user choices. Presentation version: 1. Manual unread projection remains pending until origin-safe transport is proven." } func inlineCommandLogin(ce *commands.Event) (*bridgev2.UserLogin, *InlineClient) { @@ -195,6 +196,8 @@ func inlineStatusSummary(login *bridgev2.UserLogin, client *InlineClient, curren if client != nil { fmt.Fprintf(&out, "- Account ID: `%s`\n", client.AccountID) fmt.Fprintf(&out, "- Go client logged in: `%t`\n", client.IsLoggedIn()) + fmt.Fprintf(&out, "- Notification policy: %s\n", notificationPolicyDegraded) + out.WriteString(client.projectionStatus(context.Background())) } if err != nil { fmt.Fprintf(&out, "- Sidecar: check failed: `%v`\n", err) @@ -213,3 +216,22 @@ func inlineStatusSummary(login *bridgev2.UserLogin, client *InlineClient, curren } return out.String() } + +var commandInlineMigrate = &commands.FullHandler{ + Func: fnInlineMigrate, Name: "inline-migrate", RequiresLogin: true, + Help: commands.HelpMeta{Section: commands.HelpSectionMisc, Description: "Preview or repair selected existing Inline rooms in place", Args: "[preview|apply|rollback] "}, +} + +func fnInlineMigrate(ce *commands.Event) { + _, client := inlineCommandLogin(ce) + if client == nil { + ce.Reply("Your default login is not an Inline login.") + return + } + mode, ids, err := parseMigrationCommand(ce.Args) + if err != nil { + ce.Reply("Usage: `$cmdprefix inline-migrate [preview|apply|rollback] `\n\n%s", err) + return + } + ce.Reply(client.migrateSelected(ce.Ctx, mode, ids)) +} diff --git a/pkg/connector/commands_test.go b/pkg/connector/commands_test.go index 883636d..18de90f 100644 --- a/pkg/connector/commands_test.go +++ b/pkg/connector/commands_test.go @@ -2,6 +2,7 @@ package connector import ( "errors" + "maunium.net/go/mautrix/bridgev2" "strings" "testing" @@ -78,3 +79,47 @@ func TestInlineSettingsSummaryExplainsEffectivePolicy(t *testing.T) { } } } + +func TestMigrationCommandRequiresExplicitBoundedNumericSelection(t *testing.T) { + for _, args := range [][]string{nil, {"apply"}, {"apply", "space:1"}, {"rollback", "7", "7"}, {"apply", "0"}, {"apply", "-1"}, {"anything", "7"}} { + if _, _, err := parseMigrationCommand(args); err == nil { + t.Errorf("unsafe selection accepted: %v", args) + } + } + mode, ids, err := parseMigrationCommand([]string{"7", "8"}) + if err != nil || mode != "preview" || len(ids) != 2 || ids[0] != 7 || ids[1] != 8 { + t.Fatalf("default preview=%s %v %v", mode, ids, err) + } +} +func TestSettingsDiscloseDisabledAutomaticMute(t *testing.T) { + got := inlineSettingsSummary(&InlineClient{}) + if !strings.Contains(got, "notification_policy_degraded") || !strings.Contains(got, "DMs only") { + t.Fatal("settings hide automatic-mute prerequisite or Inbox setup") + } +} + +func TestChannelMutePreferenceIsDesiredOnlyAndAccountScoped(t *testing.T) { + disabled := false + login := &bridgev2.UserLogin{Bridge: &bridgev2.Bridge{Network: &InlineConnector{Config: Config{MuteChannelsByDefault: &disabled}}}} + a := newInlineClient(login, &UserLoginMetadata{AccountID: "1", ChannelMuteDefault: "muted"}, "http://127.0.0.1:1") + b := newInlineClient(login, &UserLoginMetadata{AccountID: "2"}, "http://127.0.0.1:1") + defaults, override, effective := a.channelMuteSettings() + if defaults != "normal" || override != "muted" || effective != "muted" { + t.Fatal("account preference lost", defaults, override, effective) + } + _, override, effective = b.channelMuteSettings() + if override != "default" || effective != "normal" { + t.Fatal("preference leaked between logins") + } + if !strings.Contains(inlineSettingsSummary(a), "automatic writes disabled") { + t.Fatal("desired mute preference represented as applied") + } + for _, value := range []string{"default", "muted", "normal"} { + if _, err := parseChannelMuteCommand([]string{value}); err != nil { + t.Fatal(err) + } + } + if _, err := parseChannelMuteCommand([]string{"attested"}); err == nil { + t.Fatal("unsupported origin attestation accepted") + } +} diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index f3e9c13..b1f2666 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -13,6 +13,7 @@ import ( "maunium.net/go/mautrix/bridgev2" "maunium.net/go/mautrix/bridgev2/database" "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" "github.com/inline-chat/matrix-inline/pkg/sidecar" @@ -47,11 +48,12 @@ type InlineConnector struct { } type Config struct { - DisplayName string `yaml:"displayname" json:"displayname"` - NetworkURL string `yaml:"network_url" json:"network_url"` - NetworkIcon string `yaml:"network_icon" json:"network_icon"` - SidecarURL string `yaml:"sidecar_url" json:"sidecar_url"` - HiddenDialogs string `yaml:"hidden_dialogs" json:"hidden_dialogs"` + DisplayName string `yaml:"displayname" json:"displayname"` + NetworkURL string `yaml:"network_url" json:"network_url"` + NetworkIcon string `yaml:"network_icon" json:"network_icon"` + SidecarURL string `yaml:"sidecar_url" json:"sidecar_url"` + HiddenDialogs string `yaml:"hidden_dialogs" json:"hidden_dialogs"` + MuteChannelsByDefault *bool `yaml:"mute_channels_by_default" json:"mute_channels_by_default"` } type UserLoginMetadata struct { @@ -65,16 +67,23 @@ type UserLoginMetadata struct { BridgeRecoveryCursor int64 `json:"bridge_recovery_cursor,omitempty"` DialogSyncCursor string `json:"dialog_sync_cursor,omitempty"` HiddenDialogs string `json:"hidden_dialogs,omitempty"` + ChannelMuteDefault string `json:"channel_mute_default,omitempty"` SessionInvalidated bool `json:"session_invalidated,omitempty"` } type MessageMetadata struct { - InlineFingerprint string `json:"inline_fingerprint,omitempty"` - MediaHandled bool `json:"media_handled,omitempty"` + InlineFingerprint string `json:"inline_fingerprint,omitempty"` + MediaHandled bool `json:"media_handled,omitempty"` + InlineRevision *int64 `json:"inline_revision,omitempty"` + InlineEditTime *int64 `json:"inline_edit_time,omitempty"` + InlineMediaID string `json:"inline_media_id,omitempty"` + InlineMedia *event.MessageEventContent `json:"inline_media,omitempty"` } type PortalMetadata struct { - HistoryCheckpoint int64 `json:"history_checkpoint,omitempty"` + InlineTopic *sourceTopicState `json:"inline_topic,omitempty"` + HistoryCheckpoint int64 `json:"history_checkpoint,omitempty"` + HistoricalCheckpoint int64 `json:"historical_checkpoint,omitempty"` } func (meta *PortalMetadata) CopyFrom(other any) { @@ -103,6 +112,7 @@ var _ bridgev2.NetworkConnector = (*InlineConnector)(nil) func (ic *InlineConnector) Init(bridge *bridgev2.Bridge) { ic.br = bridge + disableImplicitHistoryRead(bridge) registerInlineCommands(bridge) } @@ -130,18 +140,18 @@ func validatePortalEventDelivery() error { } func (ic *InlineConnector) GetBridgeInfoVersion() (info, capabilities int) { - return 2, 1 + return 4, 3 } func (ic *InlineConnector) GetCapabilities() *bridgev2.NetworkGeneralCapabilities { - return &bridgev2.NetworkGeneralCapabilities{ + caps := &bridgev2.NetworkGeneralCapabilities{ AggressiveUpdateInfo: true, - ImplicitReadReceipts: true, + ImplicitReadReceipts: false, Provisioning: bridgev2.ProvisioningCapabilities{ ResolveIdentifier: bridgev2.ResolveIdentifierCapabilities{ LookupUsername: true, - Search: false, - ContactList: false, + Search: true, + ContactList: false, // Bounded discovery is not a complete contacts snapshot. CreateDM: true, }, GroupCreation: map[string]bridgev2.GroupTypeCapabilities{ @@ -154,7 +164,7 @@ func (ic *InlineConnector) GetCapabilities() *bridgev2.NetworkGeneralCapabilitie MaxLength: 256, }, Topic: bridgev2.GroupFieldCapability{ - Allowed: true, + Allowed: false, MaxLength: 4096, }, Participants: bridgev2.GroupFieldCapability{ @@ -164,6 +174,21 @@ func (ic *InlineConnector) GetCapabilities() *bridgev2.NetworkGeneralCapabilitie }, }, } + private := caps.Provisioning.GroupCreation["thread"] + private.TypeDescription = "Private Inline channel (home or workspace)" + private.Name.Required = false + private.Topic.Allowed = true + private.Parent.Allowed = true + private.Participants.Required = true + private.Participants.MinLength = 1 + caps.Provisioning.GroupCreation["thread"] = private + caps.Provisioning.GroupCreation["private_thread"] = private + public := private + public.TypeDescription = "Public Inline workspace channel" + public.Parent.Required = true + public.Participants = bridgev2.GroupFieldCapability{} + caps.Provisioning.GroupCreation["public_thread"] = public + return caps } func (ic *InlineConnector) GetName() bridgev2.BridgeName { @@ -187,6 +212,9 @@ sidecar_url: http://127.0.0.1:29342 # Whether dialogs hidden from Inline's normal chat list should be projected to Matrix. # Per-account bridge bot settings can override this default. hidden_dialogs: exclude +# Desired policy for new channels only. Automatic mute writes are currently +# disabled because target origin attribution is unverified; mute manually. +mute_channels_by_default: true `, &ic.Config, nil } @@ -351,6 +379,8 @@ func newInlineClient(login *bridgev2.UserLogin, meta *UserLoginMetadata, sidecar bridgeRecoveryCursor: meta.BridgeRecoveryCursor, dialogSyncCursor: meta.DialogSyncCursor, hiddenDialogsDefault: hiddenDialogsPolicyForLogin(login), + channelMuteDefault: channelMuteDefaultForLogin(login), + channelMuteOverride: channelMutePolicyFromMetadata(meta.ChannelMuteDefault), hiddenDialogsOverride: hiddenDialogsPolicyFromMetadata(meta.HiddenDialogs), loggedIn: !meta.SessionInvalidated, users: make(map[int64]sidecar.UserRecord), diff --git a/pkg/connector/directory.go b/pkg/connector/directory.go new file mode 100644 index 0000000..be7b3d1 --- /dev/null +++ b/pkg/connector/directory.go @@ -0,0 +1,374 @@ +package connector + +import ( + "context" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/networkid" +) + +const ( + directoryPageSize = 100 + directoryMaxPages = 4 + directoryParticipantBudget = 8 + directoryRefreshTimeout = 20 * time.Second +) + +var errDirectoryNotFound = errors.New("Inline user has no eligible directory entry; lookup does not establish reachability") +var errDirectoryAmbiguous = errors.New("Inline username is ambiguous") +var errDirectoryIncomplete = errors.New("Inline directory refresh was bounded; lookup may be incomplete") + +// Fresh means this user was returned as a participant by a successful live +// ChatParticipants read. False means source-backed cached discovery only. +// Neither value proves permission to create, edit, delete or manage membership. +type directoryUser struct { + Record sidecar.UserRecord + Fresh bool +} + +// directorySnapshot is request-local, never a global cache. Do not reuse across +// logins or persist it as permission evidence. AccountState members and Dialogs +// rows are cached even after Dialogs refreshes: its response is store-backed. +type directorySnapshot struct { + users map[int64]directoryUser + spaces map[int64]sidecar.SpaceRecord + access map[int64]string + members map[int64]map[int64]bool + receiver networkid.UserLoginID + namespace string + incomplete bool +} + +// loadDirectory performs at most four dialog pages, one account-state read and +// eight live participant reads within twenty seconds. It returns a bounded +// subset, never fills missing users from arbitrary ic.users records and never +// falls back after a source error. The sidecar client owns namespace routing. +// Hidden/deleted dialogs alone are excluded from discovery; independent +// workspace membership or live participants can still supply eligible users. +func (ic *InlineClient) loadDirectory(ctx context.Context) (*directorySnapshot, error) { + if ic.Sidecar == nil || strings.TrimSpace(ic.Sidecar.SessionNamespace) == "" || ic.UserLogin == nil || ic.UserLogin.UserLogin == nil || ic.UserLogin.ID == "" { + return nil, errors.New("Inline directory requires a scoped login") + } + ctx, cancel := context.WithTimeout(ctx, directoryRefreshTimeout) + defer cancel() + d := &directorySnapshot{users: map[int64]directoryUser{}, spaces: map[int64]sidecar.SpaceRecord{}, access: map[int64]string{}, members: map[int64]map[int64]bool{}, receiver: ic.UserLogin.ID, namespace: ic.Sidecar.SessionNamespace} + var dialogs []sidecar.DialogRecord + records := map[int64]sidecar.UserRecord{} + addRecords := func(users []sidecar.UserRecord) { + for _, u := range users { + if u.UserID > 0 { + records[u.UserID] = u + } + } + } + cursor := "" + seen := map[string]bool{} + for page := 0; page < directoryMaxPages; page++ { + limit := uint32(directoryPageSize) + result, err := ic.Sidecar.Dialogs(ctx, sidecar.DialogsRequest{Limit: &limit, Cursor: cursor, Order: sidecar.DialogsOrderStableChatID}) + if err != nil { + return nil, fmt.Errorf("refresh Inline directory dialogs: %w", err) + } + dialogs = append(dialogs, result.Dialogs...) + addRecords(result.Users) + cursor = result.NextCursor + if cursor == "" { + break + } + if seen[cursor] { + d.incomplete = true + break + } + seen[cursor] = true + if page == directoryMaxPages-1 { + d.incomplete = true + } + } + // Read after refresh: getChats can update workspace access and tombstones. + state, err := ic.Sidecar.AccountState(ctx) + if err != nil { + return nil, fmt.Errorf("read Inline directory access: %w", err) + } + addRecords(state.Users) + for _, s := range state.Spaces { + if s.SpaceID > 0 { + d.spaces[s.SpaceID] = s + } + } + for _, a := range state.SpaceAccess { + // Conflicting evidence is unknown, never whichever row happened to be last. + if prev, ok := d.access[a.SpaceID]; ok && prev != a.Status { + d.access[a.SpaceID] = "unknown" + } else { + d.access[a.SpaceID] = a.Status + } + } + allow := func(id int64, fresh bool) { + if u, ok := records[id]; ok && id > 0 { + old := d.users[id] + d.users[id] = directoryUser{Record: u, Fresh: old.Fresh || fresh} + } + } + for _, m := range state.SpaceMembers { + if m.UserID <= 0 || m.SpaceID <= 0 || d.access[m.SpaceID] != "accessible" { + continue + } + if d.members[m.SpaceID] == nil { + d.members[m.SpaceID] = map[int64]bool{} + } + d.members[m.SpaceID][m.UserID] = true + allow(m.UserID, false) + } + deleted := map[int64]bool{} + for _, id := range state.DeletedChatIDs { + deleted[id] = true + } + sort.Slice(dialogs, func(i, j int) bool { return dialogs[i].ChatID < dialogs[j].ChatID }) + seenChats := map[int64]bool{} + reads := 0 + for _, dialog := range dialogs { + kind := classifyDialog(dialog) + if seenChats[dialog.ChatID] || deleted[dialog.ChatID] || (dialog.ChatListHidden != nil && *dialog.ChatListHidden) || kind == roomUnknown { + continue + } + seenChats[dialog.ChatID] = true + if kind == roomDirect { + allow(*dialog.Structure.PeerUserID, false) + continue + } + // A known chat can retain independent access after workspace removal. Only + // a successful live participant read establishes that bounded subset. + if reads == directoryParticipantBudget { + d.incomplete = true + continue + } + reads++ + page, err := ic.Sidecar.ChatParticipants(ctx, sidecar.ChatParticipantsRequest{ChatID: dialog.ChatID}) + if err != nil { + return nil, fmt.Errorf("refresh Inline directory participants for %d: %w", dialog.ChatID, err) + } + addRecords(page.Users) + for _, p := range page.Participants { + allow(p.UserID, true) + } + } + return d, nil +} + +func directoryPositiveID(s string) (int64, bool) { + if s == "" { + return 0, false + } + for _, c := range s { + if c < '0' || c > '9' { + return 0, false + } + } + id, err := strconv.ParseInt(s, 10, 64) + return id, err == nil && id > 0 +} +func directoryUsername(s string) string { + return strings.ToLower(strings.TrimPrefix(strings.TrimSpace(s), "@")) +} + +func directoryNumericShape(s string) bool { + digits := strings.TrimPrefix(strings.TrimPrefix(s, "-"), "+") + return digits != "" && strings.Trim(digits, "0123456789") == "" +} + +// resolve accepts an exact positive decimal ID or canonical username (optional +// @, case-insensitive). IDs are never substring matched or synthesized. A +// bounded/truncated snapshot cannot prove a username unique across the account. +func (d *directorySnapshot) resolve(identifier string) (directoryUser, error) { + if d == nil { + return directoryUser{}, errDirectoryNotFound + } + identifier = strings.TrimSpace(identifier) + if id, ok := directoryPositiveID(identifier); ok { + if user, found := d.users[id]; found { + return user, nil + } + if d.incomplete { + return directoryUser{}, errDirectoryIncomplete + } + return directoryUser{}, errDirectoryNotFound + } + name := directoryUsername(identifier) + // A malformed numeric ID must not fall through to a coincidentally numeric + // username (including zero, negative values and int64 overflow). + if directoryNumericShape(name) { + return directoryUser{}, errDirectoryNotFound + } + if name == "" || strings.ContainsAny(name, "@+ \t\r\n") { + return directoryUser{}, errDirectoryNotFound + } + var found directoryUser + count := 0 + for _, user := range d.users { + if user.Record.Username != nil && directoryUsername(*user.Record.Username) == name { + count++ + found = user + } + } + if count > 1 { + return directoryUser{}, errDirectoryAmbiguous + } + if d.incomplete { + return directoryUser{}, errDirectoryIncomplete + } + if count == 1 { + return found, nil + } + return directoryUser{}, errDirectoryNotFound +} + +// search pages the same accessible subset in ascending numeric user ID order. +// Numeric queries match exact IDs only; text queries match username or names. +func (d *directorySnapshot) search(query string, offset, limit int) ([]directoryUser, error) { + if offset < 0 || limit < 1 || limit > directoryPageSize { + return nil, fmt.Errorf("Inline directory requires offset >= 0 and limit 1..%d", directoryPageSize) + } + result := []directoryUser{} + if d == nil { + return result, nil + } + query = directoryUsername(query) + numericID, numeric := directoryPositiveID(query) + if !numeric && directoryNumericShape(query) { + return result, nil + } + for id, u := range d.users { + match := query == "" + if numeric { + match = id == numericID + } else if !match { + for _, field := range []*string{u.Record.Username, u.Record.DisplayName, u.Record.FirstName, u.Record.LastName} { + if field != nil && strings.Contains(strings.ToLower(*field), query) { + match = true + break + } + } + } + if match { + result = append(result, u) + } + } + sort.Slice(result, func(i, j int) bool { return result[i].Record.UserID < result[j].Record.UserID }) + if offset >= len(result) { + return []directoryUser{}, nil + } + result = result[offset:] + if len(result) > limit { + result = result[:limit] + } + return result, nil +} + +// SearchUsers and GetContactList expose at most 100 results. These framework +// responses have no freshness field; callers needing provenance use the snapshot. +func (ic *InlineClient) SearchUsers(ctx context.Context, query string) ([]*bridgev2.ResolveIdentifierResponse, error) { + d, err := ic.loadDirectory(ctx) + if err != nil { + return nil, err + } + users, err := d.search(query, 0, directoryPageSize) + if err != nil { + return nil, err + } + results := make([]*bridgev2.ResolveIdentifierResponse, 0, len(users)) + for _, u := range users { + results = append(results, &bridgev2.ResolveIdentifierResponse{UserID: makeUserID(strconv.FormatInt(u.Record.UserID, 10)), UserInfo: userInfoFromRecord(u.Record)}) + } + return results, nil +} + +// GetContactList serves the framework compatibility endpoint as a bounded +// accessible subset. ContactList remains unadvertised because this is not a +// complete contacts snapshot; clients should prefer the search capability. +func (ic *InlineClient) GetContactList(ctx context.Context) ([]*bridgev2.ResolveIdentifierResponse, error) { + return ic.SearchUsers(ctx, "") +} + +// validateThreadCreation is pure preflight, NOT positive authorization. The +// caller must issue CreateThread on the same scoped sidecar and propagate its +// current permission error. spaceThreadGuards checks current workspace privacy, +// public-chat access and active participants; cached Role/Creator never grant it. +// Empty/thread/channel/private_thread create private ordinary channels; +// public_thread requires a workspace and no participants. True group DMs reject. +// IDs returned here are explicit deduplicated invitees; the source adds self. +func validateThreadCreation(params *bridgev2.GroupCreateParams, receiver networkid.UserLoginID, d *directorySnapshot) (sidecar.CreateThreadRequest, []networkid.UserID, error) { + var request sidecar.CreateThreadRequest + fail := func(message string) (sidecar.CreateThreadRequest, []networkid.UserID, error) { + return sidecar.CreateThreadRequest{}, nil, errors.New(message) + } + if params == nil || d == nil || receiver == "" || d.receiver != receiver || d.namespace == "" { + return fail("Inline creation requires directory facts for this login") + } + switch params.Type { + case "", "thread", "channel", "private_thread": + case "public_thread": + request.IsPublic = true + default: + return fail("unsupported Inline creation type: " + params.Type) + } + if params.Username != "" || params.Avatar != nil || params.Disappear != nil { + return fail("Inline thread username, avatar and disappearing metadata are unsupported") + } + if params.Parent != nil { + if params.Parent.Receiver != receiver || !strings.HasPrefix(string(params.Parent.ID), "space:") { + return fail("Inline parent must be a workspace belonging to this login") + } + space, ok := directoryPositiveID(strings.TrimPrefix(string(params.Parent.ID), "space:")) + if !ok || d.access[space] != "accessible" { + return fail("Inline workspace access unavailable") + } + if _, ok = d.spaces[space]; !ok { + return fail("Inline workspace metadata unavailable") + } + request.SpaceID = &space + } + if request.IsPublic && request.SpaceID == nil { + return fail("public Inline home threads are unsupported") + } + if request.IsPublic && len(params.Participants) > 0 { + return fail("public Inline workspace threads require empty participants") + } + if !request.IsPublic && len(params.Participants) == 0 { + return fail("private Inline threads require participants") + } + ids := []networkid.UserID{} + seen := map[int64]bool{} + for _, participant := range params.Participants { + id, ok := directoryPositiveID(string(participant)) + if !ok { + return fail("Inline participants must be exact positive user IDs") + } + user, err := d.resolve(strconv.FormatInt(id, 10)) + if err != nil { + return sidecar.CreateThreadRequest{}, nil, fmt.Errorf("Inline participant %q: %w", participant, err) + } + if request.SpaceID != nil && !d.members[*request.SpaceID][user.Record.UserID] { + return fail("Inline participant has no workspace membership evidence") + } + if seen[id] { + continue + } + seen[id] = true + request.Participants = append(request.Participants, sidecar.ChatCreateParticipant{UserID: id}) + ids = append(ids, makeUserID(strconv.FormatInt(id, 10))) + } + if params.Name != nil { + request.Title = optionalString(params.Name.Name) + } + if params.Topic != nil { + request.Description = optionalString(params.Topic.Topic) + } + return request, ids, nil +} diff --git a/pkg/connector/directory_integration_test.go b/pkg/connector/directory_integration_test.go new file mode 100644 index 0000000..55626c5 --- /dev/null +++ b/pkg/connector/directory_integration_test.go @@ -0,0 +1,125 @@ +package connector + +import ( + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/networkid" +) + +func TestDirectoryIntegratedLookupRejectsUnrelatedCachedUsers(t *testing.T) { + f := &directoryFixture{state: sidecar.AccountStateSnapshot{Users: []sidecar.UserRecord{dirUser(42, "ada")}}, dialogs: sidecar.DialogsPage{Dialogs: []sidecar.DialogRecord{{ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: dirPtr(int64(42))}}}}} + s := directoryServer(t, map[string]*directoryFixture{"a": f}) + ic := directoryClient(s.URL, "a") + if _, err := ic.ResolveIdentifier(t.Context(), "999", false); !errors.Is(err, errDirectoryNotFound) { + t.Fatalf("arbitrary cached user resolved: %v", err) + } + if _, err := ic.CreateChatWithGhost(t.Context(), &bridgev2.Ghost{Ghost: &database.Ghost{ID: "999"}}); !errors.Is(err, errDirectoryNotFound) { + t.Fatalf("undiscovered ghost created chat: %v", err) + } + got, err := ic.ResolveIdentifier(t.Context(), "@ADA", false) + if err != nil || got == nil || got.UserID != "42" || got.UserInfo == nil || got.UserInfo.Name == nil || *got.UserInfo.Name != "ada" { + t.Fatalf("accessible canonical username lost: %+v %v", got, err) + } +} + +func TestDirectoryGhostInviteTriesAnotherLoginAfterMiss(t *testing.T) { + var creates []string + s := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ns := r.Header.Get("X-Inline-Session-Namespace") + switch r.URL.Path { + case "/rpc/dialogs": + page := sidecar.DialogsPage{} + if ns == "b" { + page = sidecar.DialogsPage{Dialogs: []sidecar.DialogRecord{{ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: dirPtr(int64(42))}}}, Users: []sidecar.UserRecord{dirUser(42, "ada")}} + } + writeConnectorSidecarResult(t, w, "dialogs", page) + case "/rpc/state/account": + writeConnectorSidecarResult(t, w, "account_state", sidecar.AccountStateSnapshot{}) + case "/rpc/chat/create-dm": + creates = append(creates, ns) + writeConnectorSidecarResult(t, w, "created_chat", sidecar.CreatedChat{ChatID: 7}) + default: + t.Errorf("unexpected RPC %s", r.URL.Path) + http.Error(w, "unexpected", 500) + } + })) + defer s.Close() + ghost := &bridgev2.Ghost{Ghost: &database.Ghost{ID: "42"}} + _, err := directoryClient(s.URL, "a").CreateChatWithGhost(t.Context(), ghost) + if !errors.Is(err, bridgev2.ErrResolveIdentifierTryNext) || !errors.Is(err, errDirectoryNotFound) { + t.Fatalf("first login should allow retry with directory diagnostic: %v", err) + } + chat, err := directoryClient(s.URL, "b").CreateChatWithGhost(t.Context(), ghost) + if err != nil || chat == nil || len(creates) != 1 || creates[0] != "b" { + t.Fatalf("second login did not create the DM: chat=%+v err=%v creates=%v", chat, err, creates) + } +} + +func TestDirectoryIntegratedCreatePreflightAndCurrentPermission(t *testing.T) { + mutations := 0 + var last sidecar.CreateThreadRequest + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/compatibility" { + json.NewEncoder(w).Encode(sidecar.ProtocolInfo{ProtocolVersion: sidecar.ProtocolVersion}) + return + } + if r.Header.Get("X-Inline-Session-Namespace") != "a" { + t.Error("lost account namespace") + } + switch r.URL.Path { + case "/rpc/dialogs": + writeConnectorSidecarResult(t, w, "dialogs", sidecar.DialogsPage{}) + case "/rpc/state/account": + writeConnectorSidecarResult(t, w, "account_state", sidecar.AccountStateSnapshot{Spaces: []sidecar.SpaceRecord{{SpaceID: 10}}, SpaceAccess: []sidecar.SpaceAccessRecord{{SpaceID: 10, Status: "accessible"}}}) + case "/rpc/chat/create-thread": + mutations++ + if err := json.NewDecoder(r.Body).Decode(&last); err != nil { + t.Error(err) + } + json.NewEncoder(w).Encode(map[string]any{"protocol_version": sidecar.ProtocolVersion, "outcome": map[string]any{"status": "error", "data": sidecar.Error{Category: "permission_denied", Message: "current source policy denies creation"}}}) + default: + t.Errorf("unexpected RPC %s", r.URL.Path) + http.Error(w, "unexpected", 500) + } + })) + defer s.Close() + ic := directoryClient(s.URL, "a") + if _, err := ic.CreateGroup(t.Context(), &bridgev2.GroupCreateParams{Type: "group_dm"}); err == nil || mutations != 0 { + t.Fatalf("unsupported type caused source mutation: %v mutations%d", err, mutations) + } + _, err := ic.CreateGroup(t.Context(), &bridgev2.GroupCreateParams{Type: "public_thread", Parent: &networkid.PortalKey{ID: "space:10", Receiver: ic.UserLogin.ID}}) + var source *sidecar.Error + if !errors.As(err, &source) || source.Category != "permission_denied" || mutations != 1 || !last.IsPublic || last.SpaceID == nil || *last.SpaceID != 10 || len(last.Participants) != 0 { + t.Fatalf("public workspace request/current denial lost: %+v %v mutations%d", last, err, mutations) + } +} + +func TestDirectoryIntegratedPrivateChannelIsNeverInferredDM(t *testing.T) { + s := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/rpc/dialogs": + writeConnectorSidecarResult(t, w, "dialogs", sidecar.DialogsPage{Dialogs: []sidecar.DialogRecord{{ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: dirPtr(int64(42))}}}, Users: []sidecar.UserRecord{dirUser(42, "ada")}}) + case "/rpc/chat/create-thread": + writeConnectorSidecarResult(t, w, "created_chat", sidecar.CreatedChat{ChatID: 99}) + default: + t.Errorf("unexpected RPC %s", r.URL.Path) + http.Error(w, "unexpected", 500) + } + })) + defer s.Close() + ic := directoryClient(s.URL, "a") + got, err := ic.CreateGroup(t.Context(), &bridgev2.GroupCreateParams{Type: "private_thread", Participants: []networkid.UserID{"42"}}) + if err != nil { + t.Fatal(err) + } + if got.PortalKey.ID != "99" || got.PortalKey.Receiver != ic.UserLogin.ID || got.PortalInfo.Type == nil || *got.PortalInfo.Type != database.RoomTypeDefault || got.PortalInfo.Members == nil || got.PortalInfo.Members.OtherUserID != "" || got.PortalInfo.Members.IsFull { + t.Fatalf("ordinary private channel misclassified or claimed complete membership: %+v", got.PortalInfo) + } +} diff --git a/pkg/connector/directory_test.go b/pkg/connector/directory_test.go new file mode 100644 index 0000000..9d9e315 --- /dev/null +++ b/pkg/connector/directory_test.go @@ -0,0 +1,580 @@ +package connector + +import ( + "encoding/json" + "errors" + "math" + "net/http" + "net/http/httptest" + "reflect" + "strconv" + "strings" + "sync" + "testing" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "hegel.dev/go/hegel" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/event" +) + +func dirPtr[T any](v T) *T { return &v } +func dirUser(id int64, name string) sidecar.UserRecord { + return sidecar.UserRecord{UserID: id, Username: dirPtr(name), DisplayName: dirPtr(name)} +} +func dirSnapshot(users ...sidecar.UserRecord) *directorySnapshot { + d := &directorySnapshot{users: map[int64]directoryUser{}, spaces: map[int64]sidecar.SpaceRecord{}, access: map[int64]string{}, members: map[int64]map[int64]bool{}, receiver: "login-a", namespace: "a"} + for _, u := range users { + d.users[u.UserID] = directoryUser{Record: u} + } + return d +} + +func TestDirectoryExactPositiveIDProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + id := hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64)) + d := dirSnapshot(dirUser(id, "alice")) + got, err := d.resolve(strconv.FormatInt(id, 10)) + if err != nil || got.Record.UserID != id { + ht.Fatalf("exact accessible id %d: %+v %v", id, got, err) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} +func TestDirectoryArbitraryNumericNeverSynthesizesProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + id := hegel.Draw(ht, hegel.Integers[int64](math.MinInt64, math.MaxInt64)) + got, err := dirSnapshot().resolve(strconv.FormatInt(id, 10)) + if !errors.Is(err, errDirectoryNotFound) || got.Record.UserID != 0 { + ht.Fatalf("invented user %d: %+v %v", id, got, err) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func TestDirectoryExactIDMissDistinguishesIncompleteProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + id := hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64)) + present := hegel.Draw(ht, hegel.Booleans()) + incomplete := hegel.Draw(ht, hegel.Booleans()) + d := dirSnapshot() + d.incomplete = incomplete + if present { + d.users[id] = directoryUser{Record: dirUser(id, "known")} + } + got, err := d.resolve(strconv.FormatInt(id, 10)) + if present { + if err != nil || got.Record.UserID != id { + ht.Fatalf("known exact ID lost under incomplete=%v: %+v %v", incomplete, got, err) + } + } else { + want := errDirectoryNotFound + if incomplete { + want = errDirectoryIncomplete + } + if !errors.Is(err, want) || got.Record.UserID != 0 { + ht.Fatalf("ID %d miss incomplete=%v: %+v %v, want %v", id, incomplete, got, err, want) + } + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} +func TestDirectoryUsernameAndSearch(t *testing.T) { + d := dirSnapshot(dirUser(9, "Alice"), dirUser(2, "alice"), dirUser(20, "bob")) + for _, q := range []string{"alice", "@ALICE"} { + if _, err := d.resolve(q); !errors.Is(err, errDirectoryAmbiguous) { + t.Fatalf("ambiguity %q: %v", q, err) + } + } + if u, err := d.resolve("@BoB"); err != nil || u.Record.UserID != 20 { + t.Fatalf("username: %+v %v", u, err) + } + for _, q := range []string{"0", "-1", "2x", "+2", "alice@example.org", "+15555555"} { + if _, err := d.resolve(q); err == nil { + t.Fatalf("accepted %q", q) + } + } + got, err := d.search("", 0, 2) + if err != nil || len(got) != 2 || got[0].Record.UserID != 2 || got[1].Record.UserID != 9 { + t.Fatalf("ordered bounded search: %+v %v", got, err) + } + got, err = d.search("2", 0, 100) + if err != nil || len(got) != 1 || got[0].Record.UserID != 2 { + t.Fatalf("numeric search must be exact: %+v %v", got, err) + } + d.incomplete = true + if _, err = d.resolve("bob"); !errors.Is(err, errDirectoryIncomplete) { + t.Fatalf("truncated username claimed unique: %v", err) + } +} + +type directoryFixture struct { + state sidecar.AccountStateSnapshot + dialogs sidecar.DialogsPage + participants map[int64]sidecar.ChatParticipantsPage + deny string + calls []string + chatReads []int64 + mu sync.Mutex +} + +func directoryServer(t *testing.T, accounts map[string]*directoryFixture) *httptest.Server { + t.Helper() + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/compatibility" { + json.NewEncoder(w).Encode(sidecar.ProtocolInfo{ProtocolVersion: sidecar.ProtocolVersion}) + return + } + ns := r.Header.Get("X-Inline-Session-Namespace") + f := accounts[ns] + if f == nil { + t.Errorf("unscoped namespace %q", ns) + http.Error(w, "bad namespace", 400) + return + } + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, r.URL.Path) + if r.Header.Get("X-Inline-Protocol-Version") != "5" { + t.Error("missing protocol header") + } + if r.URL.Path == f.deny { + json.NewEncoder(w).Encode(map[string]any{"protocol_version": sidecar.ProtocolVersion, "outcome": map[string]any{"status": "error", "data": sidecar.Error{Category: "permission_denied", Message: "permission denied"}}}) + return + } + var typ string + var data any + switch r.URL.Path { + case "/rpc/dialogs": + typ = "dialogs" + data = f.dialogs + case "/rpc/state/account": + typ = "account_state" + data = f.state + case "/rpc/chat/participants": + var req sidecar.ChatParticipantsRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + } + f.chatReads = append(f.chatReads, req.ChatID) + typ = "chat_participants" + data = f.participants[req.ChatID] + default: + t.Errorf("unexpected directory RPC %s", r.URL.Path) + http.Error(w, "unexpected", 500) + return + } + json.NewEncoder(w).Encode(map[string]any{"protocol_version": sidecar.ProtocolVersion, "outcome": map[string]any{"status": "ok", "data": map[string]any{"type": typ, "data": data}}}) + })) + t.Cleanup(s.Close) + return s +} +func directoryClient(url, ns string) *InlineClient { + return &InlineClient{Sidecar: sidecar.NewClient(url).WithSessionNamespace(ns), UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ID: networkid.UserLoginID("login-" + ns)}}, AccountID: "1", users: map[int64]sidecar.UserRecord{999: dirUser(999, "unrelated")}} +} +func TestDirectoryAccountAccessAndFreshness(t *testing.T) { + f := &directoryFixture{state: sidecar.AccountStateSnapshot{ + Users: []sidecar.UserRecord{dirUser(2, "peer"), dirUser(3, "workspace"), dirUser(4, "removed"), dirUser(5, "unknown"), dirUser(6, "hidden"), dirUser(7, "deleted"), dirUser(8, "live"), dirUser(999, "old")}, + SpaceAccess: []sidecar.SpaceAccessRecord{{SpaceID: 10, Status: "accessible"}, {SpaceID: 11, Status: "removed"}, {SpaceID: 12, Status: "unknown"}}, + SpaceMembers: []sidecar.SpaceMemberRecord{{SpaceID: 10, UserID: 3}, {SpaceID: 11, UserID: 4}, {SpaceID: 12, UserID: 5}}, DeletedChatIDs: []int64{70}}, + dialogs: sidecar.DialogsPage{Dialogs: []sidecar.DialogRecord{ + {ChatID: 20, Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: dirPtr(int64(2))}}, + {ChatID: 60, ChatListHidden: dirPtr(true), Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: dirPtr(int64(6))}}, + {ChatID: 70, Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: dirPtr(int64(7))}}, + {ChatID: 80, Structure: &sidecar.DialogStructure{Kind: "channel", SpaceID: dirPtr(int64(11))}}, + }}, participants: map[int64]sidecar.ChatParticipantsPage{80: {Participants: []sidecar.ChatParticipantRecord{{UserID: 8}}, Users: []sidecar.UserRecord{dirUser(8, "live"), dirUser(999, "unrelated")}}}} + s := directoryServer(t, map[string]*directoryFixture{"a": f, "b": {state: sidecar.AccountStateSnapshot{Users: []sidecar.UserRecord{dirUser(2, "other-account")}}, dialogs: sidecar.DialogsPage{Dialogs: []sidecar.DialogRecord{{ChatID: 20, Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: dirPtr(int64(2))}}}}}}) + a := directoryClient(s.URL, "a") + d, err := a.loadDirectory(t.Context()) + if err != nil { + t.Fatal(err) + } + got, err := d.search("", 0, 100) + if err != nil { + t.Fatal(err) + } + ids := []int64{} + for _, u := range got { + ids = append(ids, u.Record.UserID) + } + if !reflect.DeepEqual(ids, []int64{2, 3, 8}) { + t.Fatalf("access leak/missing users: %v", ids) + } + if d.users[2].Fresh || d.users[3].Fresh || !d.users[8].Fresh { + t.Fatalf("wrong source freshness: %+v", d.users) + } + b, err := directoryClient(s.URL, "b").loadDirectory(t.Context()) + if err != nil { + t.Fatal(err) + } + if u, err := b.resolve("2"); err != nil || *u.Record.Username != "other-account" { + t.Fatalf("namespace collision: %+v %v", u, err) + } + if _, err = b.resolve("peer"); !errors.Is(err, errDirectoryNotFound) { + t.Fatalf("cross-account username leaked: %v", err) + } + if !reflect.DeepEqual(f.chatReads, []int64{80}) { + t.Fatalf("hidden/deleted/unknown dialog read: %v", f.chatReads) + } +} + +func TestDirectoryExcludedConversationDiagnosticsAndIndependentEvidence(t *testing.T) { + for _, excluded := range []string{"hidden", "deleted"} { + for _, evidence := range []string{"none", "workspace", "live"} { + t.Run(excluded+"/"+evidence, func(t *testing.T) { + f := &directoryFixture{ + state: sidecar.AccountStateSnapshot{Users: []sidecar.UserRecord{dirUser(42, "peer")}}, + dialogs: sidecar.DialogsPage{Dialogs: []sidecar.DialogRecord{{ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: dirPtr(int64(42))}}}}, + } + if excluded == "hidden" { + f.dialogs.Dialogs[0].ChatListHidden = dirPtr(true) + } else { + f.state.DeletedChatIDs = []int64{7} + } + switch evidence { + case "workspace": + f.state.SpaceAccess = []sidecar.SpaceAccessRecord{{SpaceID: 10, Status: "accessible"}} + f.state.SpaceMembers = []sidecar.SpaceMemberRecord{{SpaceID: 10, UserID: 42}} + case "live": + f.dialogs.Dialogs = append(f.dialogs.Dialogs, sidecar.DialogRecord{ChatID: 8, Structure: &sidecar.DialogStructure{Kind: "channel"}}) + f.participants = map[int64]sidecar.ChatParticipantsPage{8: {Participants: []sidecar.ChatParticipantRecord{{UserID: 42}}, Users: []sidecar.UserRecord{dirUser(42, "peer")}}} + } + s := directoryServer(t, map[string]*directoryFixture{"a": f}) + ic := directoryClient(s.URL, "a") + for _, query := range []string{"42", "@PEER"} { + got, err := ic.ResolveIdentifier(t.Context(), query, false) + if evidence == "none" { + if !errors.Is(err, errDirectoryNotFound) || got != nil || !strings.Contains(err.Error(), "eligible directory") || strings.Contains(err.Error(), "not found") { + t.Fatalf("excluded conversation misreported for %s: %+v %v", query, got, err) + } + } else if err != nil || got == nil || got.UserID != "42" { + t.Fatalf("independent %s evidence lost: %+v %v", evidence, got, err) + } + } + if evidence == "none" { + if _, err := ic.CreateChatWithGhost(t.Context(), &bridgev2.Ghost{Ghost: &database.Ghost{ID: "42"}}); !errors.Is(err, errDirectoryNotFound) { + t.Fatalf("ineligible peer caused creation: %v", err) + } + } + }) + } + } +} +func TestDirectoryRefreshDenial(t *testing.T) { + for _, path := range []string{"/rpc/dialogs", "/rpc/state/account", "/rpc/chat/participants"} { + t.Run(path, func(t *testing.T) { + f := &directoryFixture{deny: path, dialogs: sidecar.DialogsPage{Dialogs: []sidecar.DialogRecord{{ChatID: 8, Structure: &sidecar.DialogStructure{Kind: "channel"}}}}} + s := directoryServer(t, map[string]*directoryFixture{"a": f}) + d, err := directoryClient(s.URL, "a").loadDirectory(t.Context()) + if err == nil || d != nil || !strings.Contains(err.Error(), "permission denied") { + t.Fatalf("denial lost/fallback: %+v %v", d, err) + } + }) + } +} +func TestDirectoryCreationPolicy(t *testing.T) { + d := dirSnapshot(dirUser(2, "peer")) + d.spaces[10] = sidecar.SpaceRecord{SpaceID: 10, IsPublic: dirPtr(true)} + d.access[10] = "accessible" + d.members[10] = map[int64]bool{2: true} + parent := &networkid.PortalKey{ID: "space:10", Receiver: "login-a"} + for _, typ := range []string{"", "thread", "channel", "private_thread"} { + params := &bridgev2.GroupCreateParams{Type: typ, Participants: []networkid.UserID{"2", "2"}, Name: &event.RoomNameEventContent{Name: " hello "}, Topic: &event.TopicEventContent{Topic: "topic"}} + req, ids, err := validateThreadCreation(params, "login-a", d) + if err != nil || req.IsPublic || req.SpaceID != nil || len(req.Participants) != 1 || req.Participants[0].UserID != 2 || !reflect.DeepEqual(ids, []networkid.UserID{"2"}) || req.Title == nil || *req.Title != "hello" || req.Description == nil || *req.Description != "topic" { + t.Fatalf("home request %+v %v %v", req, ids, err) + } + params.Parent = parent + req, _, err = validateThreadCreation(params, "login-a", d) + if err != nil || req.SpaceID == nil || *req.SpaceID != 10 { + t.Fatalf("private workspace %+v %v", req, err) + } + } + req, ids, err := validateThreadCreation(&bridgev2.GroupCreateParams{Type: "public_thread", Parent: parent}, "login-a", d) + if err != nil || !req.IsPublic || *req.SpaceID != 10 || len(ids) != 0 { + t.Fatalf("public workspace %+v %v %v", req, ids, err) + } + bad := []*bridgev2.GroupCreateParams{nil, {}, {Type: "group_dm", Participants: []networkid.UserID{"2"}}, {Type: "direct"}, {Type: "mystery"}, {Type: "public_thread"}, {Type: "public_thread", Parent: parent, Participants: []networkid.UserID{"2"}}, {Participants: []networkid.UserID{"999"}}, {Participants: []networkid.UserID{"2"}, Username: "x"}, {Participants: []networkid.UserID{"2"}, Avatar: &event.RoomAvatarEventContent{}}, {Participants: []networkid.UserID{"2"}, Disappear: &event.BeeperDisappearingTimer{}}, {Parent: &networkid.PortalKey{ID: "space:10", Receiver: "login-b"}, Participants: []networkid.UserID{"2"}}, {Parent: &networkid.PortalKey{ID: "10", Receiver: "login-a"}, Participants: []networkid.UserID{"2"}}} + for i, p := range bad { + if _, _, err := validateThreadCreation(p, "login-a", d); err == nil { + t.Errorf("accepted invalid params %d: %+v", i, p) + } + } + d.access[10] = "removed" + if _, _, err := validateThreadCreation(&bridgev2.GroupCreateParams{Type: "public_thread", Parent: parent}, "login-a", d); err == nil { + t.Fatal("removed workspace accepted") + } +} +func TestDirectoryCreationPositiveIDsProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + id := hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64)) + space := hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64)) + d := dirSnapshot(dirUser(id, "user")) + d.spaces[space] = sidecar.SpaceRecord{SpaceID: space} + d.access[space] = "accessible" + d.members[space] = map[int64]bool{id: true} + p := &bridgev2.GroupCreateParams{Type: "private_thread", Parent: &networkid.PortalKey{ID: networkid.PortalID("space:" + strconv.FormatInt(space, 10)), Receiver: "login-a"}, Participants: []networkid.UserID{networkid.UserID(strconv.FormatInt(id, 10))}} + req, ids, err := validateThreadCreation(p, "login-a", d) + if err != nil || req.SpaceID == nil || *req.SpaceID != space || len(req.Participants) != 1 || req.Participants[0].UserID != id || len(ids) != 1 || string(ids[0]) != strconv.FormatInt(id, 10) { + ht.Fatalf("lost exact IDs: %+v %v %v", req, ids, err) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func TestDirectoryInvalidNumericNamesNeverBypassIDValidation(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + id := hegel.Draw(ht, hegel.Integers[int64](math.MinInt64, math.MaxInt64)) + if id > 0 { + return + } + q := strconv.FormatInt(id, 10) + d := dirSnapshot(dirUser(1, q)) + if _, err := d.resolve(q); !errors.Is(err, errDirectoryNotFound) { + ht.Fatalf("invalid numeric ID %q resolved through username: %v", q, err) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func TestDirectoryOverflowIDCannotBecomeUsername(t *testing.T) { + q := "9223372036854775808" + d := dirSnapshot(dirUser(1, q)) + if _, err := d.resolve(q); !errors.Is(err, errDirectoryNotFound) { + t.Fatalf("overflow ID resolved through username: %v", err) + } +} + +func TestDirectorySearchPaginationProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + ids := hegel.Draw(ht, hegel.Lists(hegel.Integers[int64](1, math.MaxInt64))) + if len(ids) > 200 { + ids = ids[:200] + } // Bound materialization, never the ID domain. + d := dirSnapshot() + expected := map[int64]bool{} + for _, id := range ids { + d.users[id] = directoryUser{Record: dirUser(id, "person")} + expected[id] = true + } + limit := hegel.Draw(ht, hegel.Integers(1, 100)) + var got []int64 + for offset := 0; ; offset += limit { + page, err := d.search("person", offset, limit) + if err != nil { + ht.Fatal(err) + } + if len(page) == 0 { + break + } + for _, u := range page { + got = append(got, u.Record.UserID) + } + } + if len(got) != len(expected) { + ht.Fatalf("pagination lost/duplicated people: %v %v", got, expected) + } + for i, id := range got { + if !expected[id] || i > 0 && got[i-1] >= id { + ht.Fatalf("unstable/foreign search results: %v", got) + } + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func TestDirectorySourceMembershipIsolationProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + id := hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64)) + status := hegel.Draw(ht, hegel.SampledFrom([]string{"accessible", "removed", "unknown", ""})) + f := &directoryFixture{state: sidecar.AccountStateSnapshot{Users: []sidecar.UserRecord{dirUser(id, "member")}, SpaceAccess: []sidecar.SpaceAccessRecord{{SpaceID: 1, Status: status}}, SpaceMembers: []sidecar.SpaceMemberRecord{{SpaceID: 1, UserID: id}}}} + // A second account has overlapping IDs but no discovery/access evidence. + b := &directoryFixture{state: sidecar.AccountStateSnapshot{Users: []sidecar.UserRecord{dirUser(id, "other")}}} + s := directoryServer(t, map[string]*directoryFixture{"a": f, "b": b}) + a, err := directoryClient(s.URL, "a").loadDirectory(t.Context()) + if err != nil { + ht.Fatal(err) + } + other, err := directoryClient(s.URL, "b").loadDirectory(t.Context()) + if err != nil { + ht.Fatal(err) + } + u, err := a.resolve(strconv.FormatInt(id, 10)) + if status == "accessible" { + if err != nil || u.Record.UserID != id || u.Fresh { + ht.Fatalf("accessible cached member lost: %+v %v", u, err) + } + } else if !errors.Is(err, errDirectoryNotFound) { + ht.Fatalf("inaccessible workspace member discovered: %+v %v", u, err) + } + if _, err = other.resolve(strconv.FormatInt(id, 10)); !errors.Is(err, errDirectoryNotFound) { + ht.Fatalf("account scope leak: %v", err) + } + s.Close() + }, hegel.WithTestCases(100), hegel.WithDatabase(t.TempDir())) +} + +func TestDirectoryBoundedRefreshAndFrameworkInterfaces(t *testing.T) { + dialogs := []sidecar.DialogRecord{} + for i := int64(1); i <= 20; i++ { + dialogs = append(dialogs, sidecar.DialogRecord{ChatID: i, Structure: &sidecar.DialogStructure{Kind: "channel"}}) + } + f := &directoryFixture{dialogs: sidecar.DialogsPage{Dialogs: dialogs, NextCursor: "repeat"}} + s := directoryServer(t, map[string]*directoryFixture{"a": f}) + d, err := directoryClient(s.URL, "a").loadDirectory(t.Context()) + if err != nil { + t.Fatal(err) + } + if !d.incomplete { + t.Fatal("bounded refresh claimed complete") + } + reads, pages := 0, 0 + for _, path := range f.calls { + if path == "/rpc/dialogs" { + pages++ + } + if path == "/rpc/chat/participants" { + reads++ + } + } + if pages > directoryMaxPages || reads != directoryParticipantBudget { + t.Fatalf("unbounded refresh %d pages %d reads", pages, reads) + } + f.dialogs = sidecar.DialogsPage{Dialogs: []sidecar.DialogRecord{{ChatID: 30, Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: dirPtr(int64(2))}}}, Users: []sidecar.UserRecord{dirUser(2, "peer")}} + ic := directoryClient(s.URL, "a") + got, err := ic.SearchUsers(t.Context(), "peer") + if err != nil || len(got) != 1 || got[0].UserID != "2" || got[0].UserInfo == nil { + t.Fatalf("search interface: %+v %v", got, err) + } + got, err = ic.GetContactList(t.Context()) + if err != nil || len(got) != 1 || got[0].UserID != "2" { + t.Fatalf("contact interface: %+v %v", got, err) + } +} + +func TestDirectoryCreationPreflightAndSourceDenial(t *testing.T) { + f := &directoryFixture{deny: "/rpc/chat/create-thread"} + s := directoryServer(t, map[string]*directoryFixture{"a": f}) + ic := directoryClient(s.URL, "a") + d := dirSnapshot(dirUser(2, "peer")) + attempt := func(p *bridgev2.GroupCreateParams) error { + req, _, err := validateThreadCreation(p, ic.UserLogin.ID, d) + if err != nil { + return err + } + _, err = ic.Sidecar.CreateThread(t.Context(), req) + return err + } + for _, p := range []*bridgev2.GroupCreateParams{{Type: "group_dm"}, {Participants: []networkid.UserID{"999"}}, {Participants: []networkid.UserID{"2"}, Avatar: &event.RoomAvatarEventContent{}}, {Type: "public_thread"}, {Participants: []networkid.UserID{"2"}, Parent: &networkid.PortalKey{ID: "space:1", Receiver: "login-b"}}} { + if err := attempt(p); err == nil { + t.Fatal("invalid creation succeeded") + } + } + if len(f.calls) != 0 { + t.Fatalf("bad preflight issued RPC: %v", f.calls) + } + err := attempt(&bridgev2.GroupCreateParams{Type: "channel", Participants: []networkid.UserID{"2"}}) + var denied *sidecar.Error + if !errors.As(err, &denied) || denied.Category != "permission_denied" || denied.Message != "permission denied" { + t.Fatalf("source authorization denial lost: %v", err) + } + if !reflect.DeepEqual(f.calls, []string{"/rpc/chat/create-thread"}) { + t.Fatalf("unexpected create flow %v", f.calls) + } +} + +func TestDirectoryZeroIDRegression(t *testing.T) { + if _, err := dirSnapshot(dirUser(1, "0")).resolve("0"); !errors.Is(err, errDirectoryNotFound) { + t.Fatalf("zero numeric ID became username: %v", err) + } +} + +func TestDirectoryEmptyUnknownAndMissingRecords(t *testing.T) { + for _, f := range []*directoryFixture{ + {}, + {state: sidecar.AccountStateSnapshot{Users: []sidecar.UserRecord{dirUser(2, "unrelated")}}}, + {dialogs: sidecar.DialogsPage{Dialogs: []sidecar.DialogRecord{{ChatID: 2, Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: dirPtr(int64(2))}}}}}, + {state: sidecar.AccountStateSnapshot{Users: []sidecar.UserRecord{dirUser(2, "unknown")}}, dialogs: sidecar.DialogsPage{Dialogs: []sidecar.DialogRecord{{ChatID: 2, PeerUserID: dirPtr(int64(2))}}}}, + } { + s := directoryServer(t, map[string]*directoryFixture{"a": f}) + d, err := directoryClient(s.URL, "a").loadDirectory(t.Context()) + if err != nil { + t.Fatal(err) + } + if _, err = d.resolve("2"); !errors.Is(err, errDirectoryNotFound) { + t.Fatalf("incomplete snapshot synthesized user: %v", err) + } + s.Close() + } + ic := directoryClient("http://127.0.0.1:1", "") + if _, err := ic.loadDirectory(t.Context()); err == nil || !strings.Contains(err.Error(), "scoped login") { + t.Fatalf("unscoped read allowed: %v", err) + } +} + +func TestDirectoryCreationRejectDirectionProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + id := hegel.Draw(ht, hegel.Integers[int64](math.MinInt64, math.MaxInt64)) + status := hegel.Draw(ht, hegel.SampledFrom([]string{"removed", "unknown", ""})) + d := dirSnapshot(dirUser(id, "cached-admin")) + d.spaces[10] = sidecar.SpaceRecord{SpaceID: 10, Creator: true} + d.access[10] = status + d.members[10] = map[int64]bool{id: true} + p := &bridgev2.GroupCreateParams{Type: "private_thread", Parent: &networkid.PortalKey{ID: "space:10", Receiver: "login-a"}, Participants: []networkid.UserID{networkid.UserID(strconv.FormatInt(id, 10))}} + req, ids, err := validateThreadCreation(p, "login-a", d) + if err == nil || req.SpaceID != nil || len(req.Participants) != 0 || len(ids) != 0 { + ht.Fatalf("cached creator/inaccessible workspace granted create: %+v %v %v", req, ids, err) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func TestDirectoryPaginationRejectsInvalidBoundsProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + offset := hegel.Draw(ht, hegel.Integers(math.MinInt, math.MaxInt)) + limit := hegel.Draw(ht, hegel.Integers(math.MinInt, math.MaxInt)) + d := dirSnapshot(dirUser(2, "peer")) + got, err := d.search("", offset, limit) + valid := offset >= 0 && limit >= 1 && limit <= 100 + if valid { + if err != nil { + ht.Fatal(err) + } + if offset == 0 && (len(got) != 1 || got[0].Record.UserID != 2) { + ht.Fatalf("valid page lost peer: %v", got) + } + if offset > 0 && len(got) != 0 { + ht.Fatalf("offset beyond end returned people: %v", got) + } + } else if err == nil { + ht.Fatalf("invalid bounds accepted: %d %d", offset, limit) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func TestDirectoryLargeSnapshot(t *testing.T) { + d := dirSnapshot() + for id := int64(1); id <= 100_000; id++ { + d.users[id] = directoryUser{Record: sidecar.UserRecord{UserID: id}} + } + got, err := d.search("", 99_900, 100) + if err != nil || len(got) != 100 || got[0].Record.UserID != 99_901 || got[99].Record.UserID != 100_000 { + t.Fatalf("large snapshot page: %v %v", len(got), err) + } + u, err := d.resolve("100000") + if err != nil || u.Record.UserID != 100_000 { + t.Fatalf("large snapshot resolve: %+v %v", u, err) + } + req, ids, err := validateThreadCreation(&bridgev2.GroupCreateParams{Participants: []networkid.UserID{"100000"}}, "login-a", d) + if err != nil || len(req.Participants) != 1 || req.Participants[0].UserID != 100_000 || !reflect.DeepEqual(ids, []networkid.UserID{"100000"}) { + t.Fatalf("large snapshot creation: %+v %v %v", req, ids, err) + } +} + +var _ bridgev2.UserSearchingNetworkAPI = (*InlineClient)(nil) +var _ bridgev2.ContactListingNetworkAPI = (*InlineClient)(nil) + +func TestDirectoryNumericSearchNeverFallsBackToNames(t *testing.T) { + for _, q := range []string{"0", "-1", "9223372036854775808"} { + d := dirSnapshot(dirUser(1, q)) + got, err := d.search(q, 0, 100) + if err != nil || len(got) != 0 { + t.Fatalf("invalid numeric query %q matched names: %+v %v", q, got, err) + } + } +} diff --git a/pkg/connector/hierarchy.go b/pkg/connector/hierarchy.go new file mode 100644 index 0000000..ba751bd --- /dev/null +++ b/pkg/connector/hierarchy.go @@ -0,0 +1,320 @@ +package connector + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "slices" + "strings" + "time" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/matrix" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/bridgev2/simplevent" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +func (ic *InlineClient) refreshWorkspaceSnapshot(ctx context.Context) error { + snapshot, err := ic.Sidecar.AccountState(ctx) + if err != nil { + return err + } + ic.cacheWorkspaceSnapshot(snapshot) + return nil +} +func (ic *InlineClient) cacheWorkspaceSnapshot(snapshot *sidecar.AccountStateSnapshot) bool { + ic.cacheUsers(snapshot.Users) + ic.mu.Lock() + defer ic.mu.Unlock() + oldSpaces, oldAccess, oldComplete := ic.spaces, ic.spaceAccess, ic.spaceAccessComplete + ic.spaces = make(map[int64]sidecar.SpaceRecord, len(snapshot.Spaces)) + ic.spaceAccess = make(map[int64]string, len(snapshot.SpaceAccess)) + for _, s := range snapshot.Spaces { + if s.SpaceID > 0 { + ic.spaces[s.SpaceID] = s + } + } + for _, a := range snapshot.SpaceAccess { + if a.SpaceID > 0 { + ic.spaceAccess[a.SpaceID] = a.Status + } + } + ic.spaceAccessComplete = snapshot.SpaceAccessComplete + changed := !reflect.DeepEqual(oldSpaces, ic.spaces) || !reflect.DeepEqual(oldAccess, ic.spaceAccess) || oldComplete != ic.spaceAccessComplete + if changed { + ic.workspaceRevision++ + } + return changed +} +func (ic *InlineClient) desiredWorkspace(d sidecar.DialogRecord) (string, bool) { + kind := classifyDialog(d) + if kind == roomUnknown { + return "", false + } + if kind == roomDirect || d.Structure.SpaceID == nil { + return "", true + } + space := *d.Structure.SpaceID + ic.mu.RLock() + defer ic.mu.RUnlock() + access, observed := ic.spaceAccess[space] + if access == "removed" || (!observed && ic.spaceAccessComplete) { + return "", true + } + if access != "accessible" { + return "", false + } // Unknown access neither creates navigation nor proves an owned link should detach. + if _, ok := ic.spaces[space]; !ok { + return "", false + } + return fmt.Sprintf("space:%d", space), true +} +func (ic *InlineClient) workspaceChatInfo(space sidecar.SpaceRecord) *bridgev2.ChatInfo { + rt := database.RoomTypeSpace + name := space.Name + return &bridgev2.ChatInfo{Name: &name, Type: &rt, Members: ic.chatMemberListForKnownSelf(context.Background()), CanBackfill: false} +} +func (ic *InlineClient) ensureWorkspace(ctx context.Context, d sidecar.DialogRecord) error { + desired, known := ic.desiredWorkspace(d) + if !known || desired == "" { + return nil + } + ic.mu.RLock() + space := ic.spaces[*d.Structure.SpaceID] + ic.mu.RUnlock() + result := ic.UserLogin.Bridge.QueueRemoteEvent(ic.UserLogin, &simplevent.ChatResync{ + EventMeta: simplevent.EventMeta{Type: bridgev2.RemoteEventChatResync, PortalKey: networkid.PortalKey{ID: networkid.PortalID(desired), Receiver: ic.UserLogin.ID}, CreatePortal: true}, + ChatInfo: ic.workspaceChatInfo(space), + }) + return eventHandlingResultError("workspace resync", result) +} +func (ic *InlineClient) projectDialogInLane(ctx context.Context, portal *bridgev2.Portal, d sidecar.DialogRecord) { + if portal == nil || portal.Bridge == nil { + return + } + s, k, err := ic.projectionJournal(ctx, string(portal.ID)) + if err != nil { + return + } + p, err := s.load(ctx, k) + if err != nil { + return + } + desired, known := ic.desiredWorkspace(d) + p.NotificationPolicy = notificationPolicyDegraded + if portal.ParentKey.ID != "" && !strings.HasPrefix(string(portal.ParentKey.ID), "space:") && p.Migration == nil { + p.ParentError = "unowned_parent_preserved" + _ = s.save(ctx, k, &p) + ic.reconcileProjection(ctx, portal) + return + } + if p.Migration != nil && p.Migration.Phase == "rolled_back" { + portal.RoomType = p.Migration.Before.Kind + ic.reconcileProjection(ctx, portal) + return + } + if p.Migration != nil && p.Migration.Phase == "conflict" { + // UpdateInfo may have assigned the source type before our hook. Preserve + // the stored value while the conflict waits for explicit reselection. + if stored, err := portal.Bridge.DB.Portal.GetByKey(ctx, portal.PortalKey); err == nil && stored != nil { + portal.RoomType = stored.RoomType + } + ic.reconcileProjection(ctx, portal) + return + } + // During an explicit migration/rollback, its journal owns presentation. + if p.Migration != nil && p.Migration.Phase != "applied" && p.Migration.Phase != "rolled_back" && p.Migration.Phase != "conflict" { + ic.runMigrationInLane(ctx, portal, s, k, &p) + return + } + if known { + p.DesiredParent = desired + p.ParentKnown = true + p.ParentPending = true + } + if portal.Parent != nil && strings.HasPrefix(string(portal.Parent.ID), "space:") && portal.Parent.Receiver == portal.Receiver && portal.Parent.MXID != "" && !slices.Contains(p.OwnedParents, portal.Parent.MXID) { + p.OwnedParents = append(p.OwnedParents, portal.Parent.MXID) + } + if err = s.save(ctx, k, &p); err != nil { + return + } // durable old and desired parent BEFORE external effects + if known && p.ParentKnown && p.ParentPending && portal.MXID != "" { + if err = ic.repairHierarchy(ctx, portal, s, k, &p); err != nil { + p.ParentError = "hierarchy_write_or_readback_failed" + _ = s.save(ctx, k, &p) + } + } + ic.reconcileProjection(ctx, portal) +} + +// hierarchyWire writes and reads the two independent Matrix state events. +// It deliberately avoids v0.28's updateParent, which retains Parent on detach. +type hierarchyWire interface { + put(context.Context, id.RoomID, event.Type, string, any) error + get(context.Context, id.RoomID, event.Type, string, any) error +} +type matrixHierarchyWire struct{ raw *matrix.ASIntent } + +func (w matrixHierarchyWire) put(ctx context.Context, room id.RoomID, typ event.Type, key string, value any) error { + _, err := w.raw.SendState(ctx, room, typ, key, &event.Content{Parsed: value}, time.Time{}) + return err +} +func (w matrixHierarchyWire) get(ctx context.Context, room id.RoomID, typ event.Type, key string, value any) error { + return w.raw.Matrix.Client.StateEvent(ctx, room, typ, key, value) +} + +func ensureHierarchyLink[T any](ctx context.Context, w hierarchyWire, room id.RoomID, typ event.Type, key string, desired T, correct func(T) bool) error { + var current T + err := w.get(ctx, room, typ, key, ¤t) + if err != nil && !errors.Is(err, mautrix.MNotFound) { + return err + } + if correct(current) { + return nil + } + if err = w.put(ctx, room, typ, key, desired); err != nil { + return err + } + var confirmed T + err = w.get(ctx, room, typ, key, &confirmed) + if err != nil && !errors.Is(err, mautrix.MNotFound) { + return err + } + if !correct(confirmed) { + return errors.New("hierarchy readback differs") + } + return nil +} +func reconcileHierarchyLinks(ctx context.Context, w hierarchyWire, child id.RoomID, owned []id.RoomID, target id.RoomID, via string) error { + for _, old := range owned { + if old == target { + continue + } + if err := ensureHierarchyLink(ctx, w, old, event.StateSpaceChild, string(child), event.SpaceChildEventContent{}, func(c event.SpaceChildEventContent) bool { return len(c.Via) == 0 }); err != nil { + return err + } + if err := ensureHierarchyLink(ctx, w, child, event.StateSpaceParent, string(old), event.SpaceParentEventContent{}, func(p event.SpaceParentEventContent) bool { return len(p.Via) == 0 && !p.Canonical }); err != nil { + return err + } + } + if target != "" { + if err := ensureHierarchyLink(ctx, w, target, event.StateSpaceChild, string(child), event.SpaceChildEventContent{Via: []string{via}}, func(c event.SpaceChildEventContent) bool { return slices.Contains(c.Via, via) }); err != nil { + return err + } + if err := ensureHierarchyLink(ctx, w, child, event.StateSpaceParent, string(target), event.SpaceParentEventContent{Via: []string{via}, Canonical: true}, func(p event.SpaceParentEventContent) bool { return slices.Contains(p.Via, via) && p.Canonical }); err != nil { + return err + } + } + return nil +} + +func (ic *InlineClient) repairHierarchy(ctx context.Context, portal *bridgev2.Portal, s *projectionStore, k projectionKey, p *projectionState) error { + return ic.repairHierarchyState(ctx, portal, s, k, p, true) +} +func (ic *InlineClient) repairHierarchyState(ctx context.Context, portal *bridgev2.Portal, s *projectionStore, k projectionKey, p *projectionState, writeLinks bool) error { + raw, ok := portal.Bridge.Bot.(*matrix.ASIntent) + if !ok || raw.Matrix == nil { + return errors.New("hierarchy readback unsupported") + } + var parent *bridgev2.Portal + var target id.RoomID + if p.DesiredParent != "" { + var err error + parent, err = portal.Bridge.GetExistingPortalByKey(ctx, networkid.PortalKey{ID: networkid.PortalID(p.DesiredParent), Receiver: portal.Receiver}) + if err != nil { + return err + } + if parent == nil || parent.MXID == "" || parent.RoomType != database.RoomTypeSpace { + return errors.New("workspace not yet created") + } + target = parent.MXID + if !slices.Contains(p.OwnedParents, target) { + p.OwnedParents = append(p.OwnedParents, target) + if err = s.save(ctx, k, p); err != nil { + return err + } + } + } + via := portal.Bridge.Matrix.ServerName() + var err error + if writeLinks { + if err = reconcileHierarchyLinks(ctx, matrixHierarchyWire{raw}, portal.MXID, p.OwnedParents, target, via); err != nil { + return err + } + } + portal.Parent = parent + portal.ParentKey = networkid.PortalKey{} + portal.InSpace = parent != nil + if parent != nil { + portal.ParentKey = parent.PortalKey + } + if verifyBridgeInfo(ctx, raw, portal) != nil { + portal.UpdateBridgeInfo(ctx) + } + if err = portal.Save(ctx); err != nil { + return err + } + stored, err := portal.Bridge.DB.Portal.GetByKey(ctx, portal.PortalKey) + if err != nil { + return err + } + if stored == nil || stored.ParentKey != portal.ParentKey || stored.InSpace != portal.InSpace { + return errors.New("portal parent readback failed") + } + if err = verifyBridgeInfo(ctx, raw, portal); err != nil { + return err + } + p.OwnedParents = nil + if target != "" { + p.OwnedParents = []id.RoomID{target} + } + p.AppliedParent = p.DesiredParent + p.ParentPending = false + p.ParentError = "" + return s.save(ctx, k, p) +} +func verifyBridgeInfo(ctx context.Context, raw *matrix.ASIntent, p *bridgev2.Portal) error { + state, err := raw.Matrix.Client.State(ctx, p.MXID) + if err != nil { + return err + } + for _, typ := range []event.Type{event.StateBridge, event.StateHalfShotBridge} { + found := false + for _, evt := range state[typ] { + var info event.BridgeEventContent + encoded, err := evt.Content.MarshalJSON() + if err != nil { + return err + } + if err = json.Unmarshal(encoded, &info); err != nil { + return err + } + if info.Channel.ID == string(p.ID) && info.Channel.Receiver == string(p.Receiver) { + if info.BeeperRoomTypeV2 != string(p.RoomType) { + return errors.New("bridge room classification readback differs") + } + if p.RoomType == database.RoomTypeDefault && info.BeeperRoomType != "" { + return errors.New("legacy room classification remains") + } + if p.Parent == nil && info.Network != nil { + return errors.New("stale bridge network after detach") + } + if p.Parent != nil && (info.Network == nil || info.Network.ID != string(p.Parent.ID)) { + return errors.New("bridge parent readback differs") + } + found = true + } + } + if !found { + return errors.New("bridge info missing") + } + } + return nil +} diff --git a/pkg/connector/hierarchy_test.go b/pkg/connector/hierarchy_test.go new file mode 100644 index 0000000..0c0ba7b --- /dev/null +++ b/pkg/connector/hierarchy_test.go @@ -0,0 +1,306 @@ +package connector + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "hegel.dev/go/hegel" + "math" + "path/filepath" + "strings" + "testing" + + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +type fixtureHierarchy struct { + state map[string][]byte + failAt, calls int + writes int +} + +func (f *fixtureHierarchy) key(room id.RoomID, typ event.Type, key string) string { + return fmt.Sprintf("%s/%s/%s", room, typ.Type, key) +} +func (f *fixtureHierarchy) put(_ context.Context, room id.RoomID, typ event.Type, key string, value any) error { + f.calls++ + f.writes++ + if f.failAt == f.calls { + return errors.New("injected state write failure") + } + raw, err := json.Marshal(value) + if err == nil { + f.state[f.key(room, typ, key)] = raw + } + return err +} +func (f *fixtureHierarchy) get(_ context.Context, room id.RoomID, typ event.Type, key string, value any) error { + f.calls++ + if f.failAt == f.calls { + return errors.New("injected readback failure") + } + raw := f.state[f.key(room, typ, key)] + if raw == nil { + return mautrix.MNotFound + } + return json.Unmarshal(raw, value) +} +func TestHierarchyBothSidesMoveDetachAndFailureRecovery(t *testing.T) { + for fail := 0; fail <= 8; fail++ { + t.Run(fmt.Sprint(fail), func(t *testing.T) { + f := &fixtureHierarchy{state: map[string][]byte{}} + ctx := t.Context() + child := id.RoomID("!child:test") + a := id.RoomID("!a:test") + b := id.RoomID("!b:test") + if err := reconcileHierarchyLinks(ctx, f, child, nil, a, "test"); err != nil { + t.Fatal(err) + } + f.calls = 0 + f.failAt = fail + err := reconcileHierarchyLinks(ctx, f, child, []id.RoomID{a, b}, b, "test") + if fail > 0 && err == nil { + t.Fatalf("boundary %d incorrectly completed", fail) + } + // Replay after a fresh owner loads the same remote state. Every partial side + // effect is repaired; no external-success callback is treated as a checkpoint. + restarted := &fixtureHierarchy{state: f.state} + if err := reconcileHierarchyLinks(ctx, restarted, child, []id.RoomID{a, b}, b, "test"); err != nil { + t.Fatal(err) + } + var old event.SpaceParentEventContent + var current event.SpaceParentEventContent + if err := restarted.get(ctx, child, event.StateSpaceParent, string(a), &old); err != nil { + t.Fatal(err) + } + if err := restarted.get(ctx, child, event.StateSpaceParent, string(b), ¤t); err != nil { + t.Fatal(err) + } + if old.Canonical || len(old.Via) != 0 || !current.Canonical || len(current.Via) == 0 { + t.Fatalf("move old=%+v current=%+v", old, current) + } + if err := reconcileHierarchyLinks(ctx, restarted, child, []id.RoomID{a, b}, "", "test"); err != nil { + t.Fatal(err) + } + current = event.SpaceParentEventContent{} + if err := restarted.get(ctx, child, event.StateSpaceParent, string(b), ¤t); err != nil { + t.Fatal(err) + } + if current.Canonical || len(current.Via) > 0 { + t.Fatal("canonical parent survived detach") + } + }) + } +} +func TestMigrationRollbackCompareAndSet(t *testing.T) { + before := presentationValue{RoomID: "!same:test", Kind: database.RoomTypeGroupDM, Parent: "space:1"} + applied := presentationValue{RoomID: "!same:test", Kind: database.RoomTypeDefault, Parent: "space:2"} + m := &presentationMigration{Before: before, Applied: applied} + if !migrationRollbackAllowed(applied, m) { + t.Fatal("own applied value rejected") + } + changed := applied + changed.Parent = "space:3" + if migrationRollbackAllowed(changed, m) { + t.Fatal("rollback overwrites user parent choice") + } + changed = applied + changed.Kind = database.RoomTypeDM + if migrationRollbackAllowed(changed, m) { + t.Fatal("rollback overwrites later room classification") + } + changed = applied + changed.RoomID = "!replacement:test" + if migrationRollbackAllowed(changed, m) { + t.Fatal("rollback targets different Matrix room") + } +} +func TestWorkspaceUnknownAndRemovedNeverLink(t *testing.T) { + d := policyDialog(t, `{"chat_id":7,"space_id":1,"structure":{"kind":"channel","space_id":1,"peer_user_id":null,"parent_chat_id":null,"parent_message_id":null}}`) + client := &InlineClient{} + if parent, _ := client.desiredWorkspace(d); parent != "" { + t.Fatal("unloaded access linked") + } + client.spaces = map[int64]sidecar.SpaceRecord{1: {SpaceID: 1, Name: "private"}} + for _, status := range []string{"unknown", "removed", ""} { + client.spaceAccess = map[int64]string{1: status} + if parent, _ := client.desiredWorkspace(d); parent != "" { + t.Fatalf("%s linked stale workspace", status) + } + } + client.spaceAccess = map[int64]string{1: "accessible"} + if parent, known := client.desiredWorkspace(d); !known || parent != "space:1" { + t.Fatal("accessible source-backed workspace not linked") + } +} + +func TestWorkspaceAccessChangesInvalidatePresentation(t *testing.T) { + ic := &InlineClient{} + snap := &sidecar.AccountStateSnapshot{Spaces: []sidecar.SpaceRecord{{SpaceID: 1, Name: "one"}}, SpaceAccess: []sidecar.SpaceAccessRecord{{SpaceID: 1, Status: "accessible"}}} + if !ic.cacheWorkspaceSnapshot(snap) { + t.Fatal("initial workspace snapshot did not change") + } + if ic.cacheWorkspaceSnapshot(snap) { + t.Fatal("unchanged snapshot reprojected") + } + snap.SpaceAccess[0].Status = "removed" + if !ic.cacheWorkspaceSnapshot(snap) { + t.Fatal("access-only loss did not invalidate hierarchy") + } + snap.Spaces[0].Name = "renamed" + if !ic.cacheWorkspaceSnapshot(snap) { + t.Fatal("name-only change did not invalidate workspace") + } +} + +func TestMigrationMixedFieldsResumeButThirdValuesConflict(t *testing.T) { + before := presentationValue{RoomID: "!same:test", Kind: database.RoomTypeGroupDM, Parent: "space:1"} + after := presentationValue{RoomID: "!same:test", Kind: database.RoomTypeDefault, Parent: "space:2"} + for _, kind := range []database.RoomType{before.Kind, after.Kind} { + for _, parent := range []string{before.Parent, after.Parent} { + if !migrationValuesCompatible(presentationValue{RoomID: before.RoomID, Kind: kind, Parent: parent}, before, after) { + t.Fatal("partial own effect cannot resume") + } + } + } + if migrationValuesCompatible(presentationValue{RoomID: before.RoomID, Kind: database.RoomTypeDM, Parent: after.Parent}, before, after) { + t.Fatal("third kind overwritten") + } + if migrationValuesCompatible(presentationValue{RoomID: before.RoomID, Kind: after.Kind, Parent: "space:3"}, before, after) { + t.Fatal("third parent overwritten") + } +} +func TestMigrationRemoteGuardRejectsUserEditWithUnchangedCache(t *testing.T) { + before := map[string]string{"kind": "group_dm", "parent": "space:1"} + after := map[string]string{"kind": "", "parent": "space:2"} + if !migrationWireCompatible(map[string]string{"kind": "", "parent": "space:1"}, before, after) { + t.Fatal("crash intermediate rejected") + } + for _, changed := range []map[string]string{{"kind": "", "parent": "space:3"}, {"kind": "", "parent": "space:2", "parent:!manual:test": "linked"}} { + if migrationWireCompatible(changed, before, after) { + t.Fatal("user Matrix change overwritten despite unchanged local cache") + } + } + if migrationWireCompatible(after, nil, nil) { + t.Fatal("missing journal permits mutation") + } +} + +func TestMigrationRollbackRestartsAtEveryLinkBoundary(t *testing.T) { + for fail := 1; fail <= 12; fail++ { + t.Run(fmt.Sprint(fail), func(t *testing.T) { + child, a, b := id.RoomID("!child:test"), id.RoomID("!a:test"), id.RoomID("!b:test") + before := map[string]string{"parent:" + string(a): `{"canonical":true,"via":["test","extra"]}`, "child:" + string(a): `{"suggested":true,"via":["test","extra"]}`, "parent:" + string(b): `{}`, "child:" + string(b): `{}`} + f := &fixtureHierarchy{state: map[string][]byte{}} + if err := reconcileHierarchyLinks(t.Context(), f, child, []id.RoomID{a, b}, b, "test"); err != nil { + t.Fatal(err) + } + store := fixtureJournal(t, filepath.Join(t.TempDir(), "migration.db")) + key := projectionKey{Bridge: "b", User: "@u:test", Login: "1", Receiver: "1", Chat: "7"} + p, _ := store.load(t.Context(), key) + p.ReadMaxID = 77 + p.ReadApplied = 33 + applied := expectedMigrationWire(before, presentationValue{}, []id.RoomID{a, b}, b, "test") + p.Migration = &presentationMigration{AppliedWire: applied, Phase: "rolling_back", Before: presentationValue{RoomID: child, Kind: database.RoomTypeGroupDM, Parent: "space:1"}, Applied: presentationValue{RoomID: child, Kind: database.RoomTypeDefault, Parent: "space:2"}, BeforeWire: before, Parents: []id.RoomID{a, b}} + if err := store.save(t.Context(), key, &p); err != nil { + t.Fatal(err) + } + f.calls = 0 + f.failAt = fail + if err := restoreMigrationLinks(t.Context(), f, child, []id.RoomID{a, b}, before, applied); err == nil { + t.Fatal("injected rollback failure was checkpointed") + } + for field, original := range before { + room, typ, key := child, event.StateSpaceParent, strings.TrimPrefix(field, "parent:") + if strings.HasPrefix(field, "child:") { + room, typ, key = id.RoomID(strings.TrimPrefix(field, "child:")), event.StateSpaceChild, string(child) + } + var content map[string]any + if err := (&fixtureHierarchy{state: f.state}).get(t.Context(), room, typ, key, &content); err != nil && !errors.Is(err, mautrix.MNotFound) { + t.Fatal(err) + } + value := "{}" + if content != nil { + value = migrationJSON(content) + } + if value != original && value != applied[field] { + t.Fatal("bridge created unjournaled intermediate", field, value) + } + } + // Crash after any put/read: journal still says rolling_back. A fresh owner + // restores exactly the saved link values, including previous metadata. + loaded, err := store.load(t.Context(), key) + if err != nil || loaded.Migration.Phase != "rolling_back" || loaded.ReadMaxID != 77 || loaded.ReadApplied != 33 { + t.Fatal("rollback changed read state or lost phase", err) + } + restarted := &fixtureHierarchy{state: f.state} + if err = restoreMigrationLinks(t.Context(), restarted, child, loaded.Migration.Parents, loaded.Migration.BeforeWire, loaded.Migration.AppliedWire); err != nil { + t.Fatal(err) + } + for _, link := range []struct { + room id.RoomID + typ event.Type + key, want string + }{{child, event.StateSpaceParent, string(a), before["parent:"+string(a)]}, {a, event.StateSpaceChild, string(child), before["child:"+string(a)]}, {child, event.StateSpaceParent, string(b), "{}"}, {b, event.StateSpaceChild, string(child), "{}"}} { + var got map[string]any + if err = restarted.get(t.Context(), link.room, link.typ, link.key, &got); err != nil || migrationJSON(got) != link.want { + t.Fatal("rollback did not restore selected fields", err) + } + } + }) + } +} + +func TestMigrationCrashStatesHaveOnlyBeforeOrAppliedFields(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + oldKind := hegel.Draw(ht, hegel.SampledFrom([]string{"", "dm", "group_dm"})) + oldParent := fmt.Sprintf("space:%d", hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64))) + newParent := fmt.Sprintf("space:%d", hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64))) + before := map[string]string{"kind": oldKind, "parent": oldParent} + after := map[string]string{"kind": "", "parent": newParent} + current := map[string]string{"kind": oldKind, "parent": oldParent} + if hegel.Draw(ht, hegel.Booleans()) { + current["kind"] = after["kind"] + } + if hegel.Draw(ht, hegel.Booleans()) { + current["parent"] = after["parent"] + } + if !migrationWireCompatible(current, before, after) { + ht.Fatal("own partial write cannot resume") + } + current["parent"] = "space:third" + if migrationWireCompatible(current, before, after) { + ht.Fatal("user parent choice allowed to overwrite") + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func TestHierarchyStableResyncDoesNotEmitMoreState(t *testing.T) { + f := &fixtureHierarchy{state: map[string][]byte{}} + child, parent := id.RoomID("!child:test"), id.RoomID("!parent:test") + if err := reconcileHierarchyLinks(t.Context(), f, child, []id.RoomID{parent}, parent, "test"); err != nil { + t.Fatal(err) + } + writes := f.writes + if err := reconcileHierarchyLinks(t.Context(), f, child, []id.RoomID{parent}, parent, "test"); err != nil { + t.Fatal(err) + } + if f.writes != writes { + t.Fatal("unchanged hierarchy resync emitted Matrix state", f.writes-writes) + } + if err := reconcileHierarchyLinks(t.Context(), f, child, []id.RoomID{parent}, "", "test"); err != nil { + t.Fatal(err) + } + writes = f.writes + if err := reconcileHierarchyLinks(t.Context(), f, child, []id.RoomID{parent}, "", "test"); err != nil { + t.Fatal(err) + } + if f.writes != writes { + t.Fatal("unchanged detach emitted Matrix state") + } +} diff --git a/pkg/connector/history.go b/pkg/connector/history.go new file mode 100644 index 0000000..38fa7c9 --- /dev/null +++ b/pkg/connector/history.go @@ -0,0 +1,182 @@ +package connector + +import ( + "context" + "errors" + "strconv" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/bridgev2/simplevent" +) + +type historicalPage struct { + ChatID int64 + Records []sidecar.MessageRecord + CatchupAfter *int64 +} + +// A completion hook can run after a fully deduplicated (or truncated) batch. +// Only actual non-fake mappings for every requested record prove delivery. +func persistedHistoryPrefix(key networkid.PortalKey, records []sidecar.MessageRecord, lookup func(networkid.MessageID) (*database.Message, error)) int64 { + var maximum int64 + for _, record := range records { + if record.ChatID <= 0 || record.MessageID <= 0 || makePortalID(strconv.FormatInt(record.ChatID, 10)) != key.ID { + return 0 + } + row, err := lookup(makeMessageID(record.ChatID, record.MessageID)) + if err != nil || row == nil || row.MXID == "" || row.HasFakeMXID() || row.Room != key { + return 0 + } + if record.MessageID > maximum { + maximum = record.MessageID + } + } + return maximum +} + +func (ic *InlineClient) persistHistoricalMappings(ctx context.Context, portal *bridgev2.Portal, records []sidecar.MessageRecord) error { + if portal == nil || ic.UserLogin == nil || ic.UserLogin.Bridge == nil { + return errors.New("history mapping store unavailable") + } + maximum := persistedHistoryPrefix(portal.PortalKey, records, func(mid networkid.MessageID) (*database.Message, error) { + return ic.UserLogin.Bridge.DB.Message.GetFirstPartByID(ctx, portal.Receiver, mid) + }) + if len(records) > 0 && maximum == 0 { + return errors.New("historical delivery awaits persisted Matrix mappings") + } + + return nil +} + +// Only an explicit source-ID catch-up request can advance forward coverage. +// A generic or cached page verifies delivery, not the absence of source gaps. +func (ic *InlineClient) advanceHistoricalCoverage(ctx context.Context, portal *bridgev2.Portal, page historicalPage) error { + if err := ic.persistHistoricalMappings(ctx, portal, page.Records); err != nil { + return err + } + if page.CatchupAfter == nil || len(page.Records) == 0 { + return nil + } + chat, err := chatIDFromPortal(portal) + if err != nil || chat != page.ChatID { + return errors.New("historical catchup belongs to another chat") + } + maximum, err := cachedHistoryPageMax(page.Records, *page.CatchupAfter) + if err != nil { + return err + } + metadata, ok := portal.Metadata.(*PortalMetadata) + if !ok || metadata == nil { + return errors.New("historical coverage metadata unavailable") + } + current := metadata.HistoricalCheckpoint + if current != *page.CatchupAfter { + if maximum <= current { + return nil + } // Already verified coverage; never leap from a stale request. + return errors.New("historical catchup frontier changed; refetch required") + } + if err = ic.persistHistoryCheckpoint(ctx, chat, maximum); err != nil { + return err + } + ic.rememberHistoryDelivered(chat, maximum) + return nil +} + +// Read-state absence is unknown. It never permits implicit history mark-read. +// If authoritative intent exists, journal failure stops delivery before effects. +func (ic *InlineClient) readDesireBeforeDelivery(ctx context.Context, chat int64) error { + state, err := ic.Sidecar.ChatState(ctx, sidecar.ChatStateRequest{ChatID: chat}) + if err != nil { + return nil + } + if state.Dialog != nil { + ic.rememberDialog(*state.Dialog) + } + if state.ReadState != nil { + return ic.storeReadDesire(ctx, state.ReadState) + } + return nil +} + +func (ic *InlineClient) queueHistoricalPage(ctx context.Context, chat int64, records []sidecar.MessageRecord, catchupAfter *int64) error { + if len(records) == 0 { + return nil + } + if err := ic.readDesireBeforeDelivery(ctx, chat); err != nil { + return err + } + result := ic.UserLogin.Bridge.QueueRemoteEvent(ic.UserLogin, &simplevent.ChatResync{ + EventMeta: simplevent.EventMeta{Type: bridgev2.RemoteEventChatResync, PortalKey: ic.portalKeyForChat(chat), CreatePortal: true, PostHandleFunc: func(ctx context.Context, p *bridgev2.Portal) { + _ = ic.fillHistoricalHolesInLane(ctx, p, records) + ic.projectionPostHandler(ctx, p) + }}, + GetChatInfoFunc: ic.migrationGatedInfoProvider(&bridgev2.ChatInfo{}), + BundledBackfillData: historicalPage{ChatID: chat, Records: records}, + CheckNeedsBackfillFunc: func(context.Context, *database.Message) (bool, error) { return true, nil }, + }) + if err := eventHandlingResultError("historical delivery", result); err != nil { + return err + } + portal, err := ic.UserLogin.Bridge.GetExistingPortalByKey(ctx, ic.portalKeyForChat(chat)) + if err != nil { + return err + } + return ic.advanceHistoricalCoverage(ctx, portal, historicalPage{ChatID: chat, Records: records, CatchupAfter: catchupAfter}) +} + +func pendingHistoricalRecords(key networkid.PortalKey, records []sidecar.MessageRecord, lookup func(networkid.MessageID) (*database.Message, error)) ([]sidecar.MessageRecord, error) { + pending := make([]sidecar.MessageRecord, 0, len(records)) + seen := map[networkid.MessageID]bool{} + for _, record := range records { + if record.ChatID <= 0 || record.MessageID <= 0 || makePortalID(strconv.FormatInt(record.ChatID, 10)) != key.ID { + return nil, errors.New("history record belongs to another chat") + } + mid := makeMessageID(record.ChatID, record.MessageID) + if seen[mid] { + continue + } + seen[mid] = true + row, err := lookup(mid) + if err != nil { + return nil, err + } + if row != nil { + if row.Room != key || row.MXID == "" || row.HasFakeMXID() { + return nil, errors.New("history mapping is unverified") + } + continue + } + pending = append(pending, record) + } + return pending, nil +} + +// The pinned high-level forward backfill cuts by display timestamp. Source-ID +// recovery must also deliver older-timestamp holes. Keep the deprecated exported +// seam in this bounded compatibility module, in the existing portal lane. It +// discards its error return, so only subsequent persisted mappings prove success. +func (ic *InlineClient) fillHistoricalHolesInLane(ctx context.Context, portal *bridgev2.Portal, records []sidecar.MessageRecord) error { + if portal.MXID == "" { + return errors.New("historical room not ready") + } + if !portal.Bridge.Config.Backfill.Enabled { + return errors.New("history delivery requires backfill.enabled") + } + pending, err := pendingHistoricalRecords(portal.PortalKey, records, func(mid networkid.MessageID) (*database.Message, error) { + return portal.Bridge.DB.Message.GetFirstPartByID(ctx, portal.Receiver, mid) + }) + if err != nil { + return err + } + if len(pending) > 0 { + messages, _ := ic.backfillMessagesFromHistory(ctx, portal, pending) + if len(messages) > 0 { + portal.Internal().SendBackfill(ctx, ic.UserLogin, messages, true, false, false, nil) + } + } + return ic.persistHistoricalMappings(ctx, portal, records) +} diff --git a/pkg/connector/history_test.go b/pkg/connector/history_test.go new file mode 100644 index 0000000..5e78b91 --- /dev/null +++ b/pkg/connector/history_test.go @@ -0,0 +1,163 @@ +package connector + +import ( + "encoding/json" + "fmt" + "hegel.dev/go/hegel" + "math" + "maunium.net/go/mautrix/id" + "path/filepath" + "testing" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "github.com/rs/zerolog" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" +) + +type frontierMatrix struct{ noPuppetMatrix } + +func (frontierMatrix) ServerName() string { return "test" } +func (frontierMatrix) Init(*bridgev2.Bridge) {} +func (frontierMatrix) BotIntent() bridgev2.MatrixAPI { return nil } + +func fixtureHistoryClient(t *testing.T, path string) *InlineClient { + t.Helper() + store := fixtureJournal(t, path) + br := bridgev2.NewBridge("fixture", store.db, zerolog.Nop(), nil, frontierMatrix{}, &InlineConnector{}, func(*bridgev2.Bridge) bridgev2.CommandProcessor { return nil }) + br.BackgroundCtx = t.Context() + if err := br.DB.Upgrade(t.Context()); err != nil { + t.Fatal(err) + } + return &InlineClient{UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ID: "login"}, Bridge: br}} +} + +func TestHistoricalCheckpointIgnoresInterleavedLiveMaximumAfterRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "history.db") + ic := fixtureHistoryClient(t, path) + var metadata PortalMetadata + if err := json.Unmarshal([]byte(`{"history_checkpoint":21,"historical_checkpoint":10}`), &metadata); err != nil { + t.Fatal(err) + } + portal := &database.Portal{PortalKey: ic.portalKeyForChat(7), Metadata: &metadata} + if err := ic.UserLogin.Bridge.DB.Portal.Insert(t.Context(), portal); err != nil { + t.Fatal(err) + } + restarted := fixtureHistoryClient(t, path) + checkpoint, err := restarted.loadHistoryCheckpoint(t.Context(), 7) + if err != nil || checkpoint != 10 { + t.Fatalf("live maximum suppressed owed historical page: checkpoint%d err%v", checkpoint, err) + } + d := sidecar.DialogRecord{ChatID: 7, LastMessageID: foundationInt64(20)} + if !restarted.needsHistoryDelivery(d) { + t.Fatal("owed history lost after live21/restart") + } + request := historyRequestForDelivery(d, checkpoint, 100) + if request.AfterMessageID == nil || *request.AfterMessageID != 10 { + t.Fatal("history request skipped pending IDs11..20") + } +} + +func TestHistoricalFrontierRequiresEveryMappingDespiteLiveInterleaving(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + start := hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64)) + count := hegel.Draw(ht, hegel.Integers(1, math.MaxInt)) + // Bound the materialized fixture, retaining full scalar ID/size domains. + if count > 12 { + count = 12 + } + if start > math.MaxInt64-int64(count)-1 { + start = math.MaxInt64 - int64(count) - 1 + } + liveFirst := hegel.Draw(ht, hegel.Booleans()) + staleRequest := hegel.Draw(ht, hegel.Booleans()) + missing := hegel.Draw(ht, hegel.Integers(0, count)) // count means complete page + path := filepath.Join(t.TempDir(), fmt.Sprintf("case-%d-%d-%t-%d.db", start, count, liveFirst, missing)) + ic := fixtureHistoryClient(t, path) + portal, err := ic.UserLogin.Bridge.GetPortalByKey(t.Context(), ic.portalKeyForChat(7)) + if err != nil { + ht.Fatal(err) + } + if err = ic.persistHistoryCheckpoint(t.Context(), 7, start); err != nil { + ht.Fatal(err) + } + live := start + int64(count) + 1 + if liveFirst { + if err = ic.persistDeliveryCheckpoint(t.Context(), 7, live, false); err != nil { + ht.Fatal(err) + } + } + records := make([]sidecar.MessageRecord, count) + for i := range count { + n := start + int64(i) + 1 + records[i] = sidecar.MessageRecord{ChatID: 7, MessageID: n, Timestamp: int64(count - i)} + if i == missing { + continue + } + if err = ic.UserLogin.Bridge.DB.Message.Insert(t.Context(), &database.Message{ID: makeMessageID(7, n), MXID: id.EventID(fmt.Sprintf("$%d", n)), Room: portal.PortalKey}); err != nil { + ht.Fatal(err) + } + } + requestAfter := start + if staleRequest { + requestAfter-- + } + err = ic.advanceHistoricalCoverage(t.Context(), portal, historicalPage{ChatID: 7, Records: records, CatchupAfter: &requestAfter}) + want := start + int64(count) + if missing < count || staleRequest { + want = start + if err == nil { + ht.Fatal("incomplete or stale page advanced frontier") + } + } else if err != nil { + ht.Fatal(err) + } + if !liveFirst { + if err = ic.persistDeliveryCheckpoint(t.Context(), 7, live, false); err != nil { + ht.Fatal(err) + } + } + restarted := fixtureHistoryClient(t, path) + got, err := restarted.loadHistoryCheckpoint(t.Context(), 7) + if err != nil || got != want { + ht.Fatalf("historical frontier %d want%d err%v", got, want, err) + } + stored, err := restarted.UserLogin.Bridge.GetPortalByKey(t.Context(), ic.portalKeyForChat(7)) + if err != nil || stored.Metadata.(*PortalMetadata).HistoryCheckpoint != live { + ht.Fatal("legacy live maximum changed", err) + } + }, hegel.WithTestCases(100), hegel.WithDatabase(t.TempDir())) +} + +func TestHistoricalCoverageRejectsStaleRequestFrontierAfterRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "stale.db") + ic := fixtureHistoryClient(t, path) + portal, err := ic.UserLogin.Bridge.GetPortalByKey(t.Context(), ic.portalKeyForChat(7)) + if err != nil { + t.Fatal(err) + } + if err = ic.persistHistoryCheckpoint(t.Context(), 7, 10); err != nil { + t.Fatal(err) + } + // This response was requested after 20, an unrelated maximum. All selected + // records are mapped, but none prove coverage of the owed gap 11..20. + row := &database.Message{ID: "7/21", MXID: "$21", Room: portal.PortalKey} + if err = ic.UserLogin.Bridge.DB.Message.Insert(t.Context(), row); err != nil { + t.Fatal(err) + } + stale := int64(20) + if err = ic.advanceHistoricalCoverage(t.Context(), portal, historicalPage{ChatID: 7, Records: []sidecar.MessageRecord{{ChatID: 7, MessageID: 21}}, CatchupAfter: &stale}); err == nil { + t.Fatal("unrelated forward frontier accepted") + } + restarted := fixtureHistoryClient(t, path) + if got, err := restarted.loadHistoryCheckpoint(t.Context(), 7); err != nil || got != 10 { + t.Fatalf("stale proof persisted %d %v", got, err) + } +} + +func TestHistoryRequestForDeliveryStartsAtExplicitZero(t *testing.T) { + req := historyRequestForDelivery(sidecar.DialogRecord{ChatID: 7}, 0, 100) + if req.AfterMessageID == nil || *req.AfterMessageID != 0 || req.BeforeMessageID != nil { + t.Fatalf("empty frontier selected recent tail instead of oldest source IDs: %+v", req) + } +} diff --git a/pkg/connector/login_test.go b/pkg/connector/login_test.go index 2a775d8..ec9d183 100644 --- a/pkg/connector/login_test.go +++ b/pkg/connector/login_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "net/http" - "net/http/httptest" "strings" "testing" @@ -30,7 +29,7 @@ func TestNewLoginStoreNamespaceIsSafeAndUnique(t *testing.T) { } func TestInlineCodeLoginStartsEmailCodeFlow(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/rpc/auth/start" { t.Fatalf("path = %s, want /rpc/auth/start", r.URL.Path) } @@ -82,7 +81,7 @@ func TestInlineCodeLoginStartsEmailCodeFlow(t *testing.T) { } func TestInlineCodeLoginDefersInviteSignup(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { writeLoginRPCResult(t, w, "auth_start", sidecar.AuthStartResult{ NeedsInviteCode: true, }) diff --git a/pkg/connector/migration.go b/pkg/connector/migration.go new file mode 100644 index 0000000..6e18064 --- /dev/null +++ b/pkg/connector/migration.go @@ -0,0 +1,363 @@ +package connector + +import ( + "context" + "errors" + "fmt" + "maps" + "maunium.net/go/mautrix/bridgev2/simplevent" + "slices" + "strconv" + "strings" + + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/matrix" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/id" +) + +type presentationValue struct { + RoomID id.RoomID `json:"room_id"` + Kind database.RoomType `json:"kind"` + Parent string `json:"parent"` +} +type presentationMigration struct { + Before presentationValue `json:"before"` + Applied presentationValue `json:"applied"` + Phase string `json:"phase"` + Parents []id.RoomID `json:"parents"` + BeforeWire map[string]string `json:"before_wire"` + AppliedWire map[string]string `json:"applied_wire"` + RollbackWire map[string]string `json:"rollback_wire"` +} + +func presentationOf(p *bridgev2.Portal) presentationValue { + return presentationValue{RoomID: p.MXID, Kind: p.RoomType, Parent: string(p.ParentKey.ID)} +} +func migrationRollbackAllowed(current presentationValue, m *presentationMigration) bool { + return current == m.Applied +} +func (ic *InlineClient) runMigrationInLane(ctx context.Context, portal *bridgev2.Portal, s *projectionStore, k projectionKey, p *projectionState) { + m := p.Migration + if m == nil || portal.MXID == "" || m.Phase == "applied" || m.Phase == "rolled_back" || m.Phase == "conflict" { + return + } + raw, ok := portal.Bridge.Bot.(*matrix.ASIntent) + if !ok || raw.Matrix == nil { + p.ParentError = "migration_readback_unsupported" + _ = s.save(ctx, k, p) + return + } + rollback := m.Phase == "rollback_pending" || m.Phase == "rolling_back" + current, err := migrationWireSnapshot(ctx, raw.Matrix.Client, portal, m.Parents) + if err != nil { + p.ParentError = "migration_readback_failed" + _ = s.save(ctx, k, p) + return + } + conflict := !migrationValuesCompatible(presentationOf(portal), m.Before, m.Applied) + if m.Phase == "rollback_pending" { + conflict = conflict || !migrationRollbackAllowed(presentationOf(portal), m) || !maps.Equal(current, m.AppliedWire) + } else if rollback { + conflict = conflict || !migrationWireCompatible(current, m.RollbackWire, m.AppliedWire) + } else { + conflict = conflict || !migrationWireCompatible(current, m.BeforeWire, m.AppliedWire) + } + if conflict { + m.Phase = "conflict" + p.ParentError = "migration_user_change_conflict" + _ = s.save(ctx, k, p) + return + } + desired := m.Applied + m.Phase = "applying" + if rollback { + desired = m.Before + m.Phase = "rolling_back" + } + if desired.RoomID != portal.MXID || portal.RoomType == database.RoomTypeSpace || desired.Kind == database.RoomTypeSpace { + m.Phase = "conflict" + _ = s.save(ctx, k, p) + return + } + p.DesiredParent = desired.Parent + p.ParentKnown = true + p.ParentPending = true + // Keep all affected links durable through every partial apply/rollback. + p.OwnedParents = slices.Clone(m.Parents) + if err = s.save(ctx, k, p); err != nil { + return + } + expected := m.AppliedWire + if rollback { + expected = m.RollbackWire + } + if err = restoreMigrationLinks(ctx, matrixHierarchyWire{raw}, portal.MXID, m.Parents, expected, m.BeforeWire, m.AppliedWire); err != nil { + p.ParentError = "migration_links_pending" + if errors.Is(err, errProjectionConflict) { + m.Phase = "conflict" + p.ParentError = "migration_user_change_conflict" + } + _ = s.save(ctx, k, p) + return + } + // Recheck immediately before metadata writes. Matrix offers no conditional + // state transaction; operators must keep this migration window controlled. + current, err = migrationWireSnapshot(ctx, raw.Matrix.Client, portal, m.Parents) + if err != nil || !migrationWireCompatible(current, m.BeforeWire, m.AppliedWire) { + p.ParentError = "migration_readback_failed" + if err == nil { + m.Phase = "conflict" + p.ParentError = "migration_user_change_conflict" + } + _ = s.save(ctx, k, p) + return + } + portal.RoomType = desired.Kind + // Exact journaled link writes above are the only link effects in migration. + if err = ic.repairHierarchyState(ctx, portal, s, k, p, false); err != nil { + p.ParentError = "migration_pending" + _ = s.save(ctx, k, p) + return + } + current, err = migrationWireSnapshot(ctx, raw.Matrix.Client, portal, m.Parents) + if err != nil || !maps.Equal(current, expected) { + p.ParentPending = true + p.ParentError = "migration_readback_failed" + _ = s.save(ctx, k, p) + return + } + if rollback { + m.Phase = "rolled_back" + } else { + m.Phase = "applied" + } + p.ParentError = "" + _ = s.save(ctx, k, p) +} + +func (ic *InlineClient) prepareMigration(ctx context.Context, portal *bridgev2.Portal, mode string) error { + s, k, err := ic.projectionJournal(ctx, string(portal.ID)) + if err != nil { + return err + } + p, err := s.load(ctx, k) + if err != nil { + return err + } + if mode == "rollback" { + if p.Migration == nil { + return errors.New("no applied migration for selected room") + } + switch p.Migration.Phase { + case "rollback_pending", "rolling_back", "rolled_back": + return nil + case "applied": + p.Migration.Phase = "rollback_pending" + default: + return errors.New("no applied migration for selected room") + } + } else { + chat, err := chatIDFromPortal(portal) + if err != nil { + return err + } + d, ok := ic.cachedDialog(chat) + if !ok || !ic.shouldProjectDialog(d) || classifyDialog(d) == roomUnknown { + return errors.New("selected room has unknown or excluded source structure") + } + parent, known := ic.desiredWorkspace(d) + if !known { + return errors.New("selected room parent is unknown") + } + kind := database.RoomTypeDefault + if isDMDialog(d) { + kind = database.RoomTypeDM + } + if portal.RoomType == database.RoomTypeSpace { + return errors.New("cannot change Matrix space type") + } + if p.Migration != nil && p.Migration.Phase != "rolled_back" && p.Migration.Phase != "conflict" { + return nil // Repeated selection resumes the same durable operation. + } + m := &presentationMigration{Before: presentationOf(portal), Applied: presentationValue{RoomID: portal.MXID, Kind: kind, Parent: parent}, Phase: "pending"} + m.Parents = slices.Clone(p.OwnedParents) + if portal.Parent != nil && portal.Parent.MXID != "" && !slices.Contains(m.Parents, portal.Parent.MXID) { + m.Parents = append(m.Parents, portal.Parent.MXID) + } + var target id.RoomID + if parent != "" { + resolved, err := portal.Bridge.GetExistingPortalByKey(ctx, networkid.PortalKey{ID: networkid.PortalID(parent), Receiver: portal.Receiver}) + if err != nil { + return err + } + if resolved == nil || resolved.MXID == "" { + return errors.New("selected workspace not ready") + } + target = resolved.MXID + if !slices.Contains(m.Parents, target) { + m.Parents = append(m.Parents, target) + } + } + raw, ok := portal.Bridge.Bot.(*matrix.ASIntent) + if !ok || raw.Matrix == nil { + return errors.New("migration readback unsupported") + } + m.BeforeWire, err = migrationWireSnapshot(ctx, raw.Matrix.Client, portal, m.Parents) + if err != nil { + return err + } + m.AppliedWire = expectedMigrationWire(m.BeforeWire, m.Applied, m.Parents, target, portal.Bridge.Matrix.ServerName()) + m.RollbackWire = migrationBridgeValues(m.BeforeWire, m.Before) + p.Migration = m + } + // Selection is durable before the event lane performs external writes. + return s.save(ctx, k, &p) +} + +func parseMigrationCommand(args []string) (string, []int64, error) { + mode := "preview" + if len(args) > 0 { + switch args[0] { + case "preview", "apply", "rollback": + mode = args[0] + args = args[1:] + } + } + if len(args) == 0 || len(args) > 50 { + return "", nil, errors.New("select 1–50 numeric chat IDs explicitly") + } + ids := make([]int64, 0, len(args)) + seen := map[int64]bool{} + for _, arg := range args { + n, err := strconv.ParseInt(arg, 10, 64) + if err != nil || n <= 0 || strconv.FormatInt(n, 10) != arg || seen[n] { + return "", nil, errors.New("select unique positive numeric chat IDs") + } + seen[n] = true + ids = append(ids, n) + } + return mode, ids, nil +} +func (ic *InlineClient) migrationPostHandler(ctx context.Context, p *bridgev2.Portal) { + s, k, err := ic.projectionJournal(ctx, string(p.ID)) + if err != nil { + return + } + state, err := s.load(ctx, k) + if err == nil { + ic.runMigrationInLane(ctx, p, s, k, &state) + } +} +func (ic *InlineClient) migrateSelected(ctx context.Context, mode string, ids []int64) string { + var out strings.Builder + fmt.Fprintf(&out, "Presentation migration: %s (version %d)\n", mode, projectionVersion) + for _, chat := range ids { + portal, err := ic.UserLogin.Bridge.GetExistingPortalByKey(ctx, ic.portalKeyForChat(chat)) + if err != nil || portal == nil || portal.MXID == "" { + fmt.Fprintf(&out, "- %d: no existing room for this login\n", chat) + continue + } + if mode == "preview" { + d, ok := ic.cachedDialog(chat) + parent, known := ic.desiredWorkspace(d) + eligibility := "eligible" + if !ok || classifyDialog(d) == roomUnknown || !known { + eligibility = "ambiguous; refresh required" + } else if !ic.shouldProjectDialog(d) { + eligibility = "hidden; excluded" + } + fmt.Fprintf(&out, "- %d: room `%s`, kind `%s` → `%s`, parent `%s` → `%s`; %s. Notification/tag and delivery state preserved.\n", chat, portal.MXID, portal.RoomType, classifyDialog(d), portal.ParentKey.ID, parent, eligibility) + continue + } + if mode == "apply" { + if d, ok := ic.cachedDialog(chat); ok { + if err := ic.ensureWorkspace(ctx, d); err != nil { + fmt.Fprintf(&out, "- %d: pending workspace creation\n", chat) + continue + } + } + } + var prepareErr error + result := ic.UserLogin.Bridge.QueueRemoteEvent(ic.UserLogin, &simplevent.ChatResync{ + EventMeta: simplevent.EventMeta{Type: bridgev2.RemoteEventChatResync, PortalKey: portal.PortalKey, PostHandleFunc: func(ctx context.Context, p *bridgev2.Portal) { + if prepareErr == nil { + ic.migrationPostHandler(ctx, p) + } + }}, + GetChatInfoFunc: func(ctx context.Context, p *bridgev2.Portal) (*bridgev2.ChatInfo, error) { + prepareErr = ic.prepareMigration(ctx, p, mode) + if prepareErr != nil { + return nil, prepareErr + } + return ic.migrationGatedChatInfo(ctx, p, &bridgev2.ChatInfo{}) + }, + }) + if prepareErr != nil { + fmt.Fprintf(&out, "- %d: %s\n", chat, prepareErr) + continue + } + if err := eventHandlingResultError("selected migration", result); err != nil { + fmt.Fprintf(&out, "- %d: pending portal handling\n", chat) + continue + } + s, k, err := ic.projectionJournal(ctx, string(portal.ID)) + if err != nil { + fmt.Fprintf(&out, "- %d: journal unavailable\n", chat) + continue + } + p, err := s.load(ctx, k) + if err != nil || p.Migration == nil { + fmt.Fprintf(&out, "- %d: journal unavailable\n", chat) + continue + } + fmt.Fprintf(&out, "- %d: %s\n", chat, p.Migration.Phase) + } + return out.String() +} + +var errMigrationPresentationHeld = errors.New("presentation updates held by migration journal") + +// This gate must run in the portal lane before the framework sees any ChatInfo. +// ExtraUpdates is too late: UpdateInfo may already have changed type/name and +// will write bridge metadata after that hook even when the hook reports conflict. +func (ic *InlineClient) guardMigrationPresentation(ctx context.Context, portal *bridgev2.Portal) error { + if portal == nil || portal.MXID == "" || strings.HasPrefix(string(portal.ID), "space:") { + return nil + } + store, key, err := ic.projectionJournal(ctx, string(portal.ID)) + if err != nil { + return err + } + state, err := store.load(ctx, key) + if err != nil { + return err + } + if state.Migration == nil || state.Migration.Phase == "applied" { + return nil + } + if state.Migration.Phase != "conflict" && state.Migration.Phase != "rolled_back" { + ic.runMigrationInLane(ctx, portal, store, key, &state) + state, err = store.load(ctx, key) + if err != nil { + return err + } + if state.Migration != nil && state.Migration.Phase == "applied" { + return nil + } + } + return errMigrationPresentationHeld +} + +func (ic *InlineClient) migrationGatedChatInfo(ctx context.Context, portal *bridgev2.Portal, info *bridgev2.ChatInfo) (*bridgev2.ChatInfo, error) { + if err := ic.guardMigrationPresentation(ctx, portal); err != nil { + return nil, err + } + return info, nil +} +func (ic *InlineClient) migrationGatedInfoProvider(info *bridgev2.ChatInfo) func(context.Context, *bridgev2.Portal) (*bridgev2.ChatInfo, error) { + return func(ctx context.Context, portal *bridgev2.Portal) (*bridgev2.ChatInfo, error) { + return ic.migrationGatedChatInfo(ctx, portal, info) + } +} diff --git a/pkg/connector/migration_wire.go b/pkg/connector/migration_wire.go new file mode 100644 index 0000000..c79efbe --- /dev/null +++ b/pkg/connector/migration_wire.go @@ -0,0 +1,199 @@ +package connector + +import ( + "context" + "encoding/json" + "errors" + "maps" + "strings" + + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +// Matrix state has no multi-event transaction. A persisted per-field guard +// allows our own partial effects after a crash, while refusing a third value. +func migrationValuesCompatible(current, before, after presentationValue) bool { + return current.RoomID == before.RoomID && current.RoomID == after.RoomID && + (current.Kind == before.Kind || current.Kind == after.Kind) && + (current.Parent == before.Parent || current.Parent == after.Parent) +} +func migrationWireCompatible(current, before, after map[string]string) bool { + if len(before) == 0 || len(current) != len(before) || len(after) != len(before) { + return false + } + for k, v := range current { + b, bok := before[k] + a, aok := after[k] + if !bok || !aok || (v != b && v != a) { + return false + } + } + return true +} +func migrationJSON(value any) string { + raw, _ := json.Marshal(value) + var canonical any + _ = json.Unmarshal(raw, &canonical) + raw, _ = json.Marshal(canonical) + return string(raw) +} +func migrationLink(content *event.Content) (string, error) { + raw, err := content.MarshalJSON() + if err != nil { + return "", err + } + var fields map[string]any + if err = json.Unmarshal(raw, &fields); err != nil { + return "", err + } + // Missing links and explicit empty tombstones have the same meaning. + if len(fields) == 0 { + return "{}", nil + } + return migrationJSON(fields), nil +} +func migrationWireSnapshot(ctx context.Context, cli *mautrix.Client, portal *bridgev2.Portal, parents []id.RoomID) (map[string]string, error) { + state, err := cli.State(ctx, portal.MXID) + if err != nil { + return nil, err + } + values := make(map[string]string) + for _, typ := range []event.Type{event.StateBridge, event.StateHalfShotBridge} { + found := false + for _, evt := range state[typ] { + var info event.BridgeEventContent + raw, err := evt.Content.MarshalJSON() + if err != nil { + return nil, err + } + if err = json.Unmarshal(raw, &info); err != nil { + return nil, err + } + if info.Channel.ID != string(portal.ID) || info.Channel.Receiver != string(portal.Receiver) { + continue + } + if found { + return nil, errors.New("ambiguous bridge info") + } + found = true + prefix := "bridge:" + typ.Type + ":" + values[prefix+"kind"] = info.BeeperRoomTypeV2 + values[prefix+"legacy"] = info.BeeperRoomType + values[prefix+"parent"] = "" + if info.Network != nil { + values[prefix+"parent"] = info.Network.ID + } + } + if !found { + return nil, errors.New("bridge info unavailable") + } + } + for _, parent := range parents { + values["parent:"+string(parent)] = "{}" + } + for key, evt := range state[event.StateSpaceParent] { + value, err := migrationLink(&evt.Content) + if err != nil { + return nil, err + } + if value != "{}" { + values["parent:"+key] = value + } + } + for _, parent := range parents { + var content map[string]any + if err = cli.StateEvent(ctx, parent, event.StateSpaceChild, string(portal.MXID), &content); err != nil && !errors.Is(err, mautrix.MNotFound) { + return nil, err + } + if content == nil { + content = map[string]any{} + } + values["child:"+string(parent)] = migrationJSON(content) + } + return values, nil +} +func migrationBridgeValues(before map[string]string, desired presentationValue) map[string]string { + values := maps.Clone(before) + for key := range values { + if !strings.HasPrefix(key, "bridge:") { + continue + } + switch { + case strings.HasSuffix(key, ":kind"): + values[key] = string(desired.Kind) + case strings.HasSuffix(key, ":legacy"): + values[key] = "" + if desired.Kind == "dm" || desired.Kind == "group_dm" { + values[key] = "dm" + } + case strings.HasSuffix(key, ":parent"): + values[key] = desired.Parent + } + } + return values +} +func expectedMigrationWire(before map[string]string, desired presentationValue, parents []id.RoomID, target id.RoomID, via string) map[string]string { + values := migrationBridgeValues(before, desired) + for _, parent := range parents { + values["parent:"+string(parent)] = "{}" + values["child:"+string(parent)] = "{}" + if parent == target { + values["parent:"+string(parent)] = migrationJSON(&event.SpaceParentEventContent{Via: []string{via}, Canonical: true}) + values["child:"+string(parent)] = migrationJSON(&event.SpaceChildEventContent{Via: []string{via}}) + } + } + return values +} +func restoreMigrationLinks(ctx context.Context, w hierarchyWire, child id.RoomID, parents []id.RoomID, before map[string]string, allowed ...map[string]string) error { + for _, parent := range parents { + for _, link := range []struct { + room id.RoomID + typ event.Type + key, value string + }{ + {child, event.StateSpaceParent, string(parent), before["parent:"+string(parent)]}, + {parent, event.StateSpaceChild, string(child), before["child:"+string(parent)]}, + } { + var content map[string]any + if err := json.Unmarshal([]byte(link.value), &content); err != nil { + return err + } + var existing map[string]any + if err := w.get(ctx, link.room, link.typ, link.key, &existing); err != nil && !errors.Is(err, mautrix.MNotFound) { + return err + } + current := "{}" + if existing != nil { + current = migrationJSON(existing) + } + field := "parent:" + string(parent) + if link.typ == event.StateSpaceChild { + field = "child:" + string(parent) + } + permitted := current == link.value + for _, values := range allowed { + permitted = permitted || current == values[field] + } + if !permitted { + return errProjectionConflict + } + if current == link.value { + continue + } + if err := w.put(ctx, link.room, link.typ, link.key, content); err != nil { + return err + } + var got map[string]any + if err := w.get(ctx, link.room, link.typ, link.key, &got); err != nil { + return err + } + if migrationJSON(got) != link.value { + return errors.New("rollback hierarchy readback differs") + } + } + } + return nil +} diff --git a/pkg/connector/migration_wire_test.go b/pkg/connector/migration_wire_test.go new file mode 100644 index 0000000..b25411b --- /dev/null +++ b/pkg/connector/migration_wire_test.go @@ -0,0 +1,193 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "maps" + "maunium.net/go/mautrix/appservice" + "maunium.net/go/mautrix/bridgev2/matrix" + "maunium.net/go/mautrix/bridgev2/simplevent" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +func TestMigrationRemoteSnapshotDetectsLaterParentChoice(t *testing.T) { + parent := "!applied:test" + kind := "" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/!child:test/state") { + events := []map[string]any{} + for _, typ := range []string{"m.bridge", "uk.half-shot.bridge"} { + events = append(events, map[string]any{"type": typ, "state_key": "fixture", "content": map[string]any{"channel": map[string]any{"id": "7", "fi.mau.receiver": "1"}, "com.beeper.room_type.v2": kind, "network": map[string]any{"id": "space:2"}}}) + } + events = append(events, map[string]any{"type": "m.space.parent", "state_key": parent, "content": map[string]any{"via": []string{"test"}, "canonical": true}}) + json.NewEncoder(w).Encode(events) + } else if strings.Contains(r.URL.Path, "/state/m.space.child/") { + json.NewEncoder(w).Encode(map[string]any{"via": []string{"test"}}) + } else { + t.Errorf("unexpected read %s", r.URL.Path) + http.Error(w, "unexpected", 500) + } + })) + defer server.Close() + cli, err := mautrix.NewClient(server.URL, "@bot:test", "fixture") + if err != nil { + t.Fatal(err) + } + portal := &bridgev2.Portal{Portal: &database.Portal{PortalKey: networkid.PortalKey{ID: "7", Receiver: "1"}, MXID: "!child:test", RoomType: database.RoomTypeDefault}} + applied, err := migrationWireSnapshot(t.Context(), cli, portal, []id.RoomID{"!applied:test"}) + if err != nil { + t.Fatal(err) + } + if applied["bridge:m.bridge:parent"] != "space:2" || applied["parent:!applied:test"] != `{"canonical":true,"via":["test"]}` { + t.Fatalf("wire snapshot dropped presentation: %+v", applied) + } + parent = "!user-choice:test" + changed, err := migrationWireSnapshot(t.Context(), cli, portal, []id.RoomID{"!applied:test"}) + if err != nil { + t.Fatal(err) + } + if maps.Equal(applied, changed) || migrationWireCompatible(changed, applied, applied) { + t.Fatal("user remote parent change hidden by unchanged portal cache") + } +} + +func TestMigrationLinkRestoreRefusesUnjournaledUserChoice(t *testing.T) { + f := &fixtureHierarchy{state: map[string][]byte{}} + child, parent := id.RoomID("!child:test"), id.RoomID("!parent:test") + before := map[string]string{"parent:" + string(parent): `{"canonical":true,"via":["before","extra"]}`, "child:" + string(parent): `{"suggested":true,"via":["before","extra"]}`} + if err := f.put(t.Context(), child, event.StateSpaceParent, string(parent), map[string]any{"via": []string{"user"}, "canonical": true}); err != nil { + t.Fatal(err) + } + writes := f.writes + if err := restoreMigrationLinks(t.Context(), f, child, []id.RoomID{parent}, before); err == nil { + t.Fatal("overwrote unjournaled user parent") + } + if f.writes != writes { + t.Fatal("conflicting link was overwritten") + } +} + +func TestMigrationGateRunsBeforeFrameworkInfoEffects(t *testing.T) { + for _, third := range []bool{true, false} { + t.Run(fmt.Sprint(third), func(t *testing.T) { + ic := fixtureHistoryClient(t, filepath.Join(t.TempDir(), "guard.db")) + br := ic.UserLogin.Bridge + ic.UserLogin.User = &bridgev2.User{User: &database.User{MXID: "@user:test"}, Bridge: br} + portal, err := br.GetPortalByKey(t.Context(), ic.portalKeyForChat(7)) + if err != nil { + t.Fatal(err) + } + portal.MXID = "!child:test" + portal.RoomType = database.RoomTypeGroupDM + if err = portal.Save(t.Context()); err != nil { + t.Fatal(err) + } + wire := map[string]map[string]any{} + for _, typ := range []string{"m.bridge", "uk.half-shot.bridge"} { + wire[typ] = map[string]any{"channel": map[string]any{"id": "7", "fi.mau.receiver": "login"}, "com.beeper.room_type.v2": "group_dm", "com.beeper.room_type": "dm"} + } + writes := 0 + hs := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/state") { + events := []map[string]any{} + for typ, content := range wire { + events = append(events, map[string]any{"type": typ, "state_key": "fixture", "content": content}) + } + json.NewEncoder(w).Encode(events) + return + } + if r.Method == http.MethodPut && strings.Contains(r.URL.Path, "/state/") { + writes++ + for _, typ := range []string{"m.bridge", "uk.half-shot.bridge"} { + if strings.Contains(r.URL.Path, "/state/"+typ+"/") { + var content map[string]any + json.NewDecoder(r.Body).Decode(&content) + wire[typ] = content + } + } + w.Write([]byte(`{"event_id":"$state"}`)) + return + } + t.Errorf("unexpected Matrix route %s %s", r.Method, r.URL.Path) + w.WriteHeader(400) + })) + defer hs.Close() + as, err := appservice.CreateFull(appservice.CreateOpts{HomeserverDomain: "test", HomeserverURL: hs.URL, Registration: &appservice.Registration{SenderLocalpart: "bot", AppToken: "synthetic"}}) + if err != nil { + t.Fatal(err) + } + raw := &matrix.ASIntent{Matrix: as.BotIntent()} + br.Bot = raw + if err = as.StateStore.SetMembership(t.Context(), portal.MXID, raw.GetMXID(), event.MembershipJoin); err != nil { + t.Fatal(err) + } + beforeWire, err := migrationWireSnapshot(t.Context(), raw.Matrix.Client, portal, nil) + if err != nil { + t.Fatal(err) + } + journal, key, err := ic.projectionJournal(t.Context(), "7") + if err != nil { + t.Fatal(err) + } + state, _ := journal.load(t.Context(), key) + state.Migration = &presentationMigration{Before: presentationOf(portal), Applied: presentationValue{RoomID: portal.MXID, Kind: database.RoomTypeDefault}, Phase: "pending", BeforeWire: beforeWire} + state.Migration.AppliedWire = migrationBridgeValues(beforeWire, state.Migration.Applied) + if err = journal.save(t.Context(), key, &state); err != nil { + t.Fatal(err) + } + if third { + for _, content := range wire { + content["com.beeper.room_type.v2"] = "external" + } + } + source := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeConnectorSidecarResult(t, w, "chat_participants", sidecar.ChatParticipantsPage{Participants: []sidecar.ChatParticipantRecord{{UserID: 1}}}) + })) + defer source.Close() + ic.Sidecar = sidecar.NewClient(source.URL) + ic.rememberDialog(sidecar.DialogRecord{ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "channel"}}) + postCalled, backfillChecked := false, false + br.Config.Backfill.Enabled = true + result := portal.Internal().HandleRemoteEvent(t.Context(), nil, bridgev2.RemoteEventChatResync, &simplevent.ChatResync{ + EventMeta: simplevent.EventMeta{PostHandleFunc: func(context.Context, *bridgev2.Portal) { postCalled = true }}, + GetChatInfoFunc: ic.GetChatInfo, + CheckNeedsBackfillFunc: func(context.Context, *database.Message) (bool, error) { backfillChecked = true; return false, nil }, + }) + if !postCalled || !backfillChecked { + t.Fatal("presentation hold blocked history check or post handler") + } + if !result.Success { + t.Fatal(result.Error) + } + state, err = journal.load(t.Context(), key) + if err != nil { + t.Fatal(err) + } + if third { + if writes != 0 || state.Migration.Phase != "conflict" || portal.RoomType != database.RoomTypeGroupDM { + t.Fatalf("conflicting metadata escaped pre-UpdateInfo gate: writes%d phase%s kind%s", writes, state.Migration.Phase, portal.RoomType) + } + for _, content := range wire { + if content["com.beeper.room_type.v2"] != "external" { + t.Fatal("framework overwrote third-party bridge metadata") + } + } + } else if state.Migration.Phase != "applied" || portal.RoomType != database.RoomTypeDefault { + t.Fatalf("pending migration did not resume before UpdateInfo: %+v kind%s", state.Migration, portal.RoomType) + } + }) + } +} diff --git a/pkg/connector/preferences.go b/pkg/connector/preferences.go new file mode 100644 index 0000000..7aebc01 --- /dev/null +++ b/pkg/connector/preferences.go @@ -0,0 +1,101 @@ +package connector + +import ( + "context" + "errors" + "fmt" + + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/commands" +) + +// This is desired policy only. No automatic local preference write is enabled +// until the target can prove bridge/user origin and preserve later choices. +func channelMuteDefaultForLogin(login *bridgev2.UserLogin) string { + if login != nil && login.Bridge != nil { + if connector, ok := login.Bridge.Network.(*InlineConnector); ok && connector.Config.MuteChannelsByDefault != nil && !*connector.Config.MuteChannelsByDefault { + return "normal" + } + } + return "muted" +} +func (ic *InlineClient) channelMuteSettings() (defaults, override, effective string) { + ic.mu.RLock() + defer ic.mu.RUnlock() + defaults = ic.channelMuteDefault + if defaults == "" { + defaults = "muted" + } + override = ic.channelMuteOverride + effective = override + if override == "" { + override = "default" + effective = defaults + } + return +} +func parseChannelMuteCommand(args []string) (string, error) { + if len(args) != 1 { + return "", errors.New("choose muted, normal or default") + } + switch args[0] { + case "default": + return "", nil + case "muted", "normal": + return args[0], nil + default: + return "", errors.New("choose muted, normal or default") + } +} +func (ic *InlineClient) persistChannelMuteOverride(ctx context.Context, policy string) error { + if policy != "" && policy != "muted" && policy != "normal" { + return errors.New("invalid channel default") + } + if ic.UserLogin == nil { + return errors.New("Inline login unavailable") + } + ic.metadataSaveMu.Lock() + defer ic.metadataSaveMu.Unlock() + meta, ok := ic.UserLogin.Metadata.(*UserLoginMetadata) + if !ok || meta == nil { + return errors.New("Inline login metadata unavailable") + } + before := meta.ChannelMuteDefault + meta.ChannelMuteDefault = policy + if err := ic.UserLogin.Save(ctx); err != nil { + meta.ChannelMuteDefault = before + return fmt.Errorf("save desired channel default: %w", err) + } + ic.mu.Lock() + ic.channelMuteOverride = policy + ic.mu.Unlock() + return nil +} + +var commandInlineChannelDefault = &commands.FullHandler{ + Name: "inline-channel-default", RequiresLogin: true, + Help: commands.HelpMeta{Section: commands.HelpSectionMisc, Description: "Set desired channel mute policy (automatic writes disabled)", Args: ""}, + Func: func(ce *commands.Event) { + _, client := inlineCommandLogin(ce) + if client == nil { + ce.Reply("Your default login is not an Inline login.") + return + } + policy, err := parseChannelMuteCommand(ce.Args) + if err == nil { + err = client.persistChannelMuteOverride(ce.Ctx, policy) + } + if err != nil { + ce.Reply("Channel default: %s", err) + return + } + ce.Reply("%s", inlineSettingsSummary(client)) + }, +} + +func channelMutePolicyFromMetadata(value string) string { + if value == "muted" || value == "normal" { + return value + } + return "" +} diff --git a/pkg/connector/projection.go b/pkg/connector/projection.go new file mode 100644 index 0000000..000e79f --- /dev/null +++ b/pkg/connector/projection.go @@ -0,0 +1,572 @@ +package connector + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "slices" + "time" + + "github.com/rs/zerolog" + "go.mau.fi/util/dbutil" + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/matrix" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" +) + +const projectionVersion = 1 + +var errProjectionConflict = errors.New("presentation state changed while reconciling") + +type projectionKey struct{ Bridge, User, Login, Receiver, Chat string } +type projectionState struct { + Revision int64 `json:"revision"` + Version int `json:"version"` + ReadMaxID int64 `json:"read_max_id,omitempty"` + ManualUnread bool `json:"manual_unread"` + ReadPending bool `json:"read_pending"` + ReadApplied int64 `json:"read_applied,omitempty"` + ReceiptObservation *receiptObservation `json:"receipt_observation,omitempty"` + ReceiptSyncToken string `json:"receipt_sync_token,omitempty"` + ReceiptTarget id.EventID `json:"receipt_target,omitempty"` + ReadError string `json:"read_error,omitempty"` + UnreadPending bool `json:"unread_pending"` + DesiredParent string `json:"desired_parent,omitempty"` + ParentKnown bool `json:"parent_known"` + OwnedParents []id.RoomID `json:"owned_parents,omitempty"` + ParentPending bool `json:"parent_pending"` + AppliedParent string `json:"applied_parent,omitempty"` + ParentError string `json:"parent_error,omitempty"` + NotificationPolicy string `json:"notification_policy,omitempty"` + Migration *presentationMigration `json:"migration,omitempty"` +} + +// Receipt evidence belongs to one desired operation, never merely a source ID. +type receiptOperation struct { + Key projectionKey `json:"key"` + Room id.RoomID `json:"room"` + User id.UserID `json:"user"` + Target id.EventID `json:"target"` + ReadMaxID int64 `json:"read_max_id"` + ManualUnread bool `json:"manual_unread"` +} +type receiptObservation struct { + Operation receiptOperation `json:"operation"` + SyncToken string `json:"sync_token"` +} + +type projectionStore struct{ db *dbutil.Database } + +func (s *projectionStore) init(ctx context.Context) error { + _, err := s.db.Exec(ctx, `CREATE TABLE IF NOT EXISTS inline_projection ( + bridge TEXT NOT NULL, matrix_user TEXT NOT NULL, login TEXT NOT NULL, + receiver TEXT NOT NULL, chat TEXT NOT NULL, revision BIGINT NOT NULL, state TEXT NOT NULL, + PRIMARY KEY (bridge,matrix_user,login,receiver,chat))`) + return err +} +func (s *projectionStore) load(ctx context.Context, k projectionKey) (projectionState, error) { + var raw string + err := s.db.QueryRow(ctx, `SELECT state FROM inline_projection WHERE bridge=$1 AND matrix_user=$2 AND login=$3 AND receiver=$4 AND chat=$5`, k.Bridge, k.User, k.Login, k.Receiver, k.Chat).Scan(&raw) + if errors.Is(err, sql.ErrNoRows) { + return projectionState{Version: projectionVersion}, nil + } + if err != nil { + return projectionState{}, err + } + var p projectionState + err = json.Unmarshal([]byte(raw), &p) + if err == nil && p.Version != projectionVersion { + err = errors.New("unsupported presentation journal version") + } + return p, err +} +func (s *projectionStore) save(ctx context.Context, k projectionKey, p *projectionState) error { + next := *p + next.Revision++ + raw, err := json.Marshal(next) + if err != nil { + return err + } + args := []any{k.Bridge, k.User, k.Login, k.Receiver, k.Chat, next.Revision, string(raw)} + var result sql.Result + if p.Revision == 0 { + result, err = s.db.Exec(ctx, `INSERT INTO inline_projection (bridge,matrix_user,login,receiver,chat,revision,state) VALUES ($1,$2,$3,$4,$5,$6,$7) ON CONFLICT DO NOTHING`, args...) + } else { + args = append(args, p.Revision) + result, err = s.db.Exec(ctx, `UPDATE inline_projection SET revision=$6,state=$7 WHERE bridge=$1 AND matrix_user=$2 AND login=$3 AND receiver=$4 AND chat=$5 AND revision=$8`, args...) + } + if err != nil { + return err + } + n, err := result.RowsAffected() + if err != nil { + return err + } + if n != 1 { + return errProjectionConflict + } + *p = next + return nil +} +func (ic *InlineClient) projectionJournal(ctx context.Context, chat string) (*projectionStore, projectionKey, error) { + if ic.UserLogin == nil || ic.UserLogin.Bridge == nil || ic.UserLogin.Bridge.DB == nil || ic.UserLogin.User == nil { + return nil, projectionKey{}, errors.New("presentation journal unavailable") + } + login := ic.UserLogin + key := projectionKey{Bridge: string(login.Bridge.DB.BridgeID), User: string(login.User.MXID), Login: string(login.ID), Receiver: string(ic.portalKeyForChat(1).Receiver), Chat: chat} + s := &projectionStore{db: login.Bridge.DB.Database} + return s, key, s.init(ctx) +} + +// Persist source intent before attempting any Matrix effects. A separate revision +// guards completion against a source update arriving during an HTTP request. +func (ic *InlineClient) storeReadDesire(ctx context.Context, read *sidecar.ReadStateRecord) error { + if read.ChatID <= 0 { + return errors.New("invalid read-state chat") + } + s, k, err := ic.projectionJournal(ctx, fmt.Sprint(read.ChatID)) + if err != nil { + return err + } + for range 5 { + p, err := s.load(ctx, k) + if err != nil { + return err + } + cursor := int64(0) + if read.ReadMaxID != nil && *read.ReadMaxID > 0 { + cursor = *read.ReadMaxID + } + if p.ReadMaxID != cursor || p.ManualUnread != read.MarkedUnread { + p.ReceiptObservation = nil + } + p.ReadMaxID = cursor + p.ManualUnread = read.MarkedUnread + p.ReadPending = true + p.UnreadPending = true + p.ReadError = "awaiting_persisted_mapping" + if err = s.save(ctx, k, &p); !errors.Is(err, errProjectionConflict) { + return err + } + } + return errProjectionConflict +} + +type receiptEvent struct { + EventID id.EventID + SourceID int64 +} + +// The actual target timeline must be a source-read prefix. IDs and display +// timestamps alone cannot establish this, especially after backward imports. +type receiptPlan struct { + Target id.EventID + CoveredMax int64 + Complete bool +} + +func receiptPlanForOrder(order []receiptEvent, cursor int64) receiptPlan { + var plan receiptPlan + barrier, blockedRead := false, false + for _, e := range order { + read := e.SourceID > 0 && e.SourceID <= cursor && e.EventID != "" + if !read { + barrier = true + continue + } + if barrier { + blockedRead = true + continue + } + plan.Target = e.EventID + if e.SourceID > plan.CoveredMax { + plan.CoveredMax = e.SourceID + } + } + plan.Complete = plan.CoveredMax >= cursor && cursor > 0 && !blockedRead + return plan +} +func safeReceiptPrefix(order []receiptEvent, cursor int64) id.EventID { + return receiptPlanForOrder(order, cursor).Target +} + +func (ic *InlineClient) projectionPostHandler(ctx context.Context, p *bridgev2.Portal) { + if p != nil { + if chat, err := chatIDFromPortal(p); err == nil { + if d, ok := ic.cachedDialog(chat); ok { + // This callback is already in the portal lane. Only retry a + // durable pending topic here; do not enqueue recursively or + // read Matrix state for every otherwise healthy message. + if meta, ok := p.Metadata.(*PortalMetadata); ok && meta != nil && meta.InlineTopic != nil && !meta.InlineTopic.UserOwned && meta.InlineTopic.Pending != nil && p.MXID != "" && ic.topicProjectionEligible(d) && ic.reserveTopicRetry(chat) { + bounded, cancel := context.WithTimeout(ctx, 5*time.Second) + if err := ic.reconcileSourceTopic(bounded, p, d); err != nil { + zerolog.Ctx(ctx).Warn().Err(err).Msg("Source topic projection remains pending") + } + cancel() + if meta.InlineTopic == nil || meta.InlineTopic.Pending == nil { + ic.clearTopicRetry(chat) + } + } + ic.projectDialogInLane(ctx, p, d) + return + } + } + } + ic.reconcileProjection(ctx, p) +} + +// Limit retries for busy rooms. Idle rooms still require a later portal event. +func (ic *InlineClient) reserveTopicRetry(chat int64) bool { + ic.mu.Lock() + defer ic.mu.Unlock() + if ic.topicRetryAfter == nil { + ic.topicRetryAfter = make(map[int64]time.Time) + } + now := time.Now() + if now.Before(ic.topicRetryAfter[chat]) { + return false + } + ic.topicRetryAfter[chat] = now.Add(30 * time.Second) + return true +} + +func (ic *InlineClient) clearTopicRetry(chat int64) { + ic.mu.Lock() + delete(ic.topicRetryAfter, chat) + ic.mu.Unlock() +} +func (ic *InlineClient) reconcileProjection(ctx context.Context, portal *bridgev2.Portal) { + if portal == nil || portal.MXID == "" { + return + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + s, k, err := ic.projectionJournal(ctx, string(portal.ID)) + if err != nil { + return + } + p, err := s.load(ctx, k) + if err != nil || p.ReadMaxID <= 0 { + return + } + // Even an applied source cursor needs a new target when a late source-read + // message is delivered. Reinspect order; avoid rewriting an unchanged target. + p.ReadPending = true + p.ReadError = "double_puppet_unavailable" + if ic.UserLogin.User == nil { + return + } + raw, ok := ic.UserLogin.User.DoublePuppet(ctx).(*matrix.ASIntent) + if ok && raw != nil && raw.Matrix != nil && raw.IsDoublePuppet() && p.ReadMaxID > 0 { + plan, err := ic.receiptTarget(ctx, portal, raw, p.ReadMaxID) + if err != nil { + p.ReadError = "read_order_unverified" + } else if plan.Target == "" { + p.ReadError = "awaiting_safe_prefix" + } else if plan.Target == p.ReceiptTarget && plan.CoveredMax == p.ReadApplied { + p.ReadPending = !plan.Complete + p.ReadError = "" + if p.ReadPending { + p.ReadError = "awaiting_safe_prefix" + } + } else { + operation := receiptOperation{Key: k, Room: portal.MXID, User: raw.GetMXID(), Target: plan.Target, ReadMaxID: p.ReadMaxID, ManualUnread: p.ManualUnread} + unread, err := applyPrivateReceiptWithEvidence(ctx, raw.Matrix.Client, operation, &p.ReceiptSyncToken, &p.ReceiptObservation) + if err != nil { + p.ReadError = "private_receipt_write_or_readback_failed" + var stage *privateReceiptError + if errors.As(err, &stage) { + p.ReadError += ":" + stage.stage + } + } else { + p.ReceiptObservation = nil + p.ReadApplied = plan.CoveredMax + p.ReceiptTarget = plan.Target + p.ReadPending = !plan.Complete + p.ReadError = "" + if p.ReadPending { + p.ReadError = "awaiting_safe_prefix" + } + if unread != nil && *unread == p.ManualUnread { + p.UnreadPending = false + } else { + p.UnreadPending = true + p.ReadError = "manual_unread_origin_unverified" + } + } + + } + } + _ = s.save(ctx, k, &p) // A stale revision intentionally leaves newer desire pending. +} + +func (ic *InlineClient) receiptTarget(ctx context.Context, p *bridgev2.Portal, raw *matrix.ASIntent, cursor int64) (receiptPlan, error) { + chat, err := chatIDFromPortal(p) + if err != nil { + return receiptPlan{}, err + } + var reverse []receiptEvent + from := "" + for range 20 { // 10k timeline events, then conservatively defer. + page, err := raw.Matrix.Client.Messages(ctx, p.MXID, from, "", mautrix.DirectionBackward, nil, 500) + if err != nil { + return receiptPlan{}, err + } + for _, evt := range page.Chunk { + // State changes are receipt-neutral. Every non-state timeline event + // needs a source mapping: sticker, poll and future content types can + // carry unread state just like text or encrypted messages. + if evt.StateKey != nil || evt.Type.IsState() { + continue + } + record, err := ic.UserLogin.Bridge.DB.Message.GetPartByMXID(ctx, evt.ID) + if err != nil { + return receiptPlan{}, err + } + n := int64(0) + if record != nil && record.Room == p.PortalKey { + sourceChat, id, err := parseMessageID(record.ID) + if err == nil && sourceChat == chat { + n = id + } + } + reverse = append(reverse, receiptEvent{EventID: evt.ID, SourceID: n}) + } + if page.End == "" || len(page.Chunk) == 0 { + slices.Reverse(reverse) + return receiptPlanForOrder(reverse, cursor), nil + } + if page.End == from { + break + } + from = page.End + } + return receiptPlan{}, errors.New("timeline order exceeds bounded inspection") +} + +// Every origin-sensitive local preference write remains disabled. Source modes +// stay cached independently; actual Matrix user changes still go to Inline. +const notificationPolicyDegraded = "notification_policy_degraded: origin attribution unverified; mute channels manually" + +// Keep imported history from being auto-read by the framework's age threshold. +func disableImplicitHistoryRead(br *bridgev2.Bridge) { br.Config.Backfill.UnreadHoursThreshold = 0 } + +func (ic *InlineClient) projectionStatus(ctx context.Context) string { + s, k, err := ic.projectionJournal(ctx, "") + if err != nil { + return "- Presentation journal: unavailable\n" + } + rows, err := s.db.Query(ctx, `SELECT state FROM inline_projection WHERE bridge=$1 AND matrix_user=$2 AND login=$3 AND receiver=$4`, k.Bridge, k.User, k.Login, k.Receiver) + if err != nil { + return "- Presentation journal: unavailable\n" + } + defer rows.Close() + var total, reads, unread, parents, migrations int + for rows.Next() { + var raw string + if err := rows.Scan(&raw); err != nil { + return "- Presentation journal: unavailable\n" + } + var p projectionState + if json.Unmarshal([]byte(raw), &p) != nil { + return "- Presentation journal: unavailable\n" + } + total++ + if p.ReadPending { + reads++ + } + if p.UnreadPending { + unread++ + } + if p.ParentPending { + parents++ + } + if p.Migration != nil && p.Migration.Phase != "applied" && p.Migration.Phase != "rolled_back" { + migrations++ + } + } + if rows.Err() != nil { + return "- Presentation journal: unavailable\n" + } + ic.mu.RLock() + sequence := ic.lastSidecarSequence + var ambiguous, unknownWorkspaces, historyPending int + for _, d := range ic.dialogs { + if classifyDialog(d) == roomUnknown { + ambiguous++ + } + if d.Structure != nil && d.Structure.SpaceID != nil { + access := ic.spaceAccess[*d.Structure.SpaceID] + if access != "accessible" && access != "removed" && !ic.spaceAccessComplete { + unknownWorkspaces++ + } + } + if _, loaded := ic.historyLoaded[d.ChatID]; loaded && d.LastMessageID != nil && *d.LastMessageID > ic.history[d.ChatID] { + historyPending++ + } + } + ic.mu.RUnlock() + backfillEnabled := ic.UserLogin.Bridge.Config != nil && ic.UserLogin.Bridge.Config.Backfill.Enabled + return fmt.Sprintf("- Presentation v%d: %d rooms; pending receipts %d, manual unread %d, hierarchy %d, migration %d; ambiguous %d, unverified workspace access %d\n- Historical pages awaiting coverage: %d; backfill enabled: %t\n- Migration: controlled window required; remote state guards are not atomic\n- Last persisted/acknowledgeable sequence: %d\n- Read fidelity: actual Matrix prefix only; unverified targets remain pending\n", projectionVersion, total, reads, unread, parents, migrations, ambiguous, unknownWorkspaces, historyPending, backfillEnabled, sequence) +} + +// applyPrivateReceipt never emits public m.read. The pinned v0.28 framework +// ignores m.read.private, so an unmarked server echo cannot change Inline. +// It verifies the private receipt itself and that manual-unread was untouched. +type privateReceiptError struct { + stage string + cause error +} + +func (e *privateReceiptError) Error() string { + return "private receipt " + e.stage + ": " + e.cause.Error() +} +func (e *privateReceiptError) Unwrap() error { return e.cause } + +func applyPrivateReceipt(ctx context.Context, cli *mautrix.Client, room id.RoomID, user id.UserID, target id.EventID, cursors ...*string) (*bool, error) { + var cursor *string + if len(cursors) > 0 { + cursor = cursors[0] + } + var evidence *receiptObservation + return applyPrivateReceiptWithEvidence(ctx, cli, receiptOperation{Room: room, User: user, Target: target}, cursor, &evidence) +} + +// An incremental sync can consume the only receipt event before a later GET +// fails. Keep its exact operation and watermark with the advanced cursor so an +// idempotent retry does not depend on the homeserver emitting it again. +func applyPrivateReceiptWithEvidence(ctx context.Context, cli *mautrix.Client, operation receiptOperation, cursorState *string, evidence **receiptObservation) (unread *bool, failure error) { + room, target := operation.Room, operation.Target + if *evidence != nil && (*evidence).Operation != operation { + *evidence = nil + } + stage := "manual_unread_before" + defer func() { + if failure != nil { + failure = &privateReceiptError{stage: stage, cause: failure} + } + }() + before, err := readManualUnread(ctx, cli, room) + if err != nil { + return nil, err + } + filter, _ := json.Marshal(map[string]any{"room": map[string]any{"rooms": []id.RoomID{room}, "timeline": map[string]any{"limit": 1}, "ephemeral": map[string]any{"types": []string{"m.receipt"}}, "state": map[string]any{"types": []string{}}, "account_data": map[string]any{"types": []string{}}}, "presence": map[string]any{"types": []string{}}, "account_data": map[string]any{"types": []string{}}}) + // Synapse may cache an initial sync response. Capture a pre-write stream + // position and read the receipt incrementally so a later target is visible. + stage = "sync_before" + cursor := "" + if cursorState != nil { + cursor = *cursorState + } + observe := func(sync *mautrix.RespSync) error { + if cursorState != nil && sync.NextBatch != "" { + *cursorState = sync.NextBatch + } + return observePrivateReceipt(sync, operation, evidence) + } + beforeSync, err := cli.SyncRequest(ctx, 0, cursor, string(filter), false, event.PresenceOffline) + if err != nil { + return nil, err + } + if err = observe(beforeSync); err != nil { + return nil, err + } + if beforeSync.NextBatch == "" { + return nil, errors.New("private receipt pre-write sync watermark missing") + } + stage = "write_markers" + if err = cli.SetReadMarkers(ctx, room, &mautrix.ReqSetReadMarkers{ReadPrivate: target, FullyRead: target}); err != nil { + return nil, err + } + var fullyRead struct { + EventID id.EventID `json:"event_id"` + } + stage = "fully_read_readback" + if err = cli.GetRoomAccountData(ctx, room, event.AccountDataFullyRead.Type, &fullyRead); err != nil { + return nil, err + } + if fullyRead.EventID != target { + return nil, errors.New("fully-read readback differs") + } + stage = "manual_unread_after" + after, err := readManualUnread(ctx, cli, room) + if err != nil { + return nil, err + } + if (before == nil) != (after == nil) || (before != nil && *before != *after) { + return nil, errors.New("manual unread changed during receipt reconciliation") + } + stage = "sync_after" + sync, err := cli.SyncRequest(ctx, 0, beforeSync.NextBatch, string(filter), false, event.PresenceOffline) + if err != nil { + return nil, err + } + if err = observe(sync); err != nil { + return nil, err + } + stage = "private_receipt_missing" + if *evidence != nil && (*evidence).Operation == operation && (*evidence).SyncToken != "" { + return after, nil + } + return nil, errors.New("private receipt readback missing") +} + +func observePrivateReceipt(sync *mautrix.RespSync, operation receiptOperation, evidence **receiptObservation) error { + joined := sync.Rooms.Join[operation.Room] + if joined == nil { + return nil + } + for _, evt := range joined.Ephemeral.Events { + if evt.Type != event.EphemeralEventReceipt { + continue + } + encoded, err := evt.Content.MarshalJSON() + if err != nil { + *evidence = nil + return err + } + var receipts event.ReceiptEventContent + if err = json.Unmarshal(encoded, &receipts); err != nil { + *evidence = nil + return err + } + found, conflicting := false, false + for target, byType := range receipts { + receipt, ok := byType[event.ReceiptTypeReadPrivate][operation.User] + // SetReadMarkers writes an unthreaded receipt. A thread-local + // receipt does not establish the room-wide operation's target. + if !ok || receipt.ThreadID != "" { + continue + } + if target == operation.Target { + found = true + } else { + conflicting = true + } + } + // Conflicting targets in one event have no usable ordering. + if conflicting { + *evidence = nil + } else if found && sync.NextBatch != "" { + *evidence = &receiptObservation{Operation: operation, SyncToken: sync.NextBatch} + } + } + return nil +} + +func readManualUnread(ctx context.Context, cli *mautrix.Client, room id.RoomID) (*bool, error) { + var content event.MarkedUnreadEventContent + if err := cli.GetRoomAccountData(ctx, room, event.AccountDataMarkedUnread.Type, &content); err != nil { + if errors.Is(err, mautrix.MNotFound) { + return nil, nil + } + return nil, err + } + return &content.Unread, nil +} diff --git a/pkg/connector/projection_test.go b/pkg/connector/projection_test.go new file mode 100644 index 0000000..fed63fa --- /dev/null +++ b/pkg/connector/projection_test.go @@ -0,0 +1,707 @@ +package connector + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "math" + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/appservice" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/matrix" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/event" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "go.mau.fi/util/dbutil" + "hegel.dev/go/hegel" + "maunium.net/go/mautrix/id" +) + +func fixtureJournal(t *testing.T, path string) *projectionStore { + t.Helper() + raw, err := sql.Open("sqlite3", path) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { raw.Close() }) + db, err := dbutil.NewWithDB(raw, "sqlite3") + if err != nil { + t.Fatal(err) + } + s := &projectionStore{db: db} + if err = s.init(t.Context()); err != nil { + t.Fatal(err) + } + return s +} +func TestProjectionJournalRestartIsolationAndCAS(t *testing.T) { + path := filepath.Join(t.TempDir(), "fixture.db") + s := fixtureJournal(t, path) + key := projectionKey{Bridge: "b", User: "@a:test", Login: "1", Receiver: "1", Chat: "7"} + a, err := s.load(t.Context(), key) + if err != nil { + t.Fatal(err) + } + a.ReadMaxID = 10 + a.ManualUnread = true + a.ReadPending = true + if err = s.save(t.Context(), key, &a); err != nil { + t.Fatal(err) + } + stale := a + a.ReadMaxID = 20 + if err = s.save(t.Context(), key, &a); err != nil { + t.Fatal(err) + } + stale.ReadApplied = 10 + if err = s.save(t.Context(), key, &stale); err == nil { + t.Fatal("stale callback overwrote later desire") + } + restarted := fixtureJournal(t, path) + got, err := restarted.load(t.Context(), key) + if err != nil || got.ReadMaxID != 20 || got.ReadApplied != 0 || !got.ManualUnread { + t.Fatalf("restart: %+v %v", got, err) + } + for _, other := range []projectionKey{{Bridge: "c", User: key.User, Login: key.Login, Receiver: key.Receiver, Chat: key.Chat}, {Bridge: key.Bridge, User: "@b:test", Login: key.Login, Receiver: key.Receiver, Chat: key.Chat}, {Bridge: key.Bridge, User: key.User, Login: "2", Receiver: key.Receiver, Chat: key.Chat}, {Bridge: key.Bridge, User: key.User, Login: key.Login, Receiver: "2", Chat: key.Chat}} { + x, e := s.load(t.Context(), other) + if e != nil || x.ReadMaxID != 0 { + t.Fatalf("cross-account leak: %+v %v", x, e) + } + } +} +func TestProjectionSafePrefix(t *testing.T) { + for _, tc := range []struct { + order []int64 + cursor int64 + want id.EventID + }{ + {[]int64{20, 10}, 10, ""}, {[]int64{10, 20}, 10, "$a"}, {[]int64{3, 10, 20}, 10, "$b"}, {[]int64{0, 10}, 10, ""}, {[]int64{10, 5, 20}, 10, "$b"}, {nil, 10, ""}, + } { + rows := make([]receiptEvent, len(tc.order)) + for i, n := range tc.order { + rows[i] = receiptEvent{EventID: id.EventID("$" + string(rune('a'+i))), SourceID: n} + } + if got := safeReceiptPrefix(rows, tc.cursor); got != tc.want { + t.Errorf("%v cursor%d got%s want%s", tc.order, tc.cursor, got, tc.want) + } + } +} +func TestProjectionPrefixNeverCoversUnread(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + ids := hegel.Draw(ht, hegel.Lists(hegel.Integers[int64](math.MinInt64, math.MaxInt64))) + cursor := int64(hegel.Draw(ht, hegel.Integers[int64](0, math.MaxInt64))) + rows := make([]receiptEvent, len(ids)) + for i, n := range ids { + rows[i] = receiptEvent{EventID: id.EventID(string(rune(i + 1))), SourceID: int64(n)} + } + got := safeReceiptPrefix(rows, cursor) + if got != "" { + for _, row := range rows { + if row.SourceID <= 0 || row.SourceID > cursor { + ht.Fatal("receipt covers unread/unknown source message") + } + if row.EventID == got { + return + } + } + ht.Fatal("unmapped receipt") + } + // Positive direction: a nonempty all-read prefix must produce its last event. + if len(rows) > 0 && rows[0].SourceID > 0 && rows[0].SourceID <= cursor { + ht.Fatal("lost valid safe prefix") + } + }, hegel.WithTestCases(3000), hegel.WithDatabase(t.TempDir())) +} + +func TestPrivateReceiptUsesOnlyPrivateMarkerAndPreservesUnread(t *testing.T) { + var posted map[string]any + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/read_markers"): + if err := json.NewDecoder(r.Body).Decode(&posted); err != nil { + t.Error(err) + } + json.NewEncoder(w).Encode(map[string]any{}) + case strings.HasSuffix(r.URL.Path, "/m.fully_read"): + json.NewEncoder(w).Encode(map[string]any{"event_id": "$read"}) + case strings.HasSuffix(r.URL.Path, "/m.marked_unread"): + if r.Method != "GET" { + t.Error("source read wrote manual-unread account data") + } + json.NewEncoder(w).Encode(map[string]any{"unread": true}) + case strings.HasSuffix(r.URL.Path, "/sync"): + w.Write([]byte(`{"next_batch":"before","rooms":{"join":{"!room:test":{"ephemeral":{"events":[{"type":"m.receipt","content":{"$read":{"m.read.private":{"@user:test":{}}}}}]}}}}}`)) + default: + t.Errorf("unexpected Matrix call %s", r.URL.Path) + http.Error(w, "unexpected", 500) + } + })) + defer server.Close() + cli, err := mautrix.NewClient(server.URL, "@user:test", "fixture-token") + if err != nil { + t.Fatal(err) + } + unread, err := applyPrivateReceipt(t.Context(), cli, "!room:test", "@user:test", "$read") + if err != nil || unread == nil || !*unread { + t.Fatalf("private receipt readback=%v %v", unread, err) + } + if posted["m.read"] != nil || posted["m.read.private"] != "$read" || posted["m.fully_read"] != "$read" { + t.Fatalf("unsafe marker request: %v", posted) + } +} +func TestFrameworkPrivateReceiptCannotEchoToInline(t *testing.T) { + // A nil bridge would panic on user/source lookup. The pinned exported seam + // must ignore private receipts entirely, including delayed replay after restart. + for range 2 { + p := testPortal("7") + content := event.ReceiptEventContent{} + content.Set("$read", event.ReceiptTypeReadPrivate, "@user:test", event.ReadReceipt{}) + result := p.Internal().HandleMatrixReceipts(t.Context(), &event.Event{Type: event.EphemeralEventReceipt, Content: event.Content{Parsed: &content}}) + if !result.Success { + t.Fatal("private receipt rejected before safe filtering") + } + } +} + +func TestHistoryCompletionRejectsMissingAndFakeMappings(t *testing.T) { + key := networkid.PortalKey{ID: "7", Receiver: "1"} + records := []sidecar.MessageRecord{{ChatID: 7, MessageID: 20}, {ChatID: 7, MessageID: 10}} + rows := map[networkid.MessageID]*database.Message{} + lookup := func(mid networkid.MessageID) (*database.Message, error) { return rows[mid], nil } + if persistedHistoryPrefix(key, records, lookup) != 0 { + t.Fatal("empty callback advanced delivery") + } + rows["7/20"] = &database.Message{Room: key, ID: "7/20", MXID: "$20"} + if persistedHistoryPrefix(key, records, lookup) != 0 { + t.Fatal("missing lower mapping skipped") + } + rows["7/10"] = &database.Message{Room: key, ID: "7/10", MXID: "$10"} + if persistedHistoryPrefix(key, records, lookup) != 20 { + t.Fatal("complete mapping page not checkpointed") + } + rows["7/10"].SetFakeMXID() + if persistedHistoryPrefix(key, records, lookup) != 0 { + t.Fatal("fake Matrix event checkpointed") + } +} + +type noPuppetMatrix struct{ bridgev2.MatrixConnector } + +func (noPuppetMatrix) NewUserIntent(context.Context, id.UserID, string) (bridgev2.MatrixAPI, string, error) { + return nil, "", errors.New("fixture has no puppet") +} +func fixtureProjectionClient(t *testing.T, store *projectionStore) *InlineClient { + t.Helper() + br := &bridgev2.Bridge{DB: &database.Database{Database: store.db, BridgeID: "fixture"}, Matrix: noPuppetMatrix{}} + user := &bridgev2.User{User: &database.User{MXID: "@user:test"}, Bridge: br} + return &InlineClient{UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ID: "login"}, User: user, Bridge: br}} +} +func TestReadIntentPrecedesDeliveryAndMissingPuppetStaysPendingAfterRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "read.db") + store := fixtureJournal(t, path) + sourceWrites := 0 + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rpc/state/chat" { + writeConnectorSidecarResult(t, w, "chat_state", sidecar.ChatStateSnapshot{ChatID: 7, ReadState: &sidecar.ReadStateRecord{ChatID: 7, ReadMaxID: foundationInt64(10), MarkedUnread: true}}) + } else { + sourceWrites++ + http.Error(w, "unexpected source write", 500) + } + })) + defer server.Close() + ic := fixtureProjectionClient(t, store) + ic.Sidecar = sidecar.NewClient(server.URL) + if err := ic.readDesireBeforeDelivery(t.Context(), 7); err != nil { + t.Fatal(err) + } + _, key, err := ic.projectionJournal(t.Context(), "7") + if err != nil { + t.Fatal(err) + } + p, err := store.load(t.Context(), key) + if err != nil || p.ReadMaxID != 10 || !p.ManualUnread || !p.ReadPending || p.ReadApplied != 0 { + t.Fatal("source intent not durable before delivery", err) + } + restarted := fixtureProjectionClient(t, fixtureJournal(t, path)) + portal := &bridgev2.Portal{Portal: &database.Portal{PortalKey: networkid.PortalKey{ID: "7", Receiver: "login"}, MXID: "!room:test"}} + restarted.reconcileProjection(t.Context(), portal) + p, err = store.load(t.Context(), key) + if err != nil || !p.ReadPending || !p.UnreadPending || p.ReadApplied != 0 || p.ReadError != "double_puppet_unavailable" || sourceWrites != 0 { + t.Fatalf("missing puppet falsely applied or echoed source: %+v writes%d %v", p, sourceWrites, err) + } +} + +func TestReceiptCoverageUsesPrefixMaximumAndRearmsForLateRead(t *testing.T) { + order := []receiptEvent{{EventID: "$20", SourceID: 20}, {EventID: "$10", SourceID: 10}} + plan := receiptPlanForOrder(order, 20) + if plan.Target != "$10" || plan.CoveredMax != 20 || !plan.Complete { + t.Fatalf("timestamp disorder falsely stays pending: %+v", plan) + } + order = append(order, receiptEvent{EventID: "$15", SourceID: 15}) + late := receiptPlanForOrder(order, 20) + if late.Target != "$15" || late.CoveredMax != 20 || !late.Complete { + t.Fatalf("late source-read event did not rearm target: %+v", late) + } + blocked := receiptPlanForOrder(append(order, receiptEvent{EventID: "$25", SourceID: 25}, receiptEvent{EventID: "$16", SourceID: 16}), 20) + if blocked.Target != "$15" || blocked.CoveredMax != 20 || blocked.Complete { + t.Fatalf("nonprefix read falsely completed: %+v", blocked) + } +} + +func TestReceiptPlanCoversOnlyActualReadPrefix(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + ids := hegel.Draw(ht, hegel.Lists(hegel.Integers[int64](math.MinInt64, math.MaxInt64))) + cursor := int64(hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64))) + rows := make([]receiptEvent, len(ids)) + for i, n := range ids { + rows[i] = receiptEvent{EventID: id.EventID(fmt.Sprintf("$%d", i)), SourceID: int64(n)} + } + plan := receiptPlanForOrder(rows, cursor) + end := 0 + for end < len(ids) && ids[end] > 0 && int64(ids[end]) <= cursor { + end++ + } + var maximum int64 + for _, n := range ids[:end] { + if int64(n) > maximum { + maximum = int64(n) + } + } + if plan.CoveredMax != maximum { + ht.Fatal("coverage differs from maximum source ID in safe prefix") + } + if end == 0 && plan.Target != "" { + ht.Fatal("empty safe prefix emitted receipt") + } + if end > 0 && plan.Target != rows[end-1].EventID { + ht.Fatal("safe prefix did not target final Matrix event") + } + allKnownReadsCovered := true + for _, n := range ids[end:] { + if n > 0 && int64(n) <= cursor { + allKnownReadsCovered = false + } + } + if plan.Complete != (maximum >= cursor && allKnownReadsCovered) { + ht.Fatal("completion hides nonprefix read message or missing cursor") + } + }, hegel.WithTestCases(3000), hegel.WithDatabase(t.TempDir())) +} + +func TestPrivateReceiptFailuresNeverImplyReadCompletion(t *testing.T) { + for _, failure := range []string{"write", "fully_read", "manual_unread", "private_readback"} { + t.Run(failure, func(t *testing.T) { + unreadReads := 0 + markerWrites := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/read_markers"): + markerWrites++ + var content map[string]any + _ = json.NewDecoder(r.Body).Decode(&content) + if _, public := content["m.read"]; public { + t.Error("source read emitted public receipt") + } + if failure == "write" { + w.WriteHeader(400) + w.Write([]byte(`{"errcode":"M_BAD_JSON","error":"injected"}`)) + return + } + w.Write([]byte(`{}`)) + case strings.HasSuffix(r.URL.Path, "/m.fully_read"): + target := "$target" + if failure == "fully_read" { + target = "$other" + } + json.NewEncoder(w).Encode(map[string]any{"event_id": target}) + case strings.HasSuffix(r.URL.Path, "/m.marked_unread"): + if r.Method != "GET" { + t.Error("source read wrote manual unread") + } + unreadReads++ + unread := true + if failure == "manual_unread" && unreadReads > 1 { + unread = false + } + json.NewEncoder(w).Encode(map[string]any{"unread": unread}) + case strings.HasSuffix(r.URL.Path, "/sync"): + w.Write([]byte(`{"next_batch":"before","rooms":{"join":{"!room:test":{"ephemeral":{"events":[]}}}}}`)) + default: + t.Errorf("unexpected call %s", r.URL.Path) + w.WriteHeader(400) + } + })) + defer server.Close() + cli, err := mautrix.NewClient(server.URL, "@user:test", "fixture") + if err != nil { + t.Fatal(err) + } + if _, err = applyPrivateReceipt(t.Context(), cli, "!room:test", "@user:test", "$target"); err == nil { + t.Fatal("failed write/readback falsely completed") + } + if markerWrites != 1 { + t.Fatal("receipt write repeated", markerWrites) + } + }) + } +} + +func TestReceiptScannerBlocksUnreadStickerPollAndUnknownEvents(t *testing.T) { + store := fixtureJournal(t, filepath.Join(t.TempDir(), "scanner.db")) + db := database.New("fixture", database.MetaTypes{}, store.db) + if err := db.Upgrade(t.Context()); err != nil { + t.Fatal(err) + } + key := networkid.PortalKey{ID: "7", Receiver: "login"} + if err := db.Portal.Insert(t.Context(), &database.Portal{PortalKey: key}); err != nil { + t.Fatal(err) + } + for _, n := range []int64{10, 20} { + if err := db.Message.Insert(t.Context(), &database.Message{ID: makeMessageID(7, n), MXID: id.EventID(fmt.Sprintf("$%d", n)), Room: key}); err != nil { + t.Fatal(err) + } + } + blocker := "m.sticker" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"chunk": []map[string]any{ + {"event_id": "$20", "type": "m.room.message", "content": map[string]any{"body": "20", "msgtype": "m.text"}}, + {"event_id": "$unknown", "type": blocker, "content": map[string]any{}}, + {"event_id": "$10", "type": "m.room.message", "content": map[string]any{"body": "10", "msgtype": "m.text"}}, + }}) + })) + defer server.Close() + cli, err := mautrix.NewClient(server.URL, "@user:test", "fixture") + if err != nil { + t.Fatal(err) + } + ic := fixtureProjectionClient(t, store) + ic.UserLogin.Bridge.DB = db + portal := &bridgev2.Portal{Portal: &database.Portal{PortalKey: key, MXID: "!room:test"}} + raw := &matrix.ASIntent{Matrix: &appservice.IntentAPI{Client: cli}} + for _, typ := range []string{"m.sticker", "org.matrix.msc3381.poll.start", "unknown.content"} { + blocker = typ + plan, err := ic.receiptTarget(t.Context(), portal, raw, 20) + if err != nil || plan.Target != "$10" || plan.Complete { + t.Fatalf("scanner crossed unread-capable %s: %+v %v", typ, plan, err) + } + } +} + +func TestHistoricalRecoverySelectsMissingIDsDespiteTimestampCutoff(t *testing.T) { + records := []sidecar.MessageRecord{{ChatID: 7, MessageID: 20, Timestamp: 100}, {ChatID: 7, MessageID: 10, Timestamp: 200}} + key := networkid.PortalKey{ID: "7", Receiver: "login"} + // Characterize the pinned framework path we must bypass for source IDs. + p := testPortal("7") + messages := []*bridgev2.BackfillMessage{{ID: "7/20", Timestamp: time.Unix(100, 0)}, {ID: "7/10", Timestamp: time.Unix(200, 0)}} + cut := p.Internal().CutoffMessages(t.Context(), messages, false, true, &database.Message{ID: "7/30", Timestamp: time.Unix(300, 0)}) + if len(cut) != 0 { + t.Fatal("pinned framework timestamp-cutoff characterization changed") + } + lookup := func(networkid.MessageID) (*database.Message, error) { return nil, nil } + missing, err := pendingHistoricalRecords(key, records, lookup) + if err != nil || len(missing) != 2 { + t.Fatal("late source messages incorrectly discarded", err) + } + rows := map[networkid.MessageID]*database.Message{"7/20": {Room: key, MXID: "$20"}} + lookup = func(mid networkid.MessageID) (*database.Message, error) { return rows[mid], nil } + missing, err = pendingHistoricalRecords(key, records, lookup) + if err != nil || len(missing) != 1 || missing[0].MessageID != 10 { + t.Fatal("recovery repeated mapped event or lost missing event", err) + } + other := append(records, sidecar.MessageRecord{ChatID: 8, MessageID: 10}) + if _, err = pendingHistoricalRecords(key, other, lookup); err == nil { + t.Fatal("cross-chat history bundle accepted") + } +} + +func TestPrivateReceiptReadbackUsesPrewriteSyncWatermark(t *testing.T) { + posted := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/m.marked_unread"): + w.Write([]byte(`{"unread":true}`)) + case strings.HasSuffix(r.URL.Path, "/read_markers"): + posted = true + w.Write([]byte(`{}`)) + case strings.HasSuffix(r.URL.Path, "/m.fully_read"): + w.Write([]byte(`{"event_id":"$late"}`)) + case strings.HasSuffix(r.URL.Path, "/sync"): + target := "$cached" + if r.URL.Query().Get("since") == "before" && posted { + target = "$late" + } + json.NewEncoder(w).Encode(map[string]any{"next_batch": "before", "rooms": map[string]any{"join": map[string]any{"!room:test": map[string]any{"ephemeral": map[string]any{"events": []map[string]any{{"type": "m.receipt", "content": map[string]any{target: map[string]any{"m.read.private": map[string]any{"@user:test": map[string]any{}}}}}}}}}}}) + default: + w.WriteHeader(400) + } + })) + defer server.Close() + cli, err := mautrix.NewClient(server.URL, "@user:test", "fixture") + if err != nil { + t.Fatal(err) + } + if _, err = applyPrivateReceipt(t.Context(), cli, "!room:test", "@user:test", "$late"); err != nil { + t.Fatal("cached initial sync hid successful private receipt", err) + } +} + +func TestPrivateReceiptRetryAdvancesCachedSync(t *testing.T) { + cursor := "" + posted := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/m.marked_unread"): + w.Write([]byte(`{"unread":true}`)) + case strings.HasSuffix(r.URL.Path, "/m.fully_read"): + w.Write([]byte(`{"event_id":"$late"}`)) + case strings.HasSuffix(r.URL.Path, "/read_markers"): + posted = true + w.Write([]byte(`{}`)) + case strings.HasSuffix(r.URL.Path, "/sync"): + since := r.URL.Query().Get("since") + next := "initial" + target := "$old" + if since == "initial" { + next = "cached" + } + if since == "cached" && posted { + next = "fresh" + target = "$late" + } + json.NewEncoder(w).Encode(map[string]any{"next_batch": next, "rooms": map[string]any{"join": map[string]any{"!room:test": map[string]any{"ephemeral": map[string]any{"events": []map[string]any{{"type": "m.receipt", "content": map[string]any{target: map[string]any{"m.read.private": map[string]any{"@user:test": map[string]any{}}}}}}}}}}}) + default: + w.WriteHeader(400) + } + })) + defer server.Close() + cli, _ := mautrix.NewClient(server.URL, "@user:test", "fixture") + for range 3 { + _, err := applyPrivateReceipt(t.Context(), cli, "!room:test", "@user:test", "$late", &cursor) + if err == nil && cursor == "fresh" { + return + } + } + t.Fatal("retry never advanced beyond cached sync cursor", cursor) +} + +func TestFinalReviewReceiptEvidenceSurvivesFailedReadback(t *testing.T) { + // This server returns incremental receipt changes, not a snapshot on every + // sync. Repeating the same receipt is an idempotent no-op. + atTarget := false + fullReads := 0 + markerChanges := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/m.marked_unread"): + w.Write([]byte(`{"unread":true}`)) + case strings.HasSuffix(r.URL.Path, "/read_markers"): + if !atTarget { + atTarget = true + markerChanges++ + } + w.Write([]byte(`{}`)) + case strings.HasSuffix(r.URL.Path, "/m.fully_read"): + fullReads++ + if fullReads <= 2 { + w.WriteHeader(503) + w.Write([]byte(`{"errcode":"M_UNKNOWN","error":"synthetic transient readback failure"}`)) + return + } + w.Write([]byte(`{"event_id":"$target"}`)) + case strings.HasSuffix(r.URL.Path, "/sync"): + events := []any{} + next := "s0" + if atTarget { + next = "s1" + if r.URL.Query().Get("since") != "s1" { + events = append(events, map[string]any{"type": "m.receipt", "content": map[string]any{"$target": map[string]any{"m.read.private": map[string]any{"@user:test": map[string]any{}}}}}) + } + } + json.NewEncoder(w).Encode(map[string]any{"next_batch": next, "rooms": map[string]any{"join": map[string]any{"!room:test": map[string]any{"ephemeral": map[string]any{"events": events}}}}}) + default: + t.Errorf("unexpected route %s", r.URL.Path) + w.WriteHeader(400) + } + })) + defer server.Close() + cli, err := mautrix.NewClient(server.URL, "@user:test", "synthetic") + if err != nil { + t.Fatal(err) + } + cli.DefaultHTTPRetries = 0 + path := filepath.Join(t.TempDir(), "receipt.db") + store := fixtureJournal(t, path) + key := projectionKey{Bridge: "b", User: "@user:test", Login: "login", Receiver: "login", Chat: "7"} + state, _ := store.load(t.Context(), key) + state.ReadMaxID = 20 + state.ReadPending = true + if err = store.save(t.Context(), key, &state); err != nil { + t.Fatal(err) + } + for attempt := 1; attempt <= 4; attempt++ { + // Match reconcileProjection: retain the returned sync token even on an + // apply error, then reload it from a new SQLite handle for retry. + state, err = store.load(t.Context(), key) + if err != nil { + t.Fatal(err) + } + _, err = applyPrivateReceiptWithEvidence(t.Context(), cli, receiptOperation{Key: key, Room: "!room:test", User: "@user:test", Target: "$target", ReadMaxID: state.ReadMaxID, ManualUnread: state.ManualUnread}, &state.ReceiptSyncToken, &state.ReceiptObservation) + t.Logf("attempt%d token=%s err=%v", attempt, state.ReceiptSyncToken, err) + if attempt <= 2 && err == nil { + t.Fatal("injected readback failure ignored") + } + if attempt > 2 && err == nil { + return + } + if saveErr := store.save(t.Context(), key, &state); saveErr != nil { + t.Fatal(saveErr) + } + store = fixtureJournal(t, path) + } + t.Fatalf("correct private and fully-read target remains pending after failures clear; token=%s, marker_changes=%d", state.ReceiptSyncToken, markerChanges) +} + +func TestPrivateReceiptEvidenceCannotCrossOperation(t *testing.T) { + base := receiptOperation{Key: projectionKey{Bridge: "bridge", User: "@login:test", Login: "login", Receiver: "receiver", Chat: "7"}, Room: "!room:test", User: "@user:test", Target: "$target", ReadMaxID: 20, ManualUnread: true} + for _, field := range []string{"same", "bridge", "account", "login", "receiver", "chat", "room", "puppet", "target", "cursor", "manual_unread"} { + t.Run(field, func(t *testing.T) { + op := base + switch field { + case "bridge": + op.Key.Bridge += "other" + case "account": + op.Key.User += "other" + case "login": + op.Key.Login += "other" + case "receiver": + op.Key.Receiver += "other" + case "chat": + op.Key.Chat = "8" + case "room": + op.Room = "!other:test" + case "puppet": + op.User = "@other:test" + case "target": + op.Target = "$other" + case "cursor": + op.ReadMaxID++ + case "manual_unread": + op.ManualUnread = false + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/m.marked_unread"): + w.Write([]byte(`{"unread":true}`)) + case strings.HasSuffix(r.URL.Path, "/read_markers"): + w.Write([]byte(`{}`)) + case strings.HasSuffix(r.URL.Path, "/m.fully_read"): + json.NewEncoder(w).Encode(map[string]any{"event_id": op.Target}) + case strings.HasSuffix(r.URL.Path, "/sync"): + w.Write([]byte(`{"next_batch":"s2","rooms":{"join":{}}}`)) + default: + t.Errorf("unexpected route %s", r.URL.Path) + w.WriteHeader(400) + } + })) + defer server.Close() + cli, err := mautrix.NewClient(server.URL, op.User, "synthetic") + if err != nil { + t.Fatal(err) + } + evidence := &receiptObservation{Operation: base, SyncToken: "s1"} + cursor := "s1" + _, err = applyPrivateReceiptWithEvidence(t.Context(), cli, op, &cursor, &evidence) + if field == "same" { + if err != nil { + t.Fatal("lost exact retained proof", err) + } + } else if err == nil || evidence != nil { + t.Fatalf("evidence crossed %s: evidence%+v err%v", field, evidence, err) + } + }) + } +} + +func TestPrivateReceiptEvidenceRejectsConflictingAndThreadedTargets(t *testing.T) { + op := receiptOperation{Room: "!room:test", User: "@user:test", Target: "$target"} + for _, tc := range []struct { + name, content string + retained bool + }{ + {"exact", `{"$target":{"m.read.private":{"@user:test":{}}}}`, true}, + {"later-other", `{"$other":{"m.read.private":{"@user:test":{}}}}`, false}, + {"ambiguous", `{"$target":{"m.read.private":{"@user:test":{}}},"$other":{"m.read.private":{"@user:test":{}}}}`, false}, + {"public-only", `{"$target":{"m.read":{"@user:test":{}}}}`, false}, + {"thread-only", `{"$target":{"m.read.private":{"@user:test":{"thread_id":"$thread"}}}}`, false}, + {"main-only", `{"$target":{"m.read.private":{"@user:test":{"thread_id":"main"}}}}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var response mautrix.RespSync + raw := `{"next_batch":"s1","rooms":{"join":{"!room:test":{"ephemeral":{"events":[{"type":"m.receipt","content":` + tc.content + `}]}}}}}` + if err := json.Unmarshal([]byte(raw), &response); err != nil { + t.Fatal(err) + } + var evidence *receiptObservation + if tc.name == "later-other" || tc.name == "ambiguous" { + evidence = &receiptObservation{Operation: op, SyncToken: "s0"} + } + if err := observePrivateReceipt(&response, op, &evidence); err != nil { + t.Fatal(err) + } + if (evidence != nil) != tc.retained { + t.Fatalf("evidence%+v wantRetained%t", evidence, tc.retained) + } + }) + } +} + +func TestReceiptEvidenceClearsOnNewDesireAndRejectsStaleCompletion(t *testing.T) { + ic := fixtureHistoryClient(t, filepath.Join(t.TempDir(), "intent.db")) + ic.UserLogin.User = &bridgev2.User{User: &database.User{MXID: "@user:test"}, Bridge: ic.UserLogin.Bridge} + read := sidecar.ReadStateRecord{ChatID: 7, ReadMaxID: foundationInt64(10), MarkedUnread: true} + if err := ic.storeReadDesire(t.Context(), &read); err != nil { + t.Fatal(err) + } + journal, key, err := ic.projectionJournal(t.Context(), "7") + if err != nil { + t.Fatal(err) + } + state, err := journal.load(t.Context(), key) + if err != nil { + t.Fatal(err) + } + state.ReceiptObservation = &receiptObservation{Operation: receiptOperation{Key: key, Room: "!room:test", User: "@user:test", Target: "$10", ReadMaxID: 10, ManualUnread: true}, SyncToken: "s1"} + if err = journal.save(t.Context(), key, &state); err != nil { + t.Fatal(err) + } + stale := state + read.ReadMaxID = foundationInt64(20) + if err = ic.storeReadDesire(t.Context(), &read); err != nil { + t.Fatal(err) + } + stale.ReadApplied = 10 + if err = journal.save(t.Context(), key, &stale); !errors.Is(err, errProjectionConflict) { + t.Fatalf("stale receipt completion accepted: %v", err) + } + // Returning to the same scalar cursor is a new desire. It must not revive + // a proof from the operation before the intervening cursor change. + read.ReadMaxID = foundationInt64(10) + if err = ic.storeReadDesire(t.Context(), &read); err != nil { + t.Fatal(err) + } + state, err = journal.load(t.Context(), key) + if err != nil || state.ReceiptObservation != nil || state.ReadApplied != 0 || !state.ReadPending { + t.Fatalf("old evidence revived: %+v %v", state, err) + } +} diff --git a/pkg/connector/reply_context.go b/pkg/connector/reply_context.go new file mode 100644 index 0000000..f40b8f1 --- /dev/null +++ b/pkg/connector/reply_context.go @@ -0,0 +1,71 @@ +package connector + +import ( + "context" + "errors" + "fmt" + "net/url" + "strings" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/event" +) + +// Matrix threads share their parent's audience; Inline child chats retain +// independent identity and access. No native flattening/creation is enabled. +func validateInlineReplyTarget(msg *bridgev2.MatrixMessage, chatID int64) error { + if msg.ThreadRoot != nil || (msg.Content != nil && msg.Content.RelatesTo != nil && msg.Content.RelatesTo.Type == event.RelThread) { + return errors.New("Inline native Matrix thread routing is unsupported; use the separate child conversation") + } + if msg.ReplyTo != nil { + if msg.Portal == nil || msg.Portal.Portal == nil || msg.ReplyTo.Room != msg.Portal.PortalKey { + return errors.New("Inline reply target must belong to this conversation and login") + } + parent, root, err := parseMessageID(msg.ReplyTo.ID) + if err != nil || parent != chatID || root <= 0 { + return errors.New("Inline reply target must belong to this conversation") + } + } + return nil +} + +// This text is written only inside the eligible child, never into the parent. +// Missing/deleted/unprojected roots retain readable source context. Lookup is +// existing-only, receiver-scoped and never creates a portal or fetches history. +func (ic *InlineClient) replyChildTopic(ctx context.Context, d sidecar.DialogRecord) (string, error) { + kind := classifyDialog(d) + if kind != roomReplyChild { + if kind == roomChannel { + return optionalStringValue(d.Description), nil + } + return "", nil + } + structure := d.Structure + text := fmt.Sprintf("Inline reply context: chat %d", *structure.ParentChatID) + if structure.ParentMessageID != nil { + text += fmt.Sprintf(", message %d", *structure.ParentMessageID) + } + text += ". This is a separate conversation." + if ic.UserLogin != nil && ic.UserLogin.UserLogin != nil && ic.UserLogin.ID != "" && ic.UserLogin.Bridge != nil && ic.UserLogin.Bridge.DB != nil && structure.ParentMessageID != nil { + key := ic.portalKeyForChat(*structure.ParentChatID) + parent, err := ic.UserLogin.Bridge.GetExistingPortalByKey(ctx, key) + if err != nil { + return "", fmt.Errorf("read parent portal mapping: %w", err) + } + if parent != nil && parent.PortalKey == key && parent.MXID != "" { + row, err := ic.UserLogin.Bridge.DB.Message.GetFirstPartByID(ctx, key.Receiver, makeMessageID(*structure.ParentChatID, *structure.ParentMessageID)) + if err != nil { + return "", fmt.Errorf("read parent message mapping: %w", err) + } + if row != nil && row.Room == key && row.MXID != "" && !row.HasFakeMXID() { + link := url.URL{Scheme: "https", Host: "matrix.to", Fragment: "/" + string(parent.MXID) + "/" + string(row.MXID)} + text += " Parent message: " + link.String() + } + } + } + if description := strings.TrimSpace(optionalStringValue(d.Description)); description != "" { + return description + "\n\n" + text, nil + } + return text, nil +} diff --git a/pkg/connector/reply_context_test.go b/pkg/connector/reply_context_test.go new file mode 100644 index 0000000..ce3eaf6 --- /dev/null +++ b/pkg/connector/reply_context_test.go @@ -0,0 +1,188 @@ +package connector + +import ( + "encoding/json" + "fmt" + "math" + "net/http" + "path/filepath" + "strings" + "testing" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "hegel.dev/go/hegel" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/event" +) + +func TestSeparateChildRetainsSourceParentContext(t *testing.T) { + parent, root := int64(3), int64(9) + d := sidecar.DialogRecord{ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "reply_child", ParentChatID: &parent, ParentMessageID: &root}} + ic := &InlineClient{AccountID: "1", UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ID: "1"}}} + info := ic.chatInfoForDialog(t.Context(), d, nil) + if info.Type == nil || *info.Type != database.RoomTypeDefault || info.ParentID != nil || !strings.Contains(mustReplyTopic(t, ic, d), "chat 3") || !strings.Contains(mustReplyTopic(t, ic, d), "message 9") { + t.Fatalf("child identity/parent context lost: %+v", info) + } + d.ChatListHidden = dirPtr(true) + if ic.shouldProjectDialog(d) { + t.Fatal("context auto-projected a hidden child") + } +} + +func TestSeparateChildNeverSendsParentLocalReplyID(t *testing.T) { + calls := 0 + s := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + mid := int64(11) + writeConnectorSidecarResult(t, w, "message", sidecar.MessageMutation{State: sidecar.TransactionCompleted, MessageID: &mid}) + })) + defer s.Close() + ic := richClient() + ic.Sidecar = sidecar.NewClient(s.URL) + for _, thread := range []bool{false, true} { + msg := &bridgev2.MatrixMessage{MatrixEventBase: bridgev2.MatrixEventBase[*event.MessageEventContent]{Portal: testPortal("7"), Content: &event.MessageEventContent{MsgType: event.MsgText, Body: "child message"}}} + if thread { + msg.ThreadRoot = &database.Message{ID: "3/9"} + } else { + msg.ReplyTo = &database.Message{ID: "3/9"} + } + if _, err := ic.HandleMatrixMessage(t.Context(), msg); err == nil || calls != 0 { + t.Fatalf("thread/cross-chat reply flattened: %v calls%d", err, calls) + } + } +} + +func TestSeparateChildThreadBackfillReturnsExplicitUnsupported(t *testing.T) { + if _, err := (&InlineClient{}).FetchMessages(t.Context(), bridgev2.FetchMessagesParams{ThreadRoot: "3/9"}); err == nil { + t.Fatal("unsupported Matrix thread reported an empty successful history") + } +} + +func TestSeparateChildBacklinkUsesPersistedReceiverMapping(t *testing.T) { + path := filepath.Join(t.TempDir(), "context.db") + ic := fixtureHistoryClient(t, path) + key := ic.portalKeyForChat(3) + if err := ic.UserLogin.Bridge.DB.Portal.Insert(t.Context(), &database.Portal{PortalKey: key, MXID: "!parent:test", Metadata: &PortalMetadata{}}); err != nil { + t.Fatal(err) + } + if err := ic.UserLogin.Bridge.DB.Message.Insert(t.Context(), &database.Message{ID: "3/9", MXID: "$root", Room: key, Metadata: &MessageMetadata{}}); err != nil { + t.Fatal(err) + } + d := sidecar.DialogRecord{ChatID: 7, Description: stringPtr("native child"), Structure: &sidecar.DialogStructure{Kind: "reply_child", ParentChatID: dirPtr(int64(3)), ParentMessageID: dirPtr(int64(9))}} + restarted := fixtureHistoryClient(t, path) + info := restarted.chatInfoForDialog(t.Context(), d, nil) + topic := mustReplyTopic(t, restarted, d) + if !strings.Contains(topic, "native child") || !strings.Contains(topic, "!parent:test/$root") { + t.Fatalf("restart lost persisted backlink: %+v", info.Topic) + } + // The same native IDs on another login cannot reuse the first receiver's + // existing portal/message mapping or create a parent as a side effect. + restarted.UserLogin.ID = "other-login" + info = restarted.chatInfoForDialog(t.Context(), d, nil) + topic = mustReplyTopic(t, restarted, d) + if strings.Contains(topic, "matrix.to") || !strings.Contains(topic, "chat 3, message 9") { + t.Fatalf("cross-receiver backlink: %+v", info.Topic) + } + if parent, err := restarted.UserLogin.Bridge.DB.Portal.GetByKey(t.Context(), restarted.portalKeyForChat(3)); err != nil || parent != nil { + t.Fatalf("context lookup created a parent: %+v %v", parent, err) + } +} + +// The pinned framework resolves relations globally by Matrix event ID. Keep its +// actual DB lookup in this regression: constructing a MatrixMessage alone would +// miss the cross-login provenance of an otherwise identical source chat ID. +func TestSeparateChildFrameworkRejectsCrossReceiverReply(t *testing.T) { + for _, mode := range []string{"cross-receiver", "same-room", "raw-thread", "raw-thread-no-fallback"} { + t.Run(mode, func(t *testing.T) { + cross := mode == "cross-receiver" + ic := fixtureHistoryClient(t, filepath.Join(t.TempDir(), "reply.db")) + ic.AccountID = "1" + ic.UserLogin.Client = ic + br := ic.UserLogin.Bridge + portal, err := br.GetPortalByKey(t.Context(), ic.portalKeyForChat(7)) + if err != nil { + t.Fatal(err) + } + portal.MXID = "!current:test" + if err = portal.Save(t.Context()); err != nil { + t.Fatal(err) + } + target := portal.PortalKey + if cross { + target.Receiver = "other-login" + if err = br.DB.Portal.Insert(t.Context(), &database.Portal{PortalKey: target, MXID: "!other:test", Metadata: &PortalMetadata{}}); err != nil { + t.Fatal(err) + } + } + if err = br.DB.Message.Insert(t.Context(), &database.Message{ID: "7/9", Room: target, MXID: "$target", Metadata: &MessageMetadata{}}); err != nil { + t.Fatal(err) + } + calls := 0 + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if r.URL.Path != "/rpc/send" { + t.Errorf("unexpected RPC %s", r.URL.Path) + } + var request sidecar.SendTextRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + } + if mode == "same-room" && (request.ReplyToMessageID == nil || *request.ReplyToMessageID != 9) { + t.Errorf("reply lost: %+v", request) + } + // Stop after recording the intended mutation boundary; no Matrix send/ghost + // fixture is needed to prove correct relation resolution and RPC admission. + writeConnectorSidecarResult(t, w, "message", sidecar.MessageMutation{State: sidecar.TransactionFailed}) + })) + defer server.Close() + ic.Sidecar = sidecar.NewClient(server.URL).WithSessionNamespace("fixture") + relation := &event.RelatesTo{InReplyTo: &event.InReplyTo{EventID: "$target"}} + if strings.HasPrefix(mode, "raw-thread") { + relation.Type, relation.EventID = event.RelThread, "$target" + if mode == "raw-thread-no-fallback" { + relation.InReplyTo = nil + } + } + result := portal.Internal().HandleMatrixMessage(t.Context(), ic.UserLogin, nil, &event.Event{ + ID: "$out", RoomID: portal.MXID, Type: event.EventMessage, + Content: event.Content{Parsed: &event.MessageEventContent{MsgType: event.MsgText, Body: "reply", RelatesTo: relation}}, + }) + want := 1 + if mode != "same-room" { + want = 0 + } + if calls != want || result.Error == nil { + t.Fatalf("mode=%s reached %d source mutations, want %d; result=%+v", mode, calls, want, result) + } + }) + } +} + +func TestSeparateChildReplyReceiverProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + receiver := networkid.UserLoginID(fmt.Sprintf("login-%d", hegel.Draw(ht, hegel.Integers[int64](math.MinInt64, math.MaxInt64)))) + same := hegel.Draw(ht, hegel.Booleans()) + portal := testPortal("7") + portal.Receiver = receiver + target := portal.PortalKey + if !same { + target.Receiver += "-other" + } + msg := &bridgev2.MatrixMessage{MatrixEventBase: bridgev2.MatrixEventBase[*event.MessageEventContent]{Portal: portal}, ReplyTo: &database.Message{ID: "7/9", Room: target}} + err := validateInlineReplyTarget(msg, 7) + if (err == nil) != same { + ht.Fatalf("receiver check same=%t: %v", same, err) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func mustReplyTopic(t *testing.T, ic *InlineClient, d sidecar.DialogRecord) string { + t.Helper() + value, err := ic.replyChildTopic(t.Context(), d) + if err != nil { + t.Fatal(err) + } + return value +} diff --git a/pkg/connector/rich_legacy_test.go b/pkg/connector/rich_legacy_test.go new file mode 100644 index 0000000..64b95cb --- /dev/null +++ b/pkg/connector/rich_legacy_test.go @@ -0,0 +1,84 @@ +package connector + +import ( + "fmt" + "path/filepath" + "testing" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/id" +) + +func TestRichLegacyFingerprintReplayAndRealChanges(t *testing.T) { + // Golden hashes of the exact JSON emitted by 537417d, before metadata was + // hashed. They deliberately do not call either current fingerprint helper. + const textHash = "f06b734cbc22a298797630f8c4f1a123d4846bf13e8c1d003dddfb3ac7fcc14d" + const mediaHash = "8395a1073f8264675a6b4e1fc2a66ec30341f460579980645c216e768213e147" + zero, no := int64(0), false + cases := []struct { + name string + change func(*sidecar.MessageRecord) + media, handled, edit bool + }{ + {name: "identical"}, + {name: "empty-slices", change: func(m *sidecar.MessageRecord) { + m.Metadata.Entities = []sidecar.MessageEntityRecord{} + m.Metadata.Attachments = []sidecar.MessageAttachmentRecord{} + m.Metadata.Actions = []sidecar.MessageActionRecord{} + }}, + {name: "text", edit: true, change: func(m *sidecar.MessageRecord) { m.Content.Text = "edited" }}, + {name: "sender", edit: true, change: func(m *sidecar.MessageRecord) { m.SenderID = 43 }}, + {name: "direction", edit: true, change: func(m *sidecar.MessageRecord) { m.IsOutgoing = true }}, + {name: "reply", edit: true, change: func(m *sidecar.MessageRecord) { m.ReplyToMessageID = &zero }}, + {name: "entities", edit: true, change: func(m *sidecar.MessageRecord) { + m.Metadata.Entities = []sidecar.MessageEntityRecord{{Kind: "TYPE_BOLD", Offset: 0, Length: 5}} + }}, + {name: "revision-zero", edit: true, change: func(m *sidecar.MessageRecord) { m.Metadata.Revision = &zero }}, + {name: "edit-time-zero", edit: true, change: func(m *sidecar.MessageRecord) { m.Metadata.EditTimestamp = &zero }}, + {name: "mentioned-false", edit: true, change: func(m *sidecar.MessageRecord) { m.Metadata.Mentioned = &no }}, + {name: "bot-false", edit: true, change: func(m *sidecar.MessageRecord) { m.Metadata.SenderIsBot = &no }}, + {name: "service", edit: true, change: func(m *sidecar.MessageRecord) { m.Metadata.IsService = true }}, + {name: "attachment", edit: true, change: func(m *sidecar.MessageRecord) { m.Metadata.Attachments = []sidecar.MessageAttachmentRecord{{}} }}, + {name: "action", edit: true, change: func(m *sidecar.MessageRecord) { m.Metadata.Actions = []sidecar.MessageActionRecord{{}} }}, + {name: "agent-session", edit: true, change: func(m *sidecar.MessageRecord) { m.Metadata.AgentSession = &sidecar.AgentSessionMessageMetadata{} }}, + {name: "media-handled", media: true, handled: true}, + {name: "media-retry", media: true, edit: true}, + {name: "media-changed", media: true, handled: true, edit: true, change: func(m *sidecar.MessageRecord) { m.Content.URL += "-new" }}, + } + ic := fixtureHistoryClient(t, filepath.Join(t.TempDir(), "legacy.db")) + key := ic.portalKeyForChat(7) + if err := ic.UserLogin.Bridge.DB.Portal.Insert(t.Context(), &database.Portal{PortalKey: key, Metadata: &PortalMetadata{}}); err != nil { + t.Fatal(err) + } + for n, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + data := sidecar.MessageRecord{ChatID: 7, MessageID: int64(n + 1), SenderID: 42, Content: sidecar.MessageContent{Type: "text", Text: "hello"}} + hash := textHash + if tc.media { + data.Content = sidecar.MessageContent{Type: "media", URL: "https://cdn.inline.test/file"} + hash = mediaHash + } + row := &database.Message{ID: makeMessageID(7, data.MessageID), MXID: id.EventID(fmt.Sprintf("$legacy-%d", n)), Room: key, Metadata: &MessageMetadata{InlineFingerprint: hash, MediaHandled: tc.handled}} + db := ic.UserLogin.Bridge.DB.Message + if err := db.Insert(t.Context(), row); err != nil { + t.Fatal(err) + } + row, err := db.GetFirstPartByID(t.Context(), key.Receiver, row.ID) + if err != nil { + t.Fatal(err) + } + if tc.change != nil { + tc.change(&data) + } + got, err := upsertExistingInlineMessage(t.Context(), testPortal("7"), nil, []*database.Message{row}, data) + want := 0 + if tc.edit { + want = 1 + } + if err != nil || len(got.SubEvents) != want || got.ContinueMessageHandling { + t.Fatalf("legacy replay result %+v %v; want %d edits", got, err, want) + } + }) + } +} diff --git a/pkg/connector/rich_message.go b/pkg/connector/rich_message.go new file mode 100644 index 0000000..39960b5 --- /dev/null +++ b/pkg/connector/rich_message.go @@ -0,0 +1,181 @@ +package connector + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "reflect" + "strconv" + + "github.com/inline-chat/matrix-inline/pkg/msgconv" + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/simplevent" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +// Partial caption-edit support requires the exact saved media envelope. Clients +// that normalize or omit attachment metadata may receive an explicit unsupported +// error; silently discarding those changes or weakening crypto identity is unsafe. +func matrixCaptionEditAllowed(target *database.Message, content *event.MessageEventContent) bool { + if target == nil || content == nil { + return false + } + meta, ok := target.Metadata.(*MessageMetadata) + if !ok || meta == nil || meta.InlineMedia == nil { + return false + } + previous := meta.InlineMedia + return previous.MsgType == content.MsgType && previous.URL == content.URL && reflect.DeepEqual(previous.File, content.File) && previous.FileName == content.FileName && reflect.DeepEqual(previous.Info, content.Info) +} + +func (ic *InlineClient) incomingMention(userID int64) (id.UserID, bool) { + if userID <= 0 || ic.Sidecar == nil || ic.Sidecar.SessionNamespace == "" || ic.UserLogin == nil || ic.UserLogin.UserLogin == nil || ic.UserLogin.ID == "" { + return "", false + } + if strconv.FormatInt(userID, 10) == ic.AccountID && ic.UserLogin.User != nil && ic.UserLogin.User.User != nil { + return ic.UserLogin.User.MXID, ic.UserLogin.User.MXID != "" + } + if user, ok := ic.cachedUser(userID); !ok || user.UserID != userID || ic.UserLogin.Bridge == nil || ic.UserLogin.Bridge.Matrix == nil { + return "", false + } + intent := ic.UserLogin.Bridge.Matrix.GhostIntent(makeUserID(strconv.FormatInt(userID, 10))) + if intent == nil { + return "", false + } + return intent.GetMXID(), intent.GetMXID() != "" +} + +func (ic *InlineClient) outgoingMentionResolver(content *event.MessageEventContent) msgconv.MatrixToInline { + return func(user id.UserID) (int64, bool) { + if content != nil && content.Mentions != nil { + listed := false + for _, mentioned := range content.Mentions.UserIDs { + if mentioned == user { + listed = true + break + } + } + if !listed { + return 0, false + } + } + if ic.Sidecar == nil || ic.Sidecar.SessionNamespace == "" || ic.UserLogin == nil || ic.UserLogin.UserLogin == nil || ic.UserLogin.ID == "" { + return 0, false + } + if ic.UserLogin.User != nil && ic.UserLogin.User.User != nil && user == ic.UserLogin.User.MXID { + self, err := parseInlineUserID(ic.AccountID) + return self, err == nil + } + if ic.UserLogin.Bridge == nil || ic.UserLogin.Bridge.Matrix == nil { + return 0, false + } + remote, ok := ic.UserLogin.Bridge.Matrix.ParseGhostMXID(user) + if !ok { + return 0, false + } + value, err := parseInlineUserID(string(remote)) + if err != nil { + return 0, false + } + verified, ok := ic.incomingMention(value) + return value, ok && verified == user + } +} + +func (ic *InlineClient) convertInlineMessage(ctx context.Context, portal *bridgev2.Portal, intent bridgev2.MatrixAPI, data sidecar.MessageRecord) (*bridgev2.ConvertedMessage, error) { + return convertInlineMessageWithResolver(ctx, portal, intent, data, ic.incomingMention) +} + +func (ic *InlineClient) convertInlineMessageEdit(ctx context.Context, portal *bridgev2.Portal, intent bridgev2.MatrixAPI, existing []*database.Message, data sidecar.MessageRecord) (*bridgev2.ConvertedEdit, error) { + return convertInlineMessageEditWithResolver(ctx, portal, intent, existing, data, ic.incomingMention) +} + +func (ic *InlineClient) upsertExistingInlineMessage(ctx context.Context, portal *bridgev2.Portal, intent bridgev2.MatrixAPI, existing []*database.Message, data sidecar.MessageRecord) (bridgev2.UpsertResult, error) { + result, err := upsertExistingInlineMessage(ctx, portal, intent, existing, data) + for _, sub := range result.SubEvents { + if edit, ok := sub.(*simplevent.Message[sidecar.MessageRecord]); ok { + edit.ConvertEditFunc = ic.convertInlineMessageEdit + } + } + return result, err +} + +// Native revisions take precedence over edit dates. A revisionless observation +// cannot overwrite a revision-bearing one. With neither revision nor edit date, +// fingerprints suppress identical replays but cannot establish a total order. +func staleInlineRevision(existing *database.Message, data sidecar.MessageRecord) bool { + if existing == nil { + return false + } + meta, ok := existing.Metadata.(*MessageMetadata) + if !ok || meta == nil { + return false + } + if meta.InlineRevision != nil { + return data.Metadata.Revision == nil || *data.Metadata.Revision <= *meta.InlineRevision + } + if data.Metadata.Revision != nil { + return false + } + if meta.InlineEditTime != nil { + return data.Metadata.EditTimestamp == nil || *data.Metadata.EditTimestamp <= *meta.InlineEditTime + } + return false +} + +func applyInlineRichContent(converted *bridgev2.ConvertedMessage, data sidecar.MessageRecord, resolve msgconv.InlineToMatrix) { + if converted == nil || len(converted.Parts) == 0 || converted.Parts[0] == nil || converted.Parts[0].Content == nil { + return + } + if data.Content.Type != "" && data.Content.Type != "text" && data.Content.Type != "media" { + return + } + text := data.Content.Text + if data.Content.Type == "media" { + text = optionalStringValue(data.Content.Caption) + if text == "" || !converted.Parts[0].Content.MsgType.IsMedia() { + return + } + } + if text == "" { + text = " " + } + rich := msgconv.ToMatrix(text, data.Metadata.Entities, resolve) + c := converted.Parts[0].Content + c.Body, c.Format, c.FormattedBody, c.Mentions = rich.Body, rich.Format, rich.FormattedBody, rich.Mentions +} + +// Signed CDN URLs may rotate while the stable native file identity stays the +// same. All other media attributes remain part of the identity; no caption +// change can trigger another download/upload, including after a restart. +func inlineMediaIdentity(content sidecar.MessageContent) string { + content.Caption, content.Text = nil, "" + if content.FileID != "" { + content.URL = "" + } + raw, _ := json.Marshal(content) + return fmt.Sprintf("%x", sha256.Sum256(raw)) +} + +func convertedInlineEditContent(ctx context.Context, portal *bridgev2.Portal, intent bridgev2.MatrixAPI, existing *database.Message, data sidecar.MessageRecord) (*bridgev2.ConvertedMessage, error) { + if data.Content.Type == "media" && existing != nil { + if meta, ok := existing.Metadata.(*MessageMetadata); ok && meta != nil && meta.InlineMedia != nil { + // A completed outbound mutation already binds this exact native + // message ID to the original Matrix media. Its first echo records + // the native file identity without transferring the bytes again. + unboundOutbound := meta.InlineMediaID == "" && existing.ID == makeMessageID(data.ChatID, data.MessageID) + if !unboundOutbound && meta.InlineMediaID != inlineMediaIdentity(data.Content) { + return nil, fmt.Errorf("Inline media replacement is unsupported: %w", bridgev2.ErrUnsupportedMessageType) + } + content := *meta.InlineMedia + content.Body = inlineMediaBody(data.Content, content.FileName) + content.Format, content.FormattedBody, content.Mentions = "", "", nil + return &bridgev2.ConvertedMessage{Parts: []*bridgev2.ConvertedMessagePart{{Type: event.EventMessage, Content: &content}}}, nil + } + } + return convertedContentFromInline(ctx, portal, intent, data.Content) +} diff --git a/pkg/connector/rich_message_test.go b/pkg/connector/rich_message_test.go new file mode 100644 index 0000000..f67ada4 --- /dev/null +++ b/pkg/connector/rich_message_test.go @@ -0,0 +1,456 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "hegel.dev/go/hegel" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +// Losing the sidecar entities at the bridge boundary must lose this formatting. +func TestRichMessageIncomingEntities(t *testing.T) { + data := sidecar.MessageRecord{ChatID: 7, MessageID: 11, Content: sidecar.MessageContent{Type: "text", Text: "hi 😀"}, Metadata: sidecar.MessageMetadata{Entities: []sidecar.MessageEntityRecord{{Kind: "TYPE_BOLD", Offset: 3, Length: 2}}}} + got, err := convertInlineMessage(t.Context(), nil, nil, data) + if err != nil { + t.Fatal(err) + } + c := got.Parts[0].Content + if c.Body != "hi 😀" || c.FormattedBody != "hi 😀" || c.Format != event.FormatHTML { + t.Fatalf("lost entities at bridge boundary: %+v", c) + } +} + +type richMatrixConnector struct{ bridgev2.MatrixConnector } + +type richGhostIntent struct { + bridgev2.MatrixAPI + user networkid.UserID +} + +func (r richGhostIntent) GetMXID() id.UserID { return id.UserID("@inline_" + string(r.user) + ":test") } +func (richMatrixConnector) GhostIntent(user networkid.UserID) bridgev2.MatrixAPI { + return richGhostIntent{user: user} +} +func (richMatrixConnector) ParseGhostMXID(user id.UserID) (networkid.UserID, bool) { + if user == "@inline_42:test" { + return "42", true + } + return "", false +} +func richClient() *InlineClient { + return &InlineClient{AccountID: "1", Sidecar: &sidecar.Client{SessionNamespace: "a"}, users: map[int64]sidecar.UserRecord{42: {UserID: 42}}, UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ID: "login-a"}, Bridge: &bridgev2.Bridge{Matrix: richMatrixConnector{}}, User: &bridgev2.User{User: &database.User{MXID: "@self:test"}}}} +} +func TestRichMessageLiveMentionsAreAccountScoped(t *testing.T) { + ic := richClient() + user := int64(42) + data := sidecar.MessageRecord{Content: sidecar.MessageContent{Type: "text", Text: "Ada"}, Metadata: sidecar.MessageMetadata{Entities: []sidecar.MessageEntityRecord{{Kind: "TYPE_MENTION", Offset: 0, Length: 3, UserID: &user}}}} + for _, allowed := range []bool{true, false} { + if !allowed { + ic.users = nil + } + got, err := ic.remoteMessageEvent(data).ConvertMessageFunc(t.Context(), nil, nil, data) + if err != nil { + t.Fatal(err) + } + mentions := got.Parts[0].Content.Mentions + if allowed && (mentions == nil || len(mentions.UserIDs) != 1 || mentions.UserIDs[0] != "@inline_42:test") { + t.Fatalf("lost verified mention: %+v", mentions) + } + if !allowed && mentions != nil && len(mentions.UserIDs) > 0 { + t.Fatalf("cross-account mention: %+v", mentions) + } + } +} + +func TestRichMessageOutgoingEntitiesReachSource(t *testing.T) { + calls := 0 + s := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + Text string `json:"text"` + Entities []sidecar.MessageEntityRecord `json:"entities"` + ParseMarkdown bool `json:"parse_markdown"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + } + calls++ + if body.Text != "😀" || body.ParseMarkdown || len(body.Entities) != 1 || body.Entities[0].Kind != "TYPE_BOLD" || body.Entities[0].Offset != 0 || body.Entities[0].Length != 2 { + t.Errorf("lost native UTF16 entities: %+v", body) + } + switch r.URL.Path { + case "/rpc/send": + mid := int64(11) + writeConnectorSidecarResult(t, w, "message", sidecar.MessageMutation{State: sidecar.TransactionCompleted, MessageID: &mid}) + case "/rpc/edit": + writeConnectorSidecarResult(t, w, "empty", map[string]bool{"ok": true}) + default: + t.Errorf("unexpected route %s", r.URL.Path) + } + })) + defer s.Close() + ic := richClient() + ic.Sidecar = sidecar.NewClient(s.URL) + c := &event.MessageEventContent{MsgType: event.MsgText, Body: "😀", Format: event.FormatHTML, FormattedBody: "😀"} + if _, err := ic.HandleMatrixMessage(t.Context(), &bridgev2.MatrixMessage{MatrixEventBase: bridgev2.MatrixEventBase[*event.MessageEventContent]{Portal: testPortal("7"), Content: c}}); err != nil { + t.Fatal(err) + } + if err := ic.HandleMatrixEdit(t.Context(), &bridgev2.MatrixEdit{MatrixEventBase: bridgev2.MatrixEventBase[*event.MessageEventContent]{Portal: testPortal("7"), Content: c}, EditTarget: &database.Message{ID: "7/11", Room: testPortal("7").PortalKey}}); err != nil { + t.Fatal(err) + } + if calls != 2 { + t.Fatalf("send/edit calls %d", calls) + } +} + +func TestRichMessageReplyFallbackPreservesReplyTarget(t *testing.T) { + calls := 0 + s := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var request sidecar.SendTextRequest + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Error(err) + return + } + calls++ + if r.URL.Path != "/rpc/send" || request.Text != "reply 😀" || request.ReplyToMessageID == nil || *request.ReplyToMessageID != 10 { + t.Errorf("reply context or body lost: path=%s request=%+v", r.URL.Path, request) + } + if len(request.Entities) != 1 || request.Entities[0].Kind != "TYPE_BOLD" || request.Entities[0].Offset != 6 || request.Entities[0].Length != 2 { + t.Errorf("reply formatting lost: %+v", request.Entities) + } + mid := int64(11) + writeConnectorSidecarResult(t, w, "message", sidecar.MessageMutation{State: sidecar.TransactionCompleted, MessageID: &mid}) + })) + defer s.Close() + ic := richClient() + ic.Sidecar = sidecar.NewClient(s.URL) + content := &event.MessageEventContent{MsgType: event.MsgText, Body: "> quoted\n\nreply 😀", Format: event.FormatHTML, FormattedBody: "
quoted
reply 😀"} + portal := testPortal("7") + msg := &bridgev2.MatrixMessage{MatrixEventBase: bridgev2.MatrixEventBase[*event.MessageEventContent]{Portal: portal, Content: content}, ReplyTo: &database.Message{ID: "7/10", Room: portal.PortalKey}} + if _, err := ic.HandleMatrixMessage(t.Context(), msg); err != nil { + t.Fatal(err) + } + if calls != 1 { + t.Fatalf("send calls %d", calls) + } +} + +func TestRichMessageOutgoingMediaReplacementRejected(t *testing.T) { + calls := 0 + s := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + writeConnectorSidecarResult(t, w, "empty", struct{}{}) + })) + defer s.Close() + ic := richClient() + ic.Sidecar = sidecar.NewClient(s.URL) + old := &event.MessageEventContent{MsgType: event.MsgImage, URL: "mxc://test/old", FileName: "image.jpg"} + for _, target := range []*database.Message{{ID: "7/11", Metadata: &MessageMetadata{InlineMedia: old}}, {ID: "7/11"}} { + target.Room = testPortal("7").PortalKey + content := &event.MessageEventContent{MsgType: event.MsgImage, Body: "replacement", URL: "mxc://test/new", FileName: "image.jpg"} + err := ic.HandleMatrixEdit(t.Context(), &bridgev2.MatrixEdit{MatrixEventBase: bridgev2.MatrixEventBase[*event.MessageEventContent]{Portal: testPortal("7"), Content: content}, EditTarget: target}) + if err == nil || calls != 0 { + t.Fatalf("replacement or unproven legacy edit silently succeeded: %v calls%d", err, calls) + } + } +} + +func TestRichMessageOutgoingMediaAckKeepsCaptionAndEchoIdentity(t *testing.T) { + ic := richClient() + content := &event.MessageEventContent{MsgType: event.MsgImage, Body: "before", FileName: "image.jpg", URL: "mxc://test/original"} + mid := int64(11) + response, err := ic.matrixSendResponse(&bridgev2.MatrixMessage{MatrixEventBase: bridgev2.MatrixEventBase[*event.MessageEventContent]{Content: content}}, 7, &sidecar.MessageMutation{State: sidecar.TransactionCompleted, MessageID: &mid}) + if err != nil { + t.Fatal(err) + } + if !matrixCaptionEditAllowed(response.DB, content) { + t.Fatal("completed media acknowledgement discarded caption identity") + } + // The first source echo binds the source media ID to that exact completed + // message; it must reuse the already uploaded Matrix object. + data := sidecar.MessageRecord{ChatID: 7, MessageID: 11, Content: sidecar.MessageContent{Type: "media", Kind: "photo", FileID: "native-file", URL: "http://unused.test/media", Caption: stringPtr("after")}} + intent := &richMediaIntent{} + edit, err := convertInlineMessageEdit(t.Context(), testPortal("7"), intent, []*database.Message{response.DB}, data) + if err != nil || intent.uploads != 0 || len(edit.ModifiedParts) != 1 || edit.ModifiedParts[0].Content.URL != "mxc://test/original" || edit.ModifiedParts[0].Content.Body != "after" { + t.Fatalf("source echo retransferred/lost original media: %+v %v uploads%d", edit, err, intent.uploads) + } +} + +func TestRichMessageOutgoingCaptionEditUsesEditRPC(t *testing.T) { + calls := 0 + s := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/rpc/edit" { + t.Errorf("caption used transfer route %s", r.URL.Path) + } + var body sidecar.EditMessageRequest + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + t.Error(err) + } + if body.Text != "new caption" { + t.Errorf("caption became %q", body.Text) + } + calls++ + writeConnectorSidecarResult(t, w, "empty", map[string]bool{"ok": true}) + })) + defer s.Close() + ic := richClient() + ic.Sidecar = sidecar.NewClient(s.URL) + c := &event.MessageEventContent{MsgType: event.MsgImage, Body: "new caption", FileName: "image.jpg", URL: "mxc://test/existing"} + if err := ic.HandleMatrixEdit(t.Context(), &bridgev2.MatrixEdit{MatrixEventBase: bridgev2.MatrixEventBase[*event.MessageEventContent]{Portal: testPortal("7"), Content: c}, EditTarget: &database.Message{ID: "7/11", Room: testPortal("7").PortalKey, Metadata: &MessageMetadata{InlineMedia: &event.MessageEventContent{MsgType: event.MsgImage, URL: "mxc://test/existing", FileName: "image.jpg"}}}}); err != nil { + t.Fatal(err) + } + if calls != 1 { + t.Fatalf("caption edit calls %d", calls) + } +} + +func TestRichMessageRevisionlessReplayRemainsFingerprintBased(t *testing.T) { + data := sidecar.MessageRecord{Content: sidecar.MessageContent{Type: "text", Text: "before"}} + meta := &MessageMetadata{InlineFingerprint: inlineMessageFingerprint(data)} + data.Content.Text = "after" + got, err := upsertExistingInlineMessage(t.Context(), testPortal("7"), nil, []*database.Message{{Metadata: meta}}, data) + if err != nil || len(got.SubEvents) != 1 { + t.Fatalf("revisionless change lost: %+v %v", got, err) + } +} + +func TestRichMessageFingerprintIncludesEntitySemantics(t *testing.T) { + a := sidecar.MessageRecord{Content: sidecar.MessageContent{Type: "text", Text: "hello"}} + b := a + b.Metadata.Entities = []sidecar.MessageEntityRecord{{Kind: "TYPE_BOLD", Offset: 0, Length: 5}} + if inlineMessageFingerprint(a) == inlineMessageFingerprint(b) { + t.Fatal("entity-only edit collapsed into replay") + } +} + +// A lower or equal revision can never emit an edit, even with a later timestamp. +func TestRichMessageRevisionReplayProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + accepted := hegel.Draw(ht, hegel.Integers[int64](math.MinInt64, math.MaxInt64)) + incoming := hegel.Draw(ht, hegel.Integers[int64](math.MinInt64, accepted)) + meta := &MessageMetadata{} + raw, _ := json.Marshal(map[string]any{"inline_revision": accepted}) + if err := json.Unmarshal(raw, meta); err != nil { + ht.Fatal(err) + } + data := sidecar.MessageRecord{ChatID: 7, MessageID: 11, Timestamp: 999, Content: sidecar.MessageContent{Type: "text", Text: "stale"}, Metadata: sidecar.MessageMetadata{Revision: &incoming}} + got, err := upsertExistingInlineMessage(t.Context(), testPortal("7"), nil, []*database.Message{{ID: "7/11", Metadata: meta}}, data) + if err != nil || len(got.SubEvents) != 0 || got.ContinueMessageHandling { + ht.Fatalf("revision %d replaced accepted %d: %+v %v", incoming, accepted, got, err) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +type richMediaIntent struct { + bridgev2.MatrixAPI + uploads int +} + +func (m *richMediaIntent) UploadMedia(context.Context, id.RoomID, []byte, string, string) (id.ContentURIString, *event.EncryptedFileInfo, error) { + m.uploads++ + return "mxc://test/new-upload", nil, nil +} + +func TestRichMessageCaptionEditReusesPersistedMatrixMedia(t *testing.T) { + downloads := 0 + s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { downloads++; w.Write([]byte("image bytes")) })) + defer s.Close() + data := sidecar.MessageRecord{ChatID: 7, MessageID: 11, Content: sidecar.MessageContent{Type: "media", Kind: "photo", FileID: "file-1", URL: s.URL, Caption: stringPtr("before")}} + intent := &richMediaIntent{} + original, err := convertInlineMessage(t.Context(), testPortal("7"), intent, data) + if err != nil { + t.Fatal(err) + } + if intent.uploads != 1 || downloads != 1 { + t.Fatal("initial media did not transfer") + } + // Persist/reopen the JSON metadata; in-memory caching alone is insufficient. + raw, err := json.Marshal(original.Parts[0].DBMetadata) + if err != nil { + t.Fatal(err) + } + meta := &MessageMetadata{} + if err := json.Unmarshal(raw, meta); err != nil { + t.Fatal(err) + } + data.Content.Caption = stringPtr("after") + data.Metadata.Entities = []sidecar.MessageEntityRecord{{Kind: "TYPE_BOLD", Offset: 0, Length: 5}} + edit, err := convertInlineMessageEdit(t.Context(), testPortal("7"), intent, []*database.Message{{ID: "7/11", Metadata: meta}}, data) + if err != nil { + t.Fatal(err) + } + if intent.uploads != 1 || downloads != 1 { + t.Fatalf("caption retransferred media: uploads=%d downloads=%d", intent.uploads, downloads) + } + c := edit.ModifiedParts[0].Content + if c.URL != "mxc://test/new-upload" || c.Body != "after" || c.FormattedBody != "after" { + t.Fatalf("caption lost media or entities: %+v", c) + } +} + +func TestRichMessageFrameworkCaptionEditTargetIsolation(t *testing.T) { + for _, mode := range []string{"cross-receiver", "cross-chat", "same-room"} { + t.Run(mode, func(t *testing.T) { + ic := fixtureHistoryClient(t, filepath.Join(t.TempDir(), "caption.db")) + ic.AccountID = "1" + ic.UserLogin.Client = ic + br := ic.UserLogin.Bridge + portal, err := br.GetPortalByKey(t.Context(), ic.portalKeyForChat(7)) + if err != nil { + t.Fatal(err) + } + portal.MXID = "!current:test" + if err = portal.Save(t.Context()); err != nil { + t.Fatal(err) + } + target := portal.PortalKey + targetID := networkid.MessageID("7/9") + if mode == "cross-receiver" { + target.Receiver = "other-login" + } + if mode == "cross-chat" { + target.ID = "8" + targetID = "8/9" + } + if mode != "same-room" { + if err = br.DB.Portal.Insert(t.Context(), &database.Portal{PortalKey: target, MXID: "!other:test", Metadata: &PortalMetadata{}}); err != nil { + t.Fatal(err) + } + } + old := &event.MessageEventContent{MsgType: event.MsgImage, URL: "mxc://test/media", FileName: "image.jpg", Info: &event.FileInfo{MimeType: "image/jpeg"}} + if err = br.DB.Message.Insert(t.Context(), &database.Message{ID: targetID, Room: target, MXID: "$target", Metadata: &MessageMetadata{InlineMedia: old}}); err != nil { + t.Fatal(err) + } + calls := 0 + server := newSidecarTestServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + calls++ + if r.URL.Path != "/rpc/edit" { + t.Errorf("unexpected route %s", r.URL.Path) + } + var req sidecar.EditMessageRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Error(err) + } + if mode == "same-room" && (req.ChatID != 7 || req.MessageID != 9 || req.Text != "caption") { + t.Errorf("caption changed %+v", req) + } + json.NewEncoder(w).Encode(map[string]any{"protocol_version": sidecar.ProtocolVersion, "outcome": map[string]any{"status": "error", "data": sidecar.Error{Category: "permission_denied", Message: "fixture stops after mutation admission"}}}) + })) + defer server.Close() + ic.Sidecar = sidecar.NewClient(server.URL).WithSessionNamespace("fixture") + wire, err := json.Marshal(old) + if err != nil { + t.Fatal(err) + } + var fresh event.MessageEventContent + if err = json.Unmarshal(wire, &fresh); err != nil { + t.Fatal(err) + } + fresh.Body = "caption" + outer := fresh + outer.NewContent = &fresh + outer.RelatesTo = &event.RelatesTo{Type: event.RelReplace, EventID: "$target"} + result := portal.Internal().HandleMatrixMessage(t.Context(), ic.UserLogin, nil, &event.Event{ID: "$edit", RoomID: portal.MXID, Type: event.EventMessage, Content: event.Content{Parsed: &outer}}) + want := 0 + if mode == "same-room" { + want = 1 + } + if calls != want || result.Error == nil { + t.Fatalf("mode%s admitted%d want%d: %+v", mode, calls, want, result) + } + }) + } +} + +func TestRichMessageRevisionDirectionsAndPersistedAcceptance(t *testing.T) { + ic := fixtureHistoryClient(t, filepath.Join(t.TempDir(), "revisions.db")) + key := ic.portalKeyForChat(7) + if err := ic.UserLogin.Bridge.DB.Portal.Insert(t.Context(), &database.Portal{PortalKey: key, Metadata: &PortalMetadata{}}); err != nil { + t.Fatal(err) + } + seq := int64(0) + hegel.Test(t, func(ht *hegel.T) { + accepted := hegel.Draw(ht, hegel.Integers[int64](math.MinInt64, math.MaxInt64)) + incoming := hegel.Draw(ht, hegel.Integers[int64](math.MinInt64, math.MaxInt64)) + oldRevision := hegel.Draw(ht, hegel.Booleans()) + newRevision := hegel.Draw(ht, hegel.Booleans()) + oldDate := hegel.Draw(ht, hegel.Booleans()) + newDate := hegel.Draw(ht, hegel.Booleans()) + meta := &MessageMetadata{InlineFingerprint: "before"} + if oldRevision { + meta.InlineRevision = &accepted + } + if oldDate { + meta.InlineEditTime = &accepted + } + seq++ + data := sidecar.MessageRecord{ChatID: 7, MessageID: seq, Content: sidecar.MessageContent{Type: "text", Text: "after"}, Metadata: sidecar.MessageMetadata{Entities: []sidecar.MessageEntityRecord{{Kind: "TYPE_BOLD", Offset: 0, Length: 5}}}} + if newRevision { + data.Metadata.Revision = &incoming + } + if newDate { + data.Metadata.EditTimestamp = &incoming + } + row := &database.Message{ID: makeMessageID(7, seq), MXID: id.EventID(fmt.Sprintf("$revision-%d", seq)), Room: key, Metadata: meta} + db := ic.UserLogin.Bridge.DB.Message + if err := db.Insert(t.Context(), row); err != nil { + ht.Fatal(err) + } + row, err := db.GetFirstPartByID(t.Context(), key.Receiver, row.ID) + if err != nil { + ht.Fatal(err) + } + stale := false + if oldRevision { + stale = !newRevision || incoming <= accepted + } else if !newRevision && oldDate { + stale = !newDate || incoming <= accepted + } + edit, err := convertInlineMessageEdit(t.Context(), testPortal("7"), nil, []*database.Message{row}, data) + if err != nil { + ht.Fatal(err) + } + if stale { + if len(edit.ModifiedParts) != 0 { + ht.Fatal("stale revision emitted content") + } + return + } + if len(edit.ModifiedParts) != 1 || edit.ModifiedParts[0].Content.Body != "after" || edit.ModifiedParts[0].Content.FormattedBody != "after" { + ht.Fatalf("new revision dropped: %+v", edit) + } + // A failed delivery does not save converted metadata. Reload the original + // row and require the same edit on retry before simulating successful save. + retry, err := db.GetFirstPartByID(t.Context(), key.Receiver, row.ID) + if err != nil { + ht.Fatal(err) + } + repeat, err := convertInlineMessageEdit(t.Context(), testPortal("7"), nil, []*database.Message{retry}, data) + if err != nil || len(repeat.ModifiedParts) != 1 { + ht.Fatalf("undelivered conversion consumed revision: %+v %v", repeat, err) + } + row.Metadata = edit.ModifiedParts[0].Part.Metadata + if err = db.Update(t.Context(), row); err != nil { + ht.Fatal(err) + } + saved, err := db.GetFirstPartByID(t.Context(), key.Receiver, row.ID) + if err != nil { + ht.Fatal(err) + } + replay, err := upsertExistingInlineMessage(t.Context(), testPortal("7"), nil, []*database.Message{saved}, data) + if err != nil || len(replay.SubEvents) != 0 || replay.ContinueMessageHandling { + ht.Fatalf("saved revision replayed: %+v %v", replay, err) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} diff --git a/pkg/connector/room_policy.go b/pkg/connector/room_policy.go new file mode 100644 index 0000000..befef9f --- /dev/null +++ b/pkg/connector/room_policy.go @@ -0,0 +1,82 @@ +package connector + +import ( + "context" + "github.com/rs/zerolog" + "reflect" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" +) + +type roomKind string + +const ( + roomUnknown roomKind = "unknown" + roomDirect roomKind = "dm" + roomChannel roomKind = "channel" + roomReplyChild roomKind = "reply_thread" +) + +// classifyDialog consumes the complete descriptor, never absence in an old row. +// Hidden/archive/notification state is independent of structural identity. +func classifyDialog(d sidecar.DialogRecord) roomKind { + s := d.Structure + if s == nil || d.ChatID <= 0 { + return roomUnknown + } + positive := func(v *int64) bool { return v != nil && *v > 0 } + invalid := func(v *int64) bool { return v != nil && *v <= 0 } + if invalid(s.PeerUserID) || invalid(s.SpaceID) || invalid(s.ParentChatID) || invalid(s.ParentMessageID) { + return roomUnknown + } + // Flat compatibility fields cannot contradict the authoritative observation. + for _, pair := range [][2]*int64{{d.PeerUserID, s.PeerUserID}, {d.SpaceID, s.SpaceID}, {d.ParentChatID, s.ParentChatID}, {d.ParentMessageID, s.ParentMessageID}} { + if pair[0] != nil && !int64PtrEqual(pair[0], pair[1]) { + return roomUnknown + } + } + switch s.Kind { + case "direct": + if positive(s.PeerUserID) && s.SpaceID == nil && s.ParentChatID == nil && s.ParentMessageID == nil { + return roomDirect + } + case "channel": + if s.PeerUserID == nil && s.ParentChatID == nil && s.ParentMessageID == nil { + return roomChannel + } + case "reply_child": + if s.PeerUserID == nil && positive(s.ParentChatID) && *s.ParentChatID != d.ChatID { + return roomReplyChild + } + } + return roomUnknown +} + +func isDMDialog(d sidecar.DialogRecord) bool { return classifyDialog(d) == roomDirect } +func sameDialogRecord(a, b sidecar.DialogRecord) bool { return reflect.DeepEqual(a, b) } + +func (ic *InlineClient) chatInfoForDialog(ctx context.Context, d sidecar.DialogRecord, name *string) *bridgev2.ChatInfo { + info := &bridgev2.ChatInfo{Name: name, CanBackfill: true, Members: ic.chatMemberListForKnownSelf(ctx)} + switch classifyDialog(d) { + case roomDirect: + rt := database.RoomTypeDM + info.Type = &rt + info.Name = nil + info.Members = ic.chatMemberListForDM(ctx, *d.Structure.PeerUserID) + case roomChannel, roomReplyChild: + rt := database.RoomTypeDefault + info.Type = &rt + } + // Parent correction uses the connector journal in the serialized portal lane. + // The framework's ParentID path retains cached parents on detach in v0.28. + info.ExtraUpdates = func(ctx context.Context, p *bridgev2.Portal) bool { + if err := ic.reconcileSourceTopic(ctx, p, d); err != nil { + zerolog.Ctx(ctx).Warn().Err(err).Msg("Source topic projection remains pending") + } + ic.projectDialogInLane(ctx, p, d) + return false + } + return info +} diff --git a/pkg/connector/room_policy_test.go b/pkg/connector/room_policy_test.go new file mode 100644 index 0000000..8dcf6a3 --- /dev/null +++ b/pkg/connector/room_policy_test.go @@ -0,0 +1,145 @@ +package connector + +import ( + "encoding/json" + "math" + "testing" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "hegel.dev/go/hegel" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" +) + +func policyDialog(t testing.TB, raw string) sidecar.DialogRecord { + t.Helper() + var d sidecar.DialogRecord + if err := json.Unmarshal([]byte(raw), &d); err != nil { + t.Fatal(err) + } + return d +} + +func TestFoundationClassification(t *testing.T) { + for _, tc := range []struct { + raw string + kind database.RoomType + known bool + }{ + {`{"chat_id":7}`, database.RoomTypeDefault, false}, + {`{"chat_id":7,"structure":{"kind":"channel","peer_user_id":null,"space_id":null,"parent_chat_id":null,"parent_message_id":null}}`, database.RoomTypeDefault, true}, + {`{"chat_id":7,"structure":{"kind":"direct","peer_user_id":42,"space_id":null,"parent_chat_id":null,"parent_message_id":null}}`, database.RoomTypeDM, true}, + {`{"chat_id":7,"structure":{"kind":"reply_child","parent_chat_id":3,"parent_message_id":9,"peer_user_id":null,"space_id":null}}`, database.RoomTypeDefault, true}, + {`{"chat_id":7,"peer_user_id":42,"structure":{"kind":"direct","peer_user_id":42,"parent_chat_id":3,"space_id":null,"parent_message_id":null}}`, database.RoomTypeDefault, false}, + } { + d := policyDialog(t, tc.raw) + info := (&InlineClient{AccountID: "1", UserLogin: &bridgev2.UserLogin{UserLogin: &database.UserLogin{ID: "1"}}}).chatInfoForDialog(t.Context(), d, nil) + if tc.known && (info.Type == nil || *info.Type != tc.kind) { + t.Errorf("%s: type=%v want %s", tc.raw, info.Type, tc.kind) + } + if !tc.known && info.Type != nil { + t.Errorf("%s: unknown structure must preserve existing type", tc.raw) + } + } +} + +func TestFoundationEqualityIncludesPresentation(t *testing.T) { + base := policyDialog(t, `{"chat_id":7,"space_id":1}`) + for _, raw := range []string{`{"chat_id":7,"space_id":2}`, `{"chat_id":7,"space_id":1,"archived":true}`, `{"chat_id":7,"space_id":1,"structure":{"kind":"channel","peer_user_id":null,"space_id":null,"parent_chat_id":null,"parent_message_id":null}}`, `{"chat_id":7,"space_id":1,"emoji":"x"}`} { + if sameDialogRecord(base, policyDialog(t, raw)) { + t.Errorf("presentation change ignored: %s", raw) + } + } +} + +func TestFoundationConflictingPeerNeverDirect(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + peer := int64(hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64))) + parent := int64(hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64))) + raw, _ := json.Marshal(map[string]any{"chat_id": 7, "peer_user_id": peer, "parent_chat_id": parent, "structure": map[string]any{"kind": "direct", "peer_user_id": peer, "parent_chat_id": parent, "space_id": nil, "parent_message_id": nil}}) + var d sidecar.DialogRecord + if err := json.Unmarshal(raw, &d); err != nil { + ht.Fatal(err) + } + if isDMDialog(d) { + ht.Fatal("contradictory parent and peer classified direct") + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func TestPartialDialogPreservesEstablishedStructure(t *testing.T) { + ic := &InlineClient{} + known := sidecar.DialogRecord{ChatID: 7, SpaceID: foundationInt64(3), Structure: &sidecar.DialogStructure{Kind: "channel", SpaceID: foundationInt64(3)}} + ic.rememberDialog(known) + ic.rememberDialog(sidecar.DialogRecord{ChatID: 7, Title: stringPtr("new name")}) + got, _ := ic.cachedDialog(7) + if classifyDialog(got) != roomChannel || got.Structure.SpaceID == nil || *got.Structure.SpaceID != 3 || got.Title == nil || *got.Title != "new name" { + t.Fatalf("partial patch lost established structure: %+v", got) + } + ic.rememberDialog(sidecar.DialogRecord{ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "channel"}}) + got, _ = ic.cachedDialog(7) + if got.Structure.SpaceID != nil || got.SpaceID != nil { + t.Fatal("complete home descriptor did not detach") + } +} + +func foundationInt64(n int64) *int64 { return &n } + +func TestClassificationCoversAuthoritativeDomain(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + chat := int64(hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64))) + peer := int64(hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64))) + parent := chat ^ 1 + if parent == 0 { + parent = 2 + } + hidden, archived := hegel.Draw(ht, hegel.Booleans()), hegel.Draw(ht, hegel.Booleans()) + kind := hegel.Draw(ht, hegel.SampledFrom([]string{"direct", "channel", "reply_child"})) + s := &sidecar.DialogStructure{Kind: kind} + want := roomChannel + switch kind { + case "direct": + s.PeerUserID = &peer + want = roomDirect + case "reply_child": + s.ParentChatID = &parent + s.ParentMessageID = &peer + want = roomReplyChild + } + if kind != "direct" && hegel.Draw(ht, hegel.Booleans()) { + s.SpaceID = &peer + } + d := sidecar.DialogRecord{ChatID: chat, Structure: s, ChatListHidden: &hidden, Archived: &archived} + if classifyDialog(d) != want { + ht.Fatal("valid source descriptor rejected") + } + d.ChatID = 0 + if classifyDialog(d) != roomUnknown { + ht.Fatal("invalid source chat accepted") + } + d.ChatID = chat + d.Structure = nil + if classifyDialog(d) != roomUnknown { + ht.Fatal("old record absence invented structure") + } + d.Structure = s + switch kind { + case "direct": + s.ParentChatID = &parent + case "channel": + s.PeerUserID = &peer + case "reply_child": + s.ParentChatID = nil + } + if classifyDialog(d) != roomUnknown { + ht.Fatal("contradictory or incomplete structural record accepted") + } + }, hegel.WithTestCases(3000), hegel.WithDatabase(t.TempDir())) +} + +func TestParentOnlyChildIsSourceBackedSeparateRoom(t *testing.T) { + d := sidecar.DialogRecord{ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "reply_child", ParentChatID: foundationInt64(6)}} + if classifyDialog(d) != roomReplyChild { + t.Fatal("valid linked child without reply anchor became unknown") + } +} diff --git a/pkg/connector/source_topic.go b/pkg/connector/source_topic.go new file mode 100644 index 0000000..1a64596 --- /dev/null +++ b/pkg/connector/source_topic.go @@ -0,0 +1,175 @@ +package connector + +import ( + "context" + "errors" + "time" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/matrix" + "maunium.net/go/mautrix/event" +) + +type sourceTopicIntent struct { + Before string `json:"before"` + Desired string `json:"desired"` +} +type sourceTopicState struct { + Owned *string `json:"owned,omitempty"` + Pending *sourceTopicIntent `json:"pending,omitempty"` + UserOwned bool `json:"user_owned,omitempty"` +} + +func (ic *InlineClient) topicProjectionEligible(d sidecar.DialogRecord) bool { + return classifyDialog(d) != roomUnknown && ic.shouldProjectDialog(d) +} + +// All writes run in the existing serialized portal lane. Intent is durable +// before a topic write, and observed state is read back before ownership moves. +// Matrix offers no atomic CAS: this retains the controlled-window limitation. +func (ic *InlineClient) reconcileSourceTopic(ctx context.Context, p *bridgev2.Portal, d sidecar.DialogRecord) error { + if !ic.topicProjectionEligible(d) { + return nil + } + meta, ok := p.Metadata.(*PortalMetadata) + if !ok || meta == nil { + return errors.New("source topic metadata unavailable") + } + state := meta.InlineTopic + if state == nil { + state = &sourceTopicState{} + } + if state.UserOwned { + return nil + } + desired, err := ic.replyChildTopic(ctx, d) + if err != nil { + if p.MXID == "" { + p.Topic = "" + p.TopicSet = false + } + return err + } + if desired == "" && state.Owned == nil && state.Pending == nil { + return nil + } + + save := func(next *sourceTopicState) error { + previous := meta.InlineTopic + meta.InlineTopic = next + if err := p.Save(ctx); err != nil { + meta.InlineTopic = previous + return err + } + state = next + return nil + } + // Room creation consumes the cached topic. Keep its intent pending until a + // later existing-room read proves that initial state reached the homeserver. + if p.MXID == "" { + if state.Pending == nil || state.Pending.Desired != desired { + before := p.Topic + if state.Pending != nil { + before = state.Pending.Before + } + if err := save(&sourceTopicState{Owned: state.Owned, Pending: &sourceTopicIntent{Before: before, Desired: desired}}); err != nil { + // ExtraUpdates cannot abort pinned room creation. Expose no + // obsolete or non-durable context to that creation request. + // The old durable intent survives; a nonempty-before fallback + // may conservatively retire ownership on later reconciliation. + p.Topic = "" + p.TopicSet = false + return err + } + } + p.Topic = state.Pending.Desired + p.TopicSet = false + return p.Save(ctx) + } + + raw, ok := p.Bridge.Bot.(*matrix.ASIntent) + if !ok || raw == nil || raw.Matrix == nil || raw.Matrix.Client == nil { + return errors.New("source topic requires verified Matrix state transport") + } + read := func() (string, error) { + var content event.TopicEventContent + err := raw.Matrix.Client.StateEvent(ctx, p.MXID, event.StateTopic, "", &content) + if errors.Is(err, mautrix.MNotFound) { + return "", nil + } + return content.Topic, err + } + current, err := read() + if err != nil { + return err + } + retire := func() error { + p.Topic = current + p.TopicSet = true + return save(&sourceTopicState{UserOwned: true}) + } + // Finish one prior operation, then allow one new source desire. This is + // bounded even if a source descriptor changes while an old intent is pending. + for step := 0; step < 2; step++ { + if state.Pending != nil && state.Pending.Desired != desired && current == state.Pending.Before { + // Nothing from the prior intent reached Matrix; supersede it + // durably rather than publish obsolete context just to retract it. + if err = save(&sourceTopicState{Owned: state.Owned, Pending: &sourceTopicIntent{Before: current, Desired: desired}}); err != nil { + return err + } + } + if state.Pending == nil { + if state.Owned == nil { + if desired == "" { + return nil + } + // Equality does not prove ownership of a pre-existing topic. + if current != "" { + return retire() + } + } else if current != *state.Owned { + return retire() + } + if current == desired { + return nil + } + if err = save(&sourceTopicState{Owned: state.Owned, Pending: &sourceTopicIntent{Before: current, Desired: desired}}); err != nil { + return err + } + } + pending := state.Pending + if current != pending.Desired { + if current != pending.Before { + return retire() + } + // No ExtraUpdates on this inner update: never recurse into this lane. + p.UpdateInfo(ctx, &bridgev2.ChatInfo{Topic: &pending.Desired}, nil, nil, time.Time{}) + current, err = read() + if err != nil { + return err + } + if current != pending.Desired { + if current != pending.Before { + return retire() + } + return errors.New("source topic write was not observed") + } + } + next := &sourceTopicState{} + if pending.Desired != "" { + value := pending.Desired + next.Owned = &value + } + p.Topic = current + p.TopicSet = true + if err = save(next); err != nil { + return err + } + if current == desired { + return nil + } + } + return nil +} diff --git a/pkg/connector/source_topic_test.go b/pkg/connector/source_topic_test.go new file mode 100644 index 0000000..02187b1 --- /dev/null +++ b/pkg/connector/source_topic_test.go @@ -0,0 +1,438 @@ +package connector + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "hegel.dev/go/hegel" + "maunium.net/go/mautrix/appservice" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/matrix" + "maunium.net/go/mautrix/event" +) + +type topicFixture struct { + path string + topic string + writes int + matrixCalls int + failWrite bool + failRead bool + failAfterWrite bool + server *httptest.Server + ic *InlineClient + portal *bridgev2.Portal +} + +func newTopicFixture(t *testing.T) *topicFixture { + t.Helper() + f := &topicFixture{path: filepath.Join(t.TempDir(), "topics.db")} + f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.matrixCalls++ + if strings.Contains(r.URL.Path, "/state/m.room.topic") { + if r.Method == http.MethodGet { + if f.failRead { + http.Error(w, "temporary read failure", 503) + return + } + json.NewEncoder(w).Encode(event.TopicEventContent{Topic: f.topic}) + return + } + if r.Method == http.MethodPut { + if f.failWrite { + http.Error(w, "temporary write failure", 503) + return + } + var c event.TopicEventContent + if err := json.NewDecoder(r.Body).Decode(&c); err != nil { + t.Error(err) + } + f.topic = c.Topic + f.writes++ + if f.failAfterWrite { + f.failRead = true + } + } + } + json.NewEncoder(w).Encode(map[string]string{"event_id": "$state"}) + })) + t.Cleanup(f.server.Close) + f.restart(t) + return f +} +func (f *topicFixture) restart(t *testing.T) { + t.Helper() + f.ic = fixtureHistoryClient(t, f.path) + f.ic.AccountID = "1" + br := f.ic.UserLogin.Bridge + as, err := appservice.CreateFull(appservice.CreateOpts{HomeserverDomain: "test", HomeserverURL: f.server.URL, Registration: &appservice.Registration{SenderLocalpart: "bot", AppToken: "synthetic"}}) + if err != nil { + t.Fatal(err) + } + as.BotIntent().Client.DefaultHTTPRetries = 0 + br.Bot = &matrix.ASIntent{Matrix: as.BotIntent()} + f.portal, err = br.GetPortalByKey(t.Context(), f.ic.portalKeyForChat(7)) + if err != nil { + t.Fatal(err) + } + f.portal.MXID = "!child:test" + if err = as.StateStore.SetMembership(t.Context(), f.portal.MXID, br.Bot.GetMXID(), event.MembershipJoin); err != nil { + t.Fatal(err) + } + if err = f.portal.Save(t.Context()); err != nil { + t.Fatal(err) + } +} +func topicChild() sidecar.DialogRecord { + return sidecar.DialogRecord{ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "reply_child", ParentChatID: dirPtr(int64(3)), ParentMessageID: dirPtr(int64(9))}} +} +func topicChannel() sidecar.DialogRecord { + return sidecar.DialogRecord{ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "channel"}} +} +func (f *topicFixture) apply(t *testing.T, d sidecar.DialogRecord) { + t.Helper() + info := f.ic.chatInfoForDialog(t.Context(), d, nil) + f.portal.UpdateInfo(t.Context(), info, nil, nil, time.Time{}) +} +func (f *topicFixture) pending(t *testing.T) bool { + t.Helper() + row, err := f.ic.UserLogin.Bridge.DB.Portal.GetByKey(t.Context(), f.portal.PortalKey) + if err != nil { + t.Fatal(err) + } + raw, err := json.Marshal(row.Metadata) + if err != nil { + t.Fatal(err) + } + var meta struct { + Topic *struct { + Pending json.RawMessage `json:"pending"` + } `json:"inline_topic"` + } + if err = json.Unmarshal(raw, &meta); err != nil { + t.Fatal(err) + } + return meta.Topic != nil && len(meta.Topic.Pending) > 0 && string(meta.Topic.Pending) != "null" +} +func TestSourceTopicDetachAndUserChoiceAcrossRestart(t *testing.T) { + for _, mode := range []string{"detach", "unknown", "user-choice"} { + t.Run(mode, func(t *testing.T) { + f := newTopicFixture(t) + f.apply(t, topicChild()) + if !strings.Contains(f.topic, "chat 3") { + t.Fatal("child context not written") + } + before := f.topic + f.restart(t) + d := topicChannel() + if mode == "unknown" { + d.Structure = nil + } + if mode == "user-choice" { + f.topic = "user choice" + } + f.apply(t, d) + want := "" + if mode == "unknown" { + want = before + } + if mode == "user-choice" { + want = "user choice" + } + if f.topic != want { + t.Fatalf("%s retained/replaced incorrect topic %q want %q", mode, f.topic, want) + } + }) + } +} +func TestSourceTopicPendingReadbackResumesWithoutDuplicateWrite(t *testing.T) { + f := newTopicFixture(t) + f.failAfterWrite = true + f.apply(t, topicChild()) + if !f.pending(t) { + t.Fatal("write without readback lost durable intent") + } + writes := f.writes + f.failAfterWrite = false + f.failRead = false + f.restart(t) + f.apply(t, topicChild()) + if f.pending(t) || f.writes != writes { + t.Fatalf("readback recovery duplicated write or stayed pending: %d => %d", writes, f.writes) + } + f.apply(t, topicChannel()) + if f.topic != "" { + t.Fatal("recovered ownership did not retract context") + } +} + +func TestSourceTopicPendingRetriesOnPortalEvent(t *testing.T) { + for _, userChoice := range []bool{false, true} { + t.Run(fmt.Sprintf("user-choice=%t", userChoice), func(t *testing.T) { + f := newTopicFixture(t) + d := topicChild() + f.ic.rememberDialog(d) + f.failWrite = true + f.apply(t, d) + if !f.pending(t) { + t.Fatal("failed write lost pending intent") + } + f.failWrite = false + if userChoice { + f.topic = "user choice" + } + // The next existing portal event invokes the real post-handler; + // there is no new dialog observation or manual topic reconciliation. + f.ic.projectionPostHandler(t.Context(), f.portal) + if f.pending(t) { + t.Fatal("portal event did not resolve pending intent") + } + if userChoice { + if f.topic != "user choice" || f.writes != 0 { + t.Fatalf("user topic overwritten: %q, writes %d", f.topic, f.writes) + } + } else if !strings.Contains(f.topic, "chat 3") || f.writes != 1 { + t.Fatalf("pending context not applied once: %q, writes %d", f.topic, f.writes) + } + calls := f.matrixCalls + f.ic.projectionPostHandler(t.Context(), f.portal) + if f.matrixCalls != calls { + t.Fatal("completed intent caused another topic request") + } + }) + } +} + +func TestSourceTopicPendingEventRetriesAreRateLimited(t *testing.T) { + f := newTopicFixture(t) + d := topicChild() + f.ic.rememberDialog(d) + f.failWrite = true + f.apply(t, d) + f.ic.projectionPostHandler(t.Context(), f.portal) + calls := f.matrixCalls + f.ic.projectionPostHandler(t.Context(), f.portal) + if !f.pending(t) || f.matrixCalls != calls { + t.Fatal("repeated portal event retried pending topic without delay") + } +} + +func TestSourceTopicIneligibleEventDoesNotDelayRetry(t *testing.T) { + f := newTopicFixture(t) + d := topicChild() + f.ic.rememberDialog(d) + f.failWrite = true + f.apply(t, d) + if !f.pending(t) { + t.Fatal("failed write lost pending intent") + } + unknown := d + unknown.Structure = &sidecar.DialogStructure{Kind: "unknown"} + f.ic.rememberDialog(unknown) + f.ic.projectionPostHandler(t.Context(), f.portal) + f.ic.rememberDialog(d) + f.failWrite = false + f.ic.projectionPostHandler(t.Context(), f.portal) + if f.pending(t) || f.writes != 1 || !strings.Contains(f.topic, "chat 3") { + t.Fatalf("ineligible event delayed eligible retry: topic %q writes %d", f.topic, f.writes) + } +} +func TestSourceTopicFailedWriteAndConflictStaySafe(t *testing.T) { + for _, conflict := range []bool{false, true} { + t.Run(map[bool]string{false: "retry", true: "conflict"}[conflict], func(t *testing.T) { + f := newTopicFixture(t) + f.failWrite = true + f.apply(t, topicChild()) + if !f.pending(t) || f.writes != 0 { + t.Fatal("failed topic send lost intent or claimed write") + } + f.failWrite = false + f.restart(t) + if conflict { + f.topic = "new user value" + } + f.apply(t, topicChild()) + if conflict { + if f.topic != "new user value" || f.writes != 0 { + t.Fatal("overwrote user during restart") + } + } else if !strings.Contains(f.topic, "chat 3") || f.pending(t) { + t.Fatal("pending write did not resume") + } + }) + } +} +func TestSourceTopicUnownedEqualTextIsNotAdopted(t *testing.T) { + f := newTopicFixture(t) + f.topic = mustReplyTopic(t, f.ic, topicChild()) + before := f.topic + f.apply(t, topicChild()) + f.restart(t) + f.apply(t, topicChannel()) + if f.topic != before || f.writes != 0 { + t.Fatal("equal unowned topic was adopted and cleared") + } +} + +func TestSourceTopicIntentSaveFailurePreventsExternalWrite(t *testing.T) { + f := newTopicFixture(t) + db := f.ic.UserLogin.Bridge.DB + if _, err := db.Exec(t.Context(), `CREATE TRIGGER deny_topic_intent BEFORE UPDATE ON portal WHEN NEW.metadata LIKE '%"pending"%' BEGIN SELECT RAISE(FAIL,'fixture intent failure'); END`); err != nil { + t.Fatal(err) + } + f.apply(t, topicChild()) + if f.writes != 0 || f.pending(t) { + t.Fatal("external topic write occurred before durable intent") + } + if _, err := db.Exec(t.Context(), `DROP TRIGGER deny_topic_intent`); err != nil { + t.Fatal(err) + } + f.restart(t) + f.apply(t, topicChild()) + if f.writes != 1 || f.pending(t) { + t.Fatal("intent-save recovery did not apply once") + } +} + +func TestSourceTopicCompletionSaveFailureRecovers(t *testing.T) { + f := newTopicFixture(t) + db := f.ic.UserLogin.Bridge.DB + if _, err := db.Exec(t.Context(), `CREATE TRIGGER deny_topic_completion BEFORE UPDATE ON portal WHEN NEW.metadata LIKE '%"owned"%' AND NEW.metadata NOT LIKE '%"pending"%' BEGIN SELECT RAISE(FAIL,'fixture completion failure'); END`); err != nil { + t.Fatal(err) + } + f.apply(t, topicChild()) + if f.writes != 1 || !f.pending(t) { + t.Fatal("completion failure lost durable pending intent") + } + if _, err := db.Exec(t.Context(), `DROP TRIGGER deny_topic_completion`); err != nil { + t.Fatal(err) + } + f.restart(t) + f.apply(t, topicChild()) + if f.writes != 1 || f.pending(t) { + t.Fatal("restart repeated an already observed topic write") + } + f.apply(t, topicChannel()) + if f.topic != "" { + t.Fatal("completion recovery lost topic ownership") + } +} + +func TestSourceTopicPendingUserChoiceProperty(t *testing.T) { + f := newTopicFixture(t) + hegel.Test(t, func(ht *hegel.T) { + value := []rune(hegel.Draw(ht, hegel.Text())) + if len(value) > 80 { + value = value[:80] + } + userChoice := hegel.Draw(ht, hegel.Booleans()) + // Each generated case begins with a fresh logical portal state in the same + // real database, avoiding thousands of open SQLite/HTTP fixture instances. + f.portal.Metadata = &PortalMetadata{} + f.portal.Topic = "" + f.portal.TopicSet = true + f.topic = "" + f.writes = 0 + f.failRead = false + f.failAfterWrite = true + if err := f.portal.Save(t.Context()); err != nil { + ht.Fatal(err) + } + f.apply(t, topicChild()) + if !f.pending(t) { + ht.Fatal("unobserved write lost intent") + } + f.failAfterWrite = false + f.failRead = false + want := f.topic + if userChoice { + want = "user-" + string(value) + f.topic = want + } + writes := f.writes + f.restart(t) + f.apply(t, topicChild()) + if f.topic != want || f.writes != writes || f.pending(t) { + ht.Fatalf("pending replay changed user=%t topic%q=>%q writes%d=>%d", userChoice, want, f.topic, writes, f.writes) + } + }, hegel.WithTestCases(100), hegel.WithDatabase(t.TempDir())) +} + +func TestSourceTopicPrecreationPendingTracksLatestDescriptor(t *testing.T) { + for _, before := range []string{"", "old cached topic"} { + for _, detach := range []bool{false, true} { + for _, fail := range []bool{false, true} { + t.Run(fmt.Sprintf("before=%q/detach=%t/fail=%t", before, detach, fail), func(t *testing.T) { + f := newTopicFixture(t) + f.portal.MXID = "" + f.portal.Topic = before + if err := f.portal.Save(t.Context()); err != nil { + t.Fatal(err) + } + f.apply(t, topicChild()) + old := f.portal.Topic + d := topicChild() + d.Structure.ParentChatID = dirPtr(int64(8)) + want := mustReplyTopic(t, f.ic, d) + if detach { + d = topicChannel() + want = "" + } + db := f.ic.UserLogin.Bridge.DB + if fail { + if _, err := db.Exec(t.Context(), `CREATE TRIGGER deny_precreation_topic BEFORE UPDATE ON portal BEGIN SELECT RAISE(FAIL,'fixture save failure'); END`); err != nil { + t.Fatal(err) + } + } + f.apply(t, d) + row, err := db.Portal.GetByKey(t.Context(), f.portal.PortalKey) + if err != nil { + t.Fatal(err) + } + pending := row.Metadata.(*PortalMetadata).InlineTopic.Pending + if pending == nil { + t.Fatal("creation topic lost pending intent") + } + if fail { + if f.portal.Topic != "" || pending.Desired != old { + t.Fatalf("failed save exposed stale/non-durable topic: cache%q intent%+v", f.portal.Topic, pending) + } + if _, err = db.Exec(t.Context(), `DROP TRIGGER deny_precreation_topic`); err != nil { + t.Fatal(err) + } + f.apply(t, d) + row, err = db.Portal.GetByKey(t.Context(), f.portal.PortalKey) + if err != nil { + t.Fatal(err) + } + pending = row.Metadata.(*PortalMetadata).InlineTopic.Pending + } + if f.portal.Topic != want || pending.Desired != want || f.matrixCalls != 0 { + t.Fatalf("creation used obsolete topic: cache%q want%q intent%+v Matrixcalls%d", f.portal.Topic, want, pending, f.matrixCalls) + } + }) + } + } + } +} + +func TestSourceTopicUnappliedIntentSupersededBeforeWireWrite(t *testing.T) { + f := newTopicFixture(t) + f.failWrite = true + f.apply(t, topicChild()) + if !f.pending(t) { + t.Fatal("missing pending source context") + } + f.failWrite = false + f.apply(t, topicChannel()) + if f.writes != 0 || f.topic != "" || f.pending(t) { + t.Fatalf("obsolete pending context was published before detach: writes%d topic%q", f.writes, f.topic) + } +} diff --git a/pkg/connector/workspace_recovery_test.go b/pkg/connector/workspace_recovery_test.go new file mode 100644 index 0000000..cb199fd --- /dev/null +++ b/pkg/connector/workspace_recovery_test.go @@ -0,0 +1,268 @@ +package connector + +import ( + "encoding/json" + "fmt" + "math" + "net/http" + "net/http/httptest" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "hegel.dev/go/hegel" + "maunium.net/go/mautrix/appservice" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/matrix" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +type workspaceWireFixture struct { + path string + wire map[string]json.RawMessage + calls int + ic *InlineClient + portal *bridgev2.Portal + server *httptest.Server +} + +func (f *workspaceWireFixture) set(room, typ, key string, value any) { + raw, _ := json.Marshal(value) + f.wire[room+"|"+typ+"|"+key] = raw +} +func newWorkspaceWireFixture(t *testing.T) *workspaceWireFixture { + t.Helper() + f := &workspaceWireFixture{path: filepath.Join(t.TempDir(), "workspace.db"), wire: map[string]json.RawMessage{}} + f.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.calls++ + rest := strings.TrimPrefix(r.URL.Path, "/_matrix/client/v3/rooms/") + if r.Method == http.MethodGet && strings.HasSuffix(rest, "/state") { + room := strings.TrimSuffix(rest, "/state") + events := []map[string]any{} + for k, raw := range f.wire { + p := strings.SplitN(k, "|", 3) + if p[0] == room { + events = append(events, map[string]any{"type": p[1], "state_key": p[2], "content": raw}) + } + } + json.NewEncoder(w).Encode(events) + return + } + p := strings.SplitN(rest, "/state/", 2) + if len(p) != 2 { + t.Errorf("unexpected Matrix route%s", r.URL.Path) + http.Error(w, "unexpected", 400) + return + } + state := strings.SplitN(p[1], "/", 2) + if len(state) != 2 { + t.Errorf("missing state key%s", r.URL.Path) + http.Error(w, "key", 400) + return + } + key := p[0] + "|" + state[0] + "|" + state[1] + if r.Method == http.MethodPut { + var raw json.RawMessage + if err := json.NewDecoder(r.Body).Decode(&raw); err != nil { + t.Error(err) + } + f.wire[key] = raw + json.NewEncoder(w).Encode(map[string]string{"event_id": "$state"}) + return + } + if raw, ok := f.wire[key]; ok { + w.Write(raw) + } else { + w.Write([]byte(`{}`)) + } + })) + t.Cleanup(f.server.Close) + f.restart(t) + f.set("!child:test", "m.space.parent", "!parent1:test", map[string]any{"via": []string{"test"}, "canonical": true}) + f.set("!parent1:test", "m.space.child", "!child:test", map[string]any{"via": []string{"test"}}) + for _, typ := range []string{"m.bridge", "uk.half-shot.bridge"} { + f.set("!child:test", typ, "fixture", map[string]any{"channel": map[string]any{"id": "7", "fi.mau.receiver": "login"}, "com.beeper.room_type.v2": "", "network": map[string]any{"id": "space:1"}}) + } + f.calls = 0 + return f +} +func (f *workspaceWireFixture) restart(t *testing.T) { + t.Helper() + f.ic = fixtureHistoryClient(t, f.path) + f.ic.AccountID = "1" + br := f.ic.UserLogin.Bridge + f.ic.UserLogin.User = &bridgev2.User{User: &database.User{MXID: "@user:test"}, Bridge: br} + as, err := appservice.CreateFull(appservice.CreateOpts{HomeserverDomain: "test", HomeserverURL: f.server.URL, Registration: &appservice.Registration{SenderLocalpart: "bot", AppToken: "synthetic"}}) + if err != nil { + t.Fatal(err) + } + as.BotIntent().Client.DefaultHTTPRetries = 0 + br.Bot = &matrix.ASIntent{Matrix: as.BotIntent()} + for n := 1; n <= 2; n++ { + key := networkid.PortalKey{ID: networkid.PortalID(fmt.Sprintf("space:%d", n)), Receiver: "login"} + row, err := br.DB.Portal.GetByKey(t.Context(), key) + if err != nil { + t.Fatal(err) + } + if row == nil { + if err = br.DB.Portal.Insert(t.Context(), &database.Portal{PortalKey: key, RoomType: database.RoomTypeSpace, MXID: networkidRoom(n), Metadata: &PortalMetadata{}}); err != nil { + t.Fatal(err) + } + } + if err = as.StateStore.SetMembership(t.Context(), networkidRoom(n), br.Bot.GetMXID(), event.MembershipJoin); err != nil { + t.Fatal(err) + } + } + key := f.ic.portalKeyForChat(7) + row, err := br.DB.Portal.GetByKey(t.Context(), key) + if err != nil { + t.Fatal(err) + } + if row == nil { + if err = br.DB.Portal.Insert(t.Context(), &database.Portal{PortalKey: key, ParentKey: networkid.PortalKey{ID: "space:1", Receiver: "login"}, InSpace: true, MXID: "!child:test", Metadata: &PortalMetadata{}}); err != nil { + t.Fatal(err) + } + } + f.portal, err = br.GetPortalByKey(t.Context(), key) + if err != nil { + t.Fatal(err) + } + if err = as.StateStore.SetMembership(t.Context(), f.portal.MXID, br.Bot.GetMXID(), event.MembershipJoin); err != nil { + t.Fatal(err) + } +} +func networkidRoom(n int) id.RoomID { return id.RoomID(fmt.Sprintf("!parent%d:test", n)) } +func workspaceDialog(space int64) sidecar.DialogRecord { + return sidecar.DialogRecord{ChatID: 7, Structure: &sidecar.DialogStructure{Kind: "channel", SpaceID: &space}} +} +func (f *workspaceWireFixture) pending(t *testing.T, target string) { + t.Helper() + s, k, err := f.ic.projectionJournal(t.Context(), "7") + if err != nil { + t.Fatal(err) + } + p, err := s.load(t.Context(), k) + if err != nil { + t.Fatal(err) + } + p.ParentKnown = true + p.ParentPending = true + p.AppliedParent = "space:1" + p.DesiredParent = target + p.OwnedParents = []id.RoomID{"!parent1:test"} + if err = s.save(t.Context(), k, &p); err != nil { + t.Fatal(err) + } +} +func TestWorkspaceRefreshFailureHoldsOwnedHierarchy(t *testing.T) { + f := newWorkspaceWireFixture(t) + f.pending(t, "space:1") + snapshot := &sidecar.AccountStateSnapshot{Spaces: []sidecar.SpaceRecord{{SpaceID: 1, Name: "one"}}, SpaceAccess: []sidecar.SpaceAccessRecord{{SpaceID: 1, Status: "accessible"}}, SpaceAccessComplete: true} + f.ic.cacheWorkspaceSnapshot(snapshot) + revision := f.ic.workspaceRevision + f.ic.workspaceProjectedRevision = revision + before, _ := json.Marshal(f.wire) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/compatibility": + json.NewEncoder(w).Encode(sidecar.ProtocolInfo{ProtocolVersion: sidecar.ProtocolVersion}) + case "/rpc/state/account": + http.Error(w, "temporary failure", 503) + case "/rpc/state/dialogs": + writeConnectorSidecarResult(t, w, "dialogs", sidecar.DialogsPage{}) + default: + t.Errorf("unexpected source%s", r.URL.Path) + http.Error(w, "unexpected", 400) + } + })) + defer server.Close() + f.ic.Sidecar = sidecar.NewClient(server.URL).WithSessionNamespace("fixture") + if err := f.ic.syncDialogs(t.Context(), false); err != nil { + t.Fatal(err) + } + f.ic.projectDialogInLane(t.Context(), f.portal, workspaceDialog(1)) + after, _ := json.Marshal(f.wire) + if f.ic.workspaceRevision != revision || f.ic.spaceAccess[1] != "accessible" || !f.ic.spaceAccessComplete || !reflect.DeepEqual(before, after) { + t.Fatalf("temporary RPC failure changed snapshot/navigation: revision%d=>%d access%v", revision, f.ic.workspaceRevision, f.ic.spaceAccess) + } + row, err := f.ic.UserLogin.Bridge.DB.Portal.GetByKey(t.Context(), f.portal.PortalKey) + if err != nil || row.ParentKey.ID != "space:1" { + t.Fatalf("owned parent lost: %+v %v", row, err) + } +} +func TestWorkspaceUnknownHoldsPendingAttachAndDetachAcrossRestart(t *testing.T) { + for _, target := range []string{"", "space:2"} { + t.Run(target, func(t *testing.T) { + f := newWorkspaceWireFixture(t) + f.pending(t, target) + before, _ := json.Marshal(f.wire) + f.restart(t) + f.calls = 0 + f.ic.projectDialogInLane(t.Context(), f.portal, workspaceDialog(2)) + after, _ := json.Marshal(f.wire) + s, k, err := f.ic.projectionJournal(t.Context(), "7") + if err != nil { + t.Fatal(err) + } + p, err := s.load(t.Context(), k) + if err != nil || f.calls != 0 || !reflect.DeepEqual(before, after) || !p.ParentPending || p.DesiredParent != target { + t.Fatalf("unknown access replayed pendingtarget%q: calls%d state%+v err%v", target, f.calls, p, err) + } + // Fresh positive access or explicit removal releases the persisted operation. + status := "accessible" + space := int64(2) + want := "space:2" + if target == "" { + status = "removed" + want = "" + } + f.ic.cacheWorkspaceSnapshot(&sidecar.AccountStateSnapshot{Spaces: []sidecar.SpaceRecord{{SpaceID: space, Name: "two"}}, SpaceAccess: []sidecar.SpaceAccessRecord{{SpaceID: space, Status: status}}}) + f.ic.projectDialogInLane(t.Context(), f.portal, workspaceDialog(space)) + p, err = s.load(t.Context(), k) + if err != nil || p.ParentPending || string(f.portal.ParentKey.ID) != want { + t.Fatalf("positive recovery failed status%s: %+v parent%+v %v", status, p, f.portal.ParentKey, err) + } + }) + } +} +func TestWorkspaceAccessThreeStateContractProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + space := hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64)) + status := hegel.Draw(ht, hegel.SampledFrom([]string{"accessible", "unknown", "removed", ""})) + complete := hegel.Draw(ht, hegel.Booleans()) + hasRecord := hegel.Draw(ht, hegel.Booleans()) + ic := &InlineClient{spaceAccess: map[int64]string{space: status}, spaceAccessComplete: complete, spaces: map[int64]sidecar.SpaceRecord{}} + if hasRecord { + ic.spaces[space] = sidecar.SpaceRecord{SpaceID: space} + } + if status == "" { + delete(ic.spaceAccess, space) + } + parent, known := ic.desiredWorkspace(workspaceDialog(space)) + expectedKnown := status == "removed" || (status == "" && complete) || (status == "accessible" && hasRecord) + expectedParent := "" + if status == "accessible" && hasRecord { + expectedParent = fmt.Sprintf("space:%d", space) + } + if known != expectedKnown || parent != expectedParent { + ht.Fatalf("access%s complete%t metadata%t => %q/%t expected%q/%t", status, complete, hasRecord, parent, known, expectedParent, expectedKnown) + } + }, hegel.WithTestCases(3000), hegel.WithDatabase(t.TempDir())) +} + +func TestWorkspaceExplicitInvalidAccessIsUnknownDespiteCompleteList(t *testing.T) { + ic := &InlineClient{spaceAccess: map[int64]string{1: ""}, spaceAccessComplete: true} + if parent, known := ic.desiredWorkspace(workspaceDialog(1)); known || parent != "" { + t.Fatal("invalid explicit access became proof of removal") + } + d := workspaceDialog(1) + d.Structure.SpaceID = nil + if parent, known := ic.desiredWorkspace(d); !known || parent != "" { + t.Fatal("known structural detach was blocked by unknown access") + } +} diff --git a/pkg/msgconv/convert.go b/pkg/msgconv/convert.go new file mode 100644 index 0000000..e66e87c --- /dev/null +++ b/pkg/msgconv/convert.go @@ -0,0 +1,39 @@ +// Package msgconv converts text and entities without network or account state. +package msgconv + +import ( + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +// InlineToMatrix resolves only identities verified for the current account. +type InlineToMatrix func(int64) (id.UserID, bool) + +// MatrixToInline resolves only identities verified for the current account. +type MatrixToInline func(id.UserID) (int64, bool) + +// ToMatrix retains the exact plain body and renders bold, italic, code/pre, +// safe links and explicitly resolved user mentions. Other kinds stay readable. +// Invalid spans/payloads and ambiguous overlapping links are omitted. Formatting +// is swept in O((text+entities) log entities) work, without recursive traversal. +// NUL/invalid UTF-8 or markup exceeding a linear input budget falls back to the +// exact body with no formatted body or mentions. No message length limit is +// expanded here: the host must enforce its existing text/transfer limits. +func ToMatrix(text string, entities []sidecar.MessageEntityRecord, resolve InlineToMatrix) event.MessageEventContent { + return render(text, entities, resolve) +} + +// FromMatrix preserves unformatted Body exactly. Explicit HTML is tokenized +// iteratively: bold/italic/code/pre, safe links and verified user pills become +// UTF-16 entities; unsupported blocks become readable text separated by lines. +// Active content and mx-reply fallback are skipped. HTML CRLF/CR becomes LF and +// NUL becomes U+FFFD; invalid UTF-8/tokenizer errors use the exact Body fallback. +// Only typed Matrix user links with an affirmative resolver can become mentions. +// No markup, URL, ID prefix or display name is trusted as account verification. +func FromMatrix(content *event.MessageEventContent, resolve MatrixToInline) (string, []sidecar.MessageEntityRecord) { + if content == nil { + return "", nil + } + return parse(content, resolve) +} diff --git a/pkg/msgconv/convert_test.go b/pkg/msgconv/convert_test.go new file mode 100644 index 0000000..455d339 --- /dev/null +++ b/pkg/msgconv/convert_test.go @@ -0,0 +1,728 @@ +package msgconv + +import ( + "fmt" + "math" + "reflect" + "strings" + "testing" + "unicode/utf16" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "golang.org/x/net/html" + "hegel.dev/go/hegel" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +func ptr[T any](v T) *T { return &v } +func ent(kind string, offset, length int64) sidecar.MessageEntityRecord { + return sidecar.MessageEntityRecord{Kind: kind, Offset: offset, Length: length} +} +func htmlText(t *testing.T, text string) string { + t.Helper() + root, err := html.Parse(strings.NewReader("
" + text + "
")) + if err != nil { + t.Fatal(err) + } + var out strings.Builder + stack := []*html.Node{root} + for len(stack) > 0 { + n := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if n.Type == html.TextNode { + out.WriteString(n.Data) + } + if n.Type == html.ElementNode && n.Data == "br" { + out.WriteByte('\n') + } + for c := n.LastChild; c != nil; c = c.PrevSibling { + stack = append(stack, c) + } + } + return out.String() +} +func allowInline(n int64) (id.UserID, bool) { return id.UserID(fmt.Sprintf("@u%d:test", n)), n > 0 } +func findEntity(entities []sidecar.MessageEntityRecord, kind string) *sidecar.MessageEntityRecord { + for _, e := range entities { + if e.Kind == kind { + return &e + } + } + return nil +} + +// Dropping entities, using byte offsets or marking all names as mentions breaks this. +func TestRenderExactUnicodeAndScopedMentions(t *testing.T) { + text := "😀 A<&\"\nA" + mention := ent("TYPE_MENTION", 3, 4) + mention.UserID = ptr(int64(9)) + got := ToMatrix(text, []sidecar.MessageEntityRecord{ent("TYPE_BOLD", 0, 2), mention, ent("TYPE_GROUP_MENTION", 8, 1)}, allowInline) + if got.Body != text || got.Format != event.FormatHTML || htmlText(t, got.FormattedBody) != text { + t.Fatalf("render: %+v", got) + } + if !strings.Contains(got.FormattedBody, "") || !strings.Contains(got.FormattedBody, "<") { + t.Fatalf("unescaped or unformatted: %q", got.FormattedBody) + } + if got.Mentions == nil || got.Mentions.Room || !reflect.DeepEqual(got.Mentions.UserIDs, []id.UserID{"@u9:test"}) { + t.Fatalf("wrong mentions: %+v", got.Mentions) + } + denied := ToMatrix(text, []sidecar.MessageEntityRecord{mention}, func(int64) (id.UserID, bool) { return "@unrelated:test", false }) + if denied.Mentions == nil || len(denied.Mentions.UserIDs) != 0 || denied.Mentions.Room { + t.Fatalf("denied resolution mentioned: %+v", denied.Mentions) + } +} + +// Malformed and ambiguous identity spans must never turn a different label into a pill. +func TestRenderRejectsMalformedAndCrossingMentions(t *testing.T) { + for _, r := range [][2]int64{{1, 1}, {-1, 2}, {0, 0}, {0, math.MaxInt64}, {math.MaxInt64, 2}, {0, -1}} { + e := ent("TYPE_MENTION", r[0], r[1]) + e.UserID = ptr(int64(9)) + got := ToMatrix("😀abc", []sidecar.MessageEntityRecord{e}, allowInline) + if got.Mentions == nil || len(got.Mentions.UserIDs) != 0 || strings.Contains(got.FormattedBody, "A é same same`} + text, entities := FromMatrix(input, func(mx id.UserID) (int64, bool) { return 42, mx == "@verified:test" }) + if text != "😀 A é same same" { + t.Fatalf("text %q", text) + } + want := []sidecar.MessageEntityRecord{ent("TYPE_BOLD", 3, 1), ent("TYPE_ITALIC", 5, 2)} + for _, w := range want { + got := findEntity(entities, w.Kind) + if got == nil || *got != w { + t.Fatalf("%s: %+v", w.Kind, entities) + } + } + mention := findEntity(entities, "TYPE_MENTION") + if mention == nil || mention.Offset != 8 || mention.Length != 4 || mention.UserID == nil || *mention.UserID != 42 { + t.Fatalf("mention: %+v", entities) + } + count := 0 + for _, e := range entities { + if e.Kind == "TYPE_MENTION" { + count++ + } + } + if count != 1 { + t.Fatalf("identity confusion: %+v", entities) + } +} + +func TestParseSafeLinksPreAndReadableUnsupportedBlocks(t *testing.T) { + input := &event.MessageEventContent{Body: "fallback", Format: event.FormatHTML, FormattedBody: `

badsafe

  • one
  • two
quote
x < y
`} + text, entities := FromMatrix(input, nil) + if text != "badsafe\none\ntwo\nquote\nx < y" { + t.Fatalf("text %q", text) + } + link := findEntity(entities, "TYPE_TEXT_URL") + if link == nil || link.Value == nil || *link.Value != `https://safe.test/?a=1&b="x"` || link.Offset != 3 || link.Length != 4 { + t.Fatalf("link: %+v", entities) + } + pre := findEntity(entities, "TYPE_PRE") + if pre == nil || pre.Value == nil || *pre.Value != "go" || pre.Length != 5 { + t.Fatalf("pre: %+v", entities) + } + if findEntity(entities, "TYPE_CODE") != nil { + t.Fatalf("duplicate pre/code: %+v", entities) + } +} + +func TestPlainContentIsNeverMarkdownOrHTMLParsed(t *testing.T) { + for _, format := range []event.Format{"", "unknown"} { + input := &event.MessageEventContent{Body: "**literal** x 😀", Format: format, FormattedBody: "ignored"} + text, entities := FromMatrix(input, nil) + if text != input.Body || len(entities) != 0 { + t.Fatalf("plain: %q %+v", text, entities) + } + } +} + +// Independent oracle enumerates UTF-16 encoded rune boundaries, including invalid full signed ranges. +func TestSignedRangesMentionExactlyTheOriginalSpan(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + raw := hegel.Draw(ht, hegel.Text()) + runes := []rune(raw) + if len(runes) > 80 { + runes = runes[:80] + } + text := string(runes) + off := hegel.Draw(ht, hegel.Integers[int64](math.MinInt64, math.MaxInt64)) + length := hegel.Draw(ht, hegel.Integers[int64](math.MinInt64, math.MaxInt64)) + e := ent("TYPE_MENTION", off, length) + e.UserID = ptr(int64(1)) + got := ToMatrix(text, []sidecar.MessageEntityRecord{e}, allowInline) + bounds := map[int64]bool{0: true} + var pos int64 + for _, r := range runes { + pos += int64(len(utf16.Encode([]rune{r}))) + bounds[pos] = true + } + valid := off >= 0 && length > 0 && off <= math.MaxInt64-length && bounds[off] && bounds[off+length] && !strings.ContainsRune(text, 0) + if got.Body != text { + ht.Fatalf("plain body changed") + } + if got.Mentions == nil || got.Mentions.Room || (len(got.Mentions.UserIDs) == 1) != valid { + ht.Fatalf("range %d+%d valid=%v mentions=%+v", off, length, valid, got.Mentions) + } + }, hegel.WithTestCases(3000), hegel.WithDatabase(t.TempDir())) +} + +func TestParseMatrixURIPillsAndAmbiguousAnchors(t *testing.T) { + input := &event.MessageEventContent{Format: event.FormatHTML, FormattedBody: `😀!`} + text, entities := FromMatrix(input, func(mx id.UserID) (int64, bool) { return 42, mx == "@verified:test" }) + mention := findEntity(entities, "TYPE_MENTION") + if text != "😀!" || mention == nil || mention.Offset != 0 || mention.Length != 2 || mention.UserID == nil || *mention.UserID != 42 { + t.Fatalf("matrix URI pill: %q %+v", text, entities) + } + input.FormattedBody = `ABC` + text, entities = FromMatrix(input, func(mx id.UserID) (int64, bool) { + if mx == "@verified:test" { + return 42, true + } + return 99, true + }) + if text != "ABC" || findEntity(entities, "TYPE_MENTION") != nil { + t.Fatalf("ambiguous nested pills: %q %+v", text, entities) + } +} + +func TestEverySupportedSourceKindPreservesUnicodeSpans(t *testing.T) { + kinds := []string{"TYPE_BOLD", "TYPE_ITALIC", "TYPE_CODE", "TYPE_PRE", "TYPE_MENTION", "TYPE_TEXT_URL", "TYPE_URL", "TYPE_EMAIL", "TYPE_PHONE_NUMBER", "TYPE_GROUP_MENTION", "TYPE_THREAD", "TYPE_USERNAME_MENTION", "TYPE_BOT_COMMAND", "TYPE_UNDERLINE", "TYPE_STRIKETHROUGH", "TYPE_HIGHLIGHT", "TYPE_MATH", "TYPE_BLOCKQUOTE", "TYPE_THREAD_TITLE", "TYPE_UNSPECIFIED", "future"} + hegel.Test(t, func(ht *hegel.T) { + raw := hegel.Draw(ht, hegel.Text()) + runes := []rune(raw) + if len(runes) > 80 { + runes = runes[:80] + } + runes = append(runes, []rune("A😀é<&>\r\n")...) + a := hegel.Draw(ht, hegel.Integers[uint64](0, math.MaxUint64)) + b := hegel.Draw(ht, hegel.Integers[uint64](0, math.MaxUint64)) + start := int(a % uint64(len(runes))) + end := start + 1 + int(b%uint64(len(runes)-start)) + kind := hegel.Draw(ht, hegel.SampledFrom(kinds)) + off := int64(len(utf16.Encode(runes[:start]))) + length := int64(len(utf16.Encode(runes[start:end]))) + e := ent(kind, off, length) + switch kind { + case "TYPE_MENTION": + e.UserID = ptr(int64(7)) + case "TYPE_GROUP_MENTION": + e.GroupID = ptr(int64(2)) + case "TYPE_THREAD": + e.ChatID = ptr(int64(3)) + case "TYPE_PRE": + e.Value = ptr("go") + case "TYPE_TEXT_URL": + e.Value = ptr(`https://safe.test/?q="<&`) + } + text := string(runes) + got := ToMatrix(text, []sidecar.MessageEntityRecord{e}, allowInline) + if got.Body != text { + ht.Fatal("body changed") + } + if got.FormattedBody != "" && htmlText(t, got.FormattedBody) != text { + ht.Fatalf("HTML text changed %q => %q", text, got.FormattedBody) + } + expectMention := kind == "TYPE_MENTION" && !strings.ContainsRune(text, 0) + if got.Mentions == nil || got.Mentions.Room || (len(got.Mentions.UserIDs) == 1) != expectMention { + ht.Fatalf("kind=%s %+v", kind, got.Mentions) + } + if expectMention { + root, err := html.Parse(strings.NewReader(got.FormattedBody)) + if err != nil { + ht.Fatal(err) + } + nodes := []*html.Node{root} + label := "" + for len(nodes) > 0 { + n := nodes[len(nodes)-1] + nodes = nodes[:len(nodes)-1] + if n.Type == html.ElementNode && n.Data == "a" { + var s strings.Builder + children := []*html.Node{n} + for len(children) > 0 { + c := children[len(children)-1] + children = children[:len(children)-1] + if c.Type == html.TextNode { + s.WriteString(c.Data) + } + if c.Type == html.ElementNode && c.Data == "br" { + s.WriteByte('\n') + } + for ch := c.LastChild; ch != nil; ch = ch.PrevSibling { + children = append(children, ch) + } + } + label += s.String() + } + for c := n.LastChild; c != nil; c = c.PrevSibling { + nodes = append(nodes, c) + } + } + if label != string(runes[start:end]) { + ht.Fatalf("wrong mention label %q expected %q", label, string(runes[start:end])) + } + } + }, hegel.WithTestCases(3000), hegel.WithDatabase(t.TempDir())) +} + +func TestParserEscapingAndExactFormattingRanges(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + raw := hegel.Draw(ht, hegel.Text()) + runes := []rune(raw) + if len(runes) > 80 { + runes = runes[:80] + } + value := string(runes) + // This oracle follows the HTML parser's specified NUL/CR normalization. + expected := strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(value, "\x00", "�"), "\r\n", "\n"), "\r", "\n") + input := &event.MessageEventContent{Body: "fallback", Format: event.FormatHTML, FormattedBody: "😀" + strings.NewReplacer("&", "&", "<", "<", ">", ">").Replace(value) + "!"} + text, entities := FromMatrix(input, nil) + if text != "😀"+expected+"!" { + ht.Fatalf("decoded text: %q want %q", text, "😀"+expected+"!") + } + bold := findEntity(entities, "TYPE_BOLD") + wantLength := int64(len(utf16.Encode([]rune(expected)))) + if wantLength == 0 { + if bold != nil { + ht.Fatal("empty span retained") + } + } else if bold == nil || bold.Offset != 2 || bold.Length != wantLength { + ht.Fatalf("exact span: %+v expected len%d", entities, wantLength) + } + }, hegel.WithTestCases(3000), hegel.WithDatabase(t.TempDir())) +} + +func TestScaleIterativeParsingAndRendering(t *testing.T) { + const count = 100000 + text := strings.Repeat("x", count) + entities := make([]sidecar.MessageEntityRecord, count) + for i := range entities { + kind := "TYPE_BOLD" + if i%2 == 1 { + kind = "TYPE_ITALIC" + } + entities[i] = ent(kind, int64(i), 1) + } + got := ToMatrix(text, entities, nil) + if got.Body != text || htmlText(t, got.FormattedBody) != text { + t.Fatal("large render lost content") + } + parsed, native := FromMatrix(&event.MessageEventContent{Format: event.FormatHTML, FormattedBody: strings.Repeat("", count) + "😀" + strings.Repeat("", count)}, nil) + if parsed != "😀" || len(native) != count { + t.Fatalf("deep parsing lost content/entities %q %d", parsed, len(native)) + } + for _, e := range native { + if e.Kind != "TYPE_BOLD" || e.Offset != 0 || e.Length != 2 { + t.Fatalf("deep entity %+v", e) + } + } +} + +// Crossings can otherwise repeat a long pre-language payload for every link. +func TestRenderCrossingPayloadExpansionIsBounded(t *testing.T) { + text := strings.Repeat("xy", 2500) + language := strings.Repeat("l", 4096) + pre := ent("TYPE_PRE", 0, int64(len(text))) + pre.Value = &language + entities := []sidecar.MessageEntityRecord{pre} + for i := 0; i < len(text); i += 2 { + e := ent("TYPE_TEXT_URL", int64(i), 1) + e.Value = ptr("https://safe.test") + entities = append(entities, e) + } + got := ToMatrix(text, entities, nil) + // A generous linear envelope catches multiplied payloads. Falling back to + // the exact body is allowed when markup would exceed the documented budget. + envelope := 64 * (len(text) + len(language) + len(entities)*len("https://safe.test")) + if got.Body != text || len(got.FormattedBody) > envelope { + t.Fatalf("unbounded expansion: text=%d payload=%d entities=%d HTML=%d", len(text), len(language), len(entities), len(got.FormattedBody)) + } +} + +func TestUnsafeSchemesActiveContentAndInvalidResolversNeverMention(t *testing.T) { + for _, href := range []string{`javascript:alert(1)`, `JaVaScRiPt:alert(1)`, `data:text/html,bad`, `file:///etc/passwd`, `//evil.test`, ` https://safe.test`, `https://safe.test/` + "\n" + `bad`} { + e := ent("TYPE_TEXT_URL", 0, 1) + e.Value = &href + got := ToMatrix("X", []sidecar.MessageEntityRecord{e}, allowInline) + if strings.Contains(got.FormattedBody, "secretok`, `secretok`, `secretok`} { + calls := 0 + body, entities := FromMatrix(&event.MessageEventContent{Format: event.FormatHTML, FormattedBody: htmlSource}, func(id.UserID) (int64, bool) { calls++; return 1, true }) + if body != "ok" || len(entities) != 0 || calls != 0 { + t.Fatalf("active/fallback content: %q %+v %d", body, entities, calls) + } + } + for _, user := range []id.UserID{"", "not-an-id", "@x:bad server", id.UserID("@" + strings.Repeat("x", 300) + ":test")} { + e := ent("TYPE_MENTION", 0, 1) + e.UserID = ptr(int64(1)) + got := ToMatrix("X", []sidecar.MessageEntityRecord{e}, func(int64) (id.UserID, bool) { return user, true }) + if len(got.Mentions.UserIDs) != 0 { + t.Fatalf("invalid resolved user %q accepted", user) + } + } +} + +// A renderer that simply drops crossing styles would pass text-only checks. +func TestCrossingStylesMatchIndependentPerUTF16Coverage(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + raw := hegel.Draw(ht, hegel.Text()) + runes := []rune(raw) + if len(runes) > 80 { + runes = runes[:80] + } + runes = append(runes, []rune("😀éxy")...) + text := string(runes) + var units []int64 + units = append(units, 0) + for _, r := range runes { + units = append(units, units[len(units)-1]+int64(len(utf16.Encode([]rune{r})))) + } + var entities []sidecar.MessageEntityRecord + want := make([]uint8, units[len(units)-1]) + for i, kind := range []string{"TYPE_BOLD", "TYPE_ITALIC"} { + a := hegel.Draw(ht, hegel.Integers[uint64](0, math.MaxUint64)) + b := hegel.Draw(ht, hegel.Integers[uint64](0, math.MaxUint64)) + start := int(a % uint64(len(runes))) + end := start + 1 + int(b%uint64(len(runes)-start)) + entities = append(entities, ent(kind, units[start], units[end]-units[start])) + for p := units[start]; p < units[end]; p++ { + want[p] |= 1 << i + } + } + got := ToMatrix(text, entities, nil) + if got.Body != text { + ht.Fatal("body changed") + } + if strings.ContainsRune(text, 0) { + if got.FormattedBody != "" { + ht.Fatal("NUL fallback missing") + } + return + } + root, err := html.Parse(strings.NewReader("
" + got.FormattedBody + "
")) + if err != nil { + ht.Fatal(err) + } + type item struct { + node *html.Node + mask uint8 + } + stack := []item{{root, 0}} + var masks []uint8 + for len(stack) > 0 { + x := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if x.node.Type == html.ElementNode { + switch x.node.Data { + case "strong": + x.mask |= 1 + case "em": + x.mask |= 2 + case "br": + masks = append(masks, x.mask) + } + } + if x.node.Type == html.TextNode { + for _, r := range x.node.Data { + for range utf16.Encode([]rune{r}) { + masks = append(masks, x.mask) + } + } + } + for child := x.node.LastChild; child != nil; child = child.PrevSibling { + stack = append(stack, item{child, x.mask}) + } + } + if !reflect.DeepEqual(masks, want) { + ht.Fatalf("style coverage got%v want%v HTML%q", masks, want, got.FormattedBody) + } + }, hegel.WithTestCases(3000), hegel.WithDatabase(t.TempDir())) +} + +func TestParserEveryNativeFormattingKindAndIdentityDecision(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + tag := hegel.Draw(ht, hegel.SampledFrom([]string{"b", "strong", "i", "em", "code", "pre", "a"})) + allow := hegel.Draw(ht, hegel.SampledFrom([]bool{false, true})) + userID := hegel.Draw(ht, hegel.Integers[int64](1, math.MaxInt64)) + value := "A😀é<&" + open := "<" + tag + ">" + close := "" + kind := map[string]string{"b": "TYPE_BOLD", "strong": "TYPE_BOLD", "i": "TYPE_ITALIC", "em": "TYPE_ITALIC", "code": "TYPE_CODE", "pre": "TYPE_PRE", "a": "TYPE_TEXT_URL"}[tag] + if tag == "a" { + open = `` + if allow { + kind = "TYPE_MENTION" + } + } + text, entities := FromMatrix(&event.MessageEventContent{Format: event.FormatHTML, FormattedBody: open + "A😀é<&" + close}, func(mx id.UserID) (int64, bool) { return userID, allow && mx == "@verified:test" }) + if text != value || len(entities) != 1 { + ht.Fatalf("text/entities %q %+v", text, entities) + } + e := entities[0] + if e.Kind != kind || e.Offset != 0 || e.Length != int64(len(utf16.Encode([]rune(value)))) { + ht.Fatalf("native entity %+v", e) + } + if kind == "TYPE_MENTION" && (e.UserID == nil || *e.UserID != userID || e.Value != nil) { + ht.Fatalf("wrong target %+v", e) + } + }, hegel.WithTestCases(3000), hegel.WithDatabase(t.TempDir())) +} + +func TestNilInvalidUTF8AndMalformedIdentityUseExactFallback(t *testing.T) { + text, entities := FromMatrix(nil, nil) + if text != "" || len(entities) != 0 { + t.Fatalf("nil: %q %+v", text, entities) + } + invalid := string([]byte{'A', 0xff, 'B'}) + mention := ent("TYPE_MENTION", 0, 1) + mention.UserID = ptr(int64(1)) + calls := 0 + got := ToMatrix(invalid, []sidecar.MessageEntityRecord{mention}, func(int64) (id.UserID, bool) { calls++; return "@x:test", true }) + if got.Body != invalid || got.FormattedBody != "" || len(got.Mentions.UserIDs) != 0 || calls != 0 { + t.Fatal("invalid UTF8 rendered or resolved") + } + input := &event.MessageEventContent{Body: "exact **fallback**", Format: event.FormatHTML, FormattedBody: invalid} + text, entities = FromMatrix(input, func(id.UserID) (int64, bool) { calls++; return 1, true }) + if text != input.Body || len(entities) != 0 || calls != 0 { + t.Fatalf("bad HTML fallback: %q %+v", text, entities) + } + mention.GroupID = ptr(int64(5)) + got = ToMatrix("X", []sidecar.MessageEntityRecord{mention}, func(int64) (id.UserID, bool) { calls++; return "@x:test", true }) + if len(got.Mentions.UserIDs) != 0 || calls != 0 { + t.Fatal("conflicting identity payload resolved") + } + for _, userID := range []int64{0, -1, math.MinInt64} { + text, entities = FromMatrix(&event.MessageEventContent{Format: event.FormatHTML, FormattedBody: `X`}, func(id.UserID) (int64, bool) { return userID, true }) + if text != "X" || findEntity(entities, "TYPE_MENTION") != nil { + t.Fatalf("invalid native target %d: %q %+v", userID, text, entities) + } + } +} + +func TestInactiveVoidEmbedPreservesSuffix(t *testing.T) { + for _, markup := range []string{`AB`, `AB`} { + calls := 0 + body, entities := FromMatrix(&event.MessageEventContent{Body: "AB", Format: event.FormatHTML, FormattedBody: markup}, func(id.UserID) (int64, bool) { calls++; return 7, true }) + if body != "AB" || len(entities) != 0 || calls != 0 { + t.Fatalf("void element lost suffix or resolved identity: %q %+v calls%d", body, entities, calls) + } + } +} + +func TestInactiveSelfClosingContainersDoNotLeakContent(t *testing.T) { + for _, tag := range []string{"script", "style", "iframe", "object", "template", "mx-reply"} { + for _, closed := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/closed=%v", tag, closed), func(t *testing.T) { + markup := "A<" + tag + "/>hiddensecret" + want := "A" + if closed { + markup += "B" + want = "AB" + } + calls := 0 + got, entities := FromMatrix(&event.MessageEventContent{Body: "fallback", Format: event.FormatHTML, FormattedBody: markup}, func(id.UserID) (int64, bool) { calls++; return 7, true }) + if got != want || len(entities) != 0 || calls != 0 { + t.Fatalf("inactive content leaked or resolved: %q want%q entities%+v calls%d", got, want, entities, calls) + } + }) + } + } + for _, tag := range []string{"svg", "math"} { + got, entities := FromMatrix(&event.MessageEventContent{Body: "fallback", Format: event.FormatHTML, FormattedBody: "A<" + tag + "/>B"}, nil) + if got != "AB" || len(entities) != 0 { + t.Fatalf("closed foreign element consumed suffix: %q %+v", got, entities) + } + } +} + +func TestInactiveSelfClosingVisibleBoundaryProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + tag := hegel.Draw(ht, hegel.SampledFrom([]string{"script", "style", "iframe", "object", "template", "mx-reply"})) + closed := hegel.Draw(ht, hegel.Booleans()) + parts := make([]string, 2) + for i := range parts { + runes := []rune(hegel.Draw(ht, hegel.Text())) + if len(runes) > 80 { + runes = runes[:80] + } + parts[i] = string(runes) + } + normalize := strings.NewReplacer("\r\n", "\n", "\r", "\n", "\x00", "�") + escape := strings.NewReplacer("&", "&", "<", "<", ">", ">") + markup := escape.Replace(parts[0]) + "A<" + tag + "/>hiddensecret" + want := normalize.Replace(parts[0]) + "A" + if closed { + markup += "B" + escape.Replace(parts[1]) + want += "B" + normalize.Replace(parts[1]) + } + calls := 0 + got, entities := FromMatrix(&event.MessageEventContent{Body: "fallback", Format: event.FormatHTML, FormattedBody: markup}, func(id.UserID) (int64, bool) { calls++; return 7, true }) + if got != want || len(entities) != 0 || calls != 0 { + ht.Fatalf("visible boundary changed: %q want%q entities%+v calls%d", got, want, entities, calls) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func TestInactiveNestedSelfClosingContainersRemainBlockedProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + tag := hegel.Draw(ht, hegel.SampledFrom([]string{"template", "object", "mx-reply"})) + depth := 1 + int(hegel.Draw(ht, hegel.Integers[uint64](0, math.MaxUint64))%16) + var markup strings.Builder + markup.WriteString("A") + for range depth { + markup.WriteString("<" + tag) + if hegel.Draw(ht, hegel.Booleans()) { + markup.WriteString("/") + } + markup.WriteString(">") + } + for range depth { + markup.WriteString(`hidden") + } + markup.WriteString("B") + calls := 0 + got, entities := FromMatrix(&event.MessageEventContent{Body: "fallback", Format: event.FormatHTML, FormattedBody: markup.String()}, func(id.UserID) (int64, bool) { calls++; return 7, true }) + if got != "AB" || len(entities) != 0 || calls != 0 { + ht.Fatalf("nested inactive content leaked: %q entities%+v calls%d HTML%q", got, entities, calls, markup.String()) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func TestInactiveVoidEmbedSuffixProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + runes := []rune(hegel.Draw(ht, hegel.Text())) + if len(runes) > 80 { + runes = runes[:80] + } + suffix := string(runes) + "B" + expected := strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(suffix, "\x00", "�"), "\r\n", "\n"), "\r", "\n") + encoded := strings.NewReplacer("&", "&", "<", "<", ">", ">").Replace(suffix) + calls := 0 + got, entities := FromMatrix(&event.MessageEventContent{Body: "fallback", Format: event.FormatHTML, FormattedBody: `A` + encoded}, func(id.UserID) (int64, bool) { calls++; return 7, true }) + if got != "A"+expected || len(entities) != 0 || calls != 0 { + ht.Fatalf("suffix lost: %q want %q, entities%+v resolver%d", got, "A"+expected, entities, calls) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +// This oracle requires exact positive decoration coverage; empty HTML cannot +// pass by preserving only the plain body. It does not call converter helpers. +func TestCodePreAndSafeLinkHaveExactPositiveCoverage(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + runes := []rune(hegel.Draw(ht, hegel.Text())) + if len(runes) > 80 { + runes = runes[:80] + } + runes = append(runes, []rune("A😀éxy")...) + text := string(runes) + a := hegel.Draw(ht, hegel.Integers[uint64](0, math.MaxUint64)) + b := hegel.Draw(ht, hegel.Integers[uint64](0, math.MaxUint64)) + start := int(a % uint64(len(runes))) + end := start + 1 + int(b%uint64(len(runes)-start)) + kind := hegel.Draw(ht, hegel.SampledFrom([]string{"TYPE_CODE", "TYPE_PRE", "TYPE_TEXT_URL"})) + off := len(utf16.Encode(runes[:start])) + length := len(utf16.Encode(runes[start:end])) + entity := ent(kind, int64(off), int64(length)) + tag := "code" + href := "https://safe.test/?q=%22%3C%26" + if kind == "TYPE_PRE" { + tag = "pre" + entity.Value = ptr("go") + } + if kind == "TYPE_TEXT_URL" { + tag = "a" + entity.Value = &href + } + got := ToMatrix(text, []sidecar.MessageEntityRecord{entity}, nil) + if got.Body != text { + ht.Fatal("plain body changed") + } + if strings.ContainsRune(text, 0) { + if got.FormattedBody != "" { + ht.Fatal("NUL must keep plain fallback") + } + return + } + if got.Format != event.FormatHTML || got.FormattedBody == "" { + ht.Fatalf("supported%s decoration dropped", kind) + } + if htmlText(t, got.FormattedBody) != text { + ht.Fatal("decorated text changed") + } + root, err := html.Parse(strings.NewReader("
" + got.FormattedBody + "
")) + if err != nil { + ht.Fatal(err) + } + type nodeState struct { + node *html.Node + active bool + } + stack := []nodeState{{node: root}} + var actual []bool + for len(stack) > 0 { + x := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if x.node.Type == html.ElementNode && x.node.Data == tag { + x.active = true + if tag == "a" { + found := false + for _, attribute := range x.node.Attr { + if attribute.Key == "href" { + found = attribute.Val == href + } + } + if !found { + ht.Fatal("safe link destination changed") + } + } + } + if x.node.Type == html.TextNode { + for _, r := range x.node.Data { + for range utf16.Encode([]rune{r}) { + actual = append(actual, x.active) + } + } + } + if x.node.Type == html.ElementNode && x.node.Data == "br" { + actual = append(actual, x.active) + } + for child := x.node.LastChild; child != nil; child = child.PrevSibling { + stack = append(stack, nodeState{child, x.active}) + } + } + want := make([]bool, len(utf16.Encode(runes))) + for i := off; i < off+length; i++ { + want[i] = true + } + if !reflect.DeepEqual(actual, want) { + ht.Fatalf("%s coverage differs: got%v want%v", kind, actual, want) + } + }, hegel.WithTestCases(3000), hegel.WithDatabase(t.TempDir())) +} + +func TestInactiveVoidEmbedKeepsSuffixEntityUTF16Offset(t *testing.T) { + body, entities := FromMatrix(&event.MessageEventContent{Body: "A😀B", Format: event.FormatHTML, FormattedBody: `A😀B`}, nil) + if body != "A😀B" || len(entities) != 1 || entities[0].Kind != "TYPE_BOLD" || entities[0].Offset != 1 || entities[0].Length != 3 { + t.Fatalf("void tag shifted suffix entity: %q %+v", body, entities) + } +} diff --git a/pkg/msgconv/parse.go b/pkg/msgconv/parse.go new file mode 100644 index 0000000..269d00e --- /dev/null +++ b/pkg/msgconv/parse.go @@ -0,0 +1,240 @@ +package msgconv + +import ( + "io" + "strings" + "unicode/utf8" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "golang.org/x/net/html" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +type parseFrame struct { + tag string + previous int + start int64 + entity sidecar.MessageEntityRecord + href string +} + +func blockTag(tag string) bool { + switch tag { + case "p", "div", "li", "ul", "ol", "blockquote", "pre", "h1", "h2", "h3", "h4", "h5", "h6", "hr", "table", "tr": + return true + } + return false +} +func inactiveTag(tag string) bool { + switch tag { + case "script", "style", "iframe", "object", "embed", "svg", "math", "template", "mx-reply": + return true + } + return false +} + +// HTML ignores closing slashes on non-void containers. Foreign self-closing +// SVG/Math and void embed end immediately, including inside another blocker. +func opensInactiveTag(kind html.TokenType, tag string) bool { + return tag != "embed" && (kind == html.StartTagToken || + (kind == html.SelfClosingTagToken && tag != "svg" && tag != "math")) +} + +func attr(token html.Token, name string) string { + for _, a := range token.Attr { + if a.Key == name { + return a.Val + } + } + return "" +} + +func parse(content *event.MessageEventContent, resolve MatrixToInline) (string, []sidecar.MessageEntityRecord) { + if content.Format != event.FormatHTML || content.FormattedBody == "" || !formattedUTF8(content) { + return content.Body, nil + } + tokenizer := html.NewTokenizer(strings.NewReader(content.FormattedBody)) + var text strings.Builder + var position int64 + pendingBreak := false + lastNewline := false + write := func(value string) { + if value == "" { + return + } + if pendingBreak && text.Len() > 0 && !lastNewline { + text.WriteByte('\n') + position++ + } + pendingBreak = false + text.WriteString(value) + for _, r := range value { + position++ + if r > 0xffff { + position++ + } + } + lastNewline = strings.HasSuffix(value, "\n") + } + flushBreak := func() { + if pendingBreak && text.Len() > 0 && !lastNewline { + text.WriteByte('\n') + position++ + lastNewline = true + } + pendingBreak = false + } + stack := []parseFrame{} + latest := map[string]int{} + entities := []sidecar.MessageEntityRecord{} + links := []linkSpan{} + type resolved struct { + userID int64 + ok bool + } + identities := map[id.UserID]resolved{} + closeTo := func(index int) { + for i := len(stack) - 1; i >= index; i-- { + frame := stack[i] + if frame.previous < 0 { + delete(latest, frame.tag) + } else { + latest[frame.tag] = frame.previous + } + length := position - frame.start + if length <= 0 { + continue + } + if frame.tag == "a" { + links = append(links, linkSpan{start: int(frame.start), end: int(position), href: frame.href}) + continue + } + if frame.entity.Kind != "" { + e := frame.entity + e.Offset = frame.start + e.Length = length + entities = append(entities, e) + } + } + stack = stack[:index] + } + blocked := "" + blockedDepth := 0 + for { + kind := tokenizer.Next() + if kind == html.ErrorToken { + if tokenizer.Err() != io.EOF { + return content.Body, nil + } + break + } + token := tokenizer.Token() + if blocked != "" { + if token.Data == blocked && opensInactiveTag(kind, token.Data) { + blockedDepth++ + } + if kind == html.EndTagToken && token.Data == blocked { + blockedDepth-- + if blockedDepth == 0 { + blocked = "" + } + } + continue + } + if kind == html.TextToken { + write(strings.ReplaceAll(token.Data, "\x00", "�")) + continue + } + if kind != html.StartTagToken && kind != html.SelfClosingTagToken && kind != html.EndTagToken { + continue + } + tag := token.Data + if inactiveTag(tag) { + if opensInactiveTag(kind, tag) { + blocked = tag + blockedDepth = 1 + } + continue + } + if kind == html.EndTagToken { + if index, ok := latest[tag]; ok { + closeTo(index) + } + if blockTag(tag) { + pendingBreak = text.Len() > 0 + } + continue + } + if tag == "br" { + write("\n") + continue + } + if blockTag(tag) { + pendingBreak = text.Len() > 0 + } + frame := parseFrame{tag: tag, previous: -1} + switch tag { + case "b", "strong": + frame.entity.Kind = "TYPE_BOLD" + case "i", "em": + frame.entity.Kind = "TYPE_ITALIC" + case "code": + if pre, ok := latest["pre"]; ok { + for _, class := range strings.Fields(attr(token, "class")) { + if strings.HasPrefix(class, "language-") { + language := strings.TrimPrefix(class, "language-") + stack[pre].entity.Value = &language + break + } + } + continue + } + frame.entity.Kind = "TYPE_CODE" + case "pre": + frame.entity.Kind = "TYPE_PRE" + case "a": + frame.href = attr(token, "href") + default: + continue + } + flushBreak() + frame.start = position + if previous, ok := latest[tag]; ok { + frame.previous = previous + } + latest[tag] = len(stack) + stack = append(stack, frame) + if kind == html.SelfClosingTagToken { + closeTo(len(stack) - 1) + } + } + closeTo(0) + for _, link := range unambiguousLinks(links) { + e := sidecar.MessageEntityRecord{Kind: "TYPE_TEXT_URL", Offset: int64(link.start), Length: int64(link.end - link.start), Value: &link.href} + if uri, err := id.ParseMatrixURIOrMatrixToURL(link.href); err == nil && uri.UserID() != "" && validMatrixUser(uri.UserID()) && resolve != nil { + user := uri.UserID() + r, seen := identities[user] + if !seen { + r.userID, r.ok = resolve(user) + r.ok = r.ok && r.userID > 0 + identities[user] = r + } + if r.ok { + e.Kind = "TYPE_MENTION" + e.Value = nil + e.UserID = &r.userID + } + } + if e.Kind == "TYPE_MENTION" || safeURL(link.href) { + entities = append(entities, e) + } + } + return text.String(), entities +} + +// Avoid accepting an invalid UTF-8 formatted buffer as an alternate source of +// identity labels. The ordinary body remains the host's exact fallback. +func formattedUTF8(content *event.MessageEventContent) bool { + return utf8.ValidString(content.FormattedBody) +} diff --git a/pkg/msgconv/render.go b/pkg/msgconv/render.go new file mode 100644 index 0000000..4f6a1a7 --- /dev/null +++ b/pkg/msgconv/render.go @@ -0,0 +1,317 @@ +package msgconv + +import ( + "html" + "net/url" + "sort" + "strings" + "unicode" + "unicode/utf8" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" +) + +type linkSpan struct { + start, end int + href string + user id.UserID +} + +// Reject every ambiguous connected overlap, not just the later link. Identical +// spans and targets are deduplicated first. Touching endpoints do not overlap. +func unambiguousLinks(links []linkSpan) []linkSpan { + sort.Slice(links, func(i, j int) bool { + a, b := links[i], links[j] + if a.start != b.start { + return a.start < b.start + } + if a.end != b.end { + return a.end < b.end + } + if a.href != b.href { + return a.href < b.href + } + return a.user < b.user + }) + unique := links[:0] + for _, l := range links { + if len(unique) == 0 || unique[len(unique)-1] != l { + unique = append(unique, l) + } + } + out := unique[:0] + for i := 0; i < len(unique); { + j, end := i+1, unique[i].end + for j < len(unique) && unique[j].start < end { + if unique[j].end > end { + end = unique[j].end + } + j++ + } + if j == i+1 { + out = append(out, unique[i]) + } + i = j + } + return out +} + +func safeURL(raw string) bool { + if raw == "" || strings.IndexFunc(raw, func(r rune) bool { return unicode.IsControl(r) || unicode.IsSpace(r) }) >= 0 { + return false + } + parsed, err := url.Parse(raw) + if err != nil { + return false + } + switch strings.ToLower(parsed.Scheme) { + case "http", "https": + return parsed.Host != "" && parsed.User == nil + case "mailto", "tel": + return parsed.Opaque != "" && !strings.HasPrefix(parsed.Opaque, "//") + default: + return false + } +} +func validMatrixUser(user id.UserID) bool { + local, server, err := user.ParseAndValidateRelaxed() + return err == nil && local != "" && server != "" && strings.IndexFunc(string(user), func(r rune) bool { return unicode.IsSpace(r) || unicode.IsControl(r) }) < 0 +} +func payloadAllowed(e sidecar.MessageEntityRecord) bool { + if e.UserID != nil && e.Kind != "TYPE_MENTION" && e.Kind != "TYPE_BOT_COMMAND" { + return false + } + if e.AgentID != nil && (e.Kind != "TYPE_MENTION" || *e.AgentID <= 0) { + return false + } + if e.GroupID != nil && e.Kind != "TYPE_GROUP_MENTION" { + return false + } + if e.ChatID != nil && e.Kind != "TYPE_THREAD" { + return false + } + return e.Value == nil || e.Kind == "TYPE_PRE" || e.Kind == "TYPE_TEXT_URL" +} + +type renderEvent struct { + at, style, delta int + link *linkSpan +} +type wrapper struct{ open, close string } + +func render(text string, entities []sidecar.MessageEntityRecord, resolve InlineToMatrix) event.MessageEventContent { + out := event.MessageEventContent{MsgType: event.MsgText, Body: text, Mentions: &event.Mentions{}} + // HTML cannot faithfully carry NUL. Keep the exact plain body and no pills. + if !utf8.ValidString(text) || strings.ContainsRune(text, 0) { + return out + } + boundaries := map[int64]int{0: 0} + var units int64 + for index, r := range text { + width := int64(1) + if r > 0xffff { + width = 2 + } + units += width + boundaries[units] = index + utf8.RuneLen(r) + } + budget := markupBudget(text, entities) + events := make([]renderEvent, 0, len(entities)*2) + links := make([]linkSpan, 0) + preLanguage := "" + languageSet, languageConflict := false, false + type resolved struct { + user id.UserID + href string + ok bool + } + identities := map[int64]resolved{} + for _, e := range entities { + if e.Offset < 0 || e.Length <= 0 || e.Offset > units || e.Length > units-e.Offset || !payloadAllowed(e) { + continue + } + start, ok := boundaries[e.Offset] + if !ok { + continue + } + end, ok := boundaries[e.Offset+e.Length] + if !ok { + continue + } + style := -1 + href := "" + var user id.UserID + switch e.Kind { + case "TYPE_BOLD": + style = 0 + case "TYPE_ITALIC": + style = 1 + case "TYPE_CODE": + style = 2 + case "TYPE_PRE": + style = 3 + lang := "" + if e.Value != nil { + lang = *e.Value + } + if !languageSet { + preLanguage = lang + languageSet = true + } else if preLanguage != lang { + languageConflict = true + } + case "TYPE_MENTION": + if e.UserID == nil || *e.UserID <= 0 || resolve == nil { + continue + } + r, seen := identities[*e.UserID] + if !seen { + r.user, r.ok = resolve(*e.UserID) + r.ok = r.ok && validMatrixUser(r.user) + if r.ok { + r.href = r.user.URI().MatrixToURL() + } + identities[*e.UserID] = r + } + if !r.ok { + continue + } + user = r.user + href = r.href + case "TYPE_TEXT_URL": + if e.Value != nil { + href = *e.Value + } + case "TYPE_URL": + href = text[start:end] + case "TYPE_EMAIL": + href = "mailto:" + text[start:end] + case "TYPE_PHONE_NUMBER": + href = "tel:" + text[start:end] + } + if style >= 0 { + events = append(events, renderEvent{at: start, style: style, delta: 1}, renderEvent{at: end, style: style, delta: -1}) + } else if href != "" && safeURL(href) { + links = append(links, linkSpan{start, end, href, user}) + } + } + links = unambiguousLinks(links) + linkOpen := make(map[*linkSpan]string, len(links)) + for i := range links { + link := &links[i] + linkOpen[link] = `` + budget = addBudget(budget, len(link.href), 6) + events = append(events, renderEvent{at: link.start, style: -1, delta: 1, link: link}, renderEvent{at: link.end, style: -1, delta: -1, link: link}) + out.Mentions.Add(link.user) + } + if len(events) == 0 { + return out + } + sort.Slice(events, func(i, j int) bool { + if events[i].at != events[j].at { + return events[i].at < events[j].at + } + return events[i].delta < events[j].delta + }) + preOpen := `
`
+	if preLanguage != "" && !languageConflict {
+		preOpen = `
`
+	}
+	var htmlOut strings.Builder
+	counts := [4]int{}
+	var active *linkSpan
+	previous := []wrapper{}
+	setWrappers := func(next []wrapper) {
+		common := 0
+		for common < len(previous) && common < len(next) && previous[common] == next[common] {
+			common++
+		}
+		for i := len(previous) - 1; i >= common; i-- {
+			htmlOut.WriteString(previous[i].close)
+		}
+		for i := common; i < len(next); i++ {
+			htmlOut.WriteString(next[i].open)
+		}
+		previous = next
+	}
+	position := 0
+	for i := 0; i < len(events); {
+		at := events[i].at
+		if at > position {
+			writeEscaped(&htmlOut, text[position:at])
+		}
+		j := i
+		for j < len(events) && events[j].at == at {
+			e := events[j]
+			if e.style >= 0 {
+				counts[e.style] += e.delta
+			} else if e.delta < 0 {
+				active = nil
+			} else {
+				active = e.link
+			}
+			j++
+		}
+		next := make([]wrapper, 0, 5)
+		if active != nil {
+			next = append(next, wrapper{linkOpen[active], ``})
+		}
+		if counts[3] > 0 {
+			next = append(next, wrapper{preOpen, `
`}) + } + if counts[0] > 0 { + next = append(next, wrapper{``, ``}) + } + if counts[1] > 0 { + next = append(next, wrapper{``, ``}) + } + if counts[2] > 0 && counts[3] == 0 { + next = append(next, wrapper{``, ``}) + } + setWrappers(next) + if htmlOut.Len() > budget { + return event.MessageEventContent{MsgType: event.MsgText, Body: text, Mentions: &event.Mentions{}} + } + position = at + i = j + } + if position < len(text) { + writeEscaped(&htmlOut, text[position:]) + } + setWrappers(nil) + if htmlOut.Len() > budget { + return event.MessageEventContent{MsgType: event.MsgText, Body: text, Mentions: &event.Mentions{}} + } + out.Format = event.FormatHTML + out.FormattedBody = htmlOut.String() + return out +} +func writeEscaped(out *strings.Builder, text string) { + escaped := html.EscapeString(text) + escaped = strings.ReplaceAll(escaped, "\r", " ") + escaped = strings.ReplaceAll(escaped, "\n", "
") + out.WriteString(escaped) +} + +// Six bytes covers HTML escaping each input byte. 256 bytes per entity covers +// reopening the fixed wrapper stack at both endpoints. Payload bytes are +// charged once; pathological crossings cannot multiply them without fallback. +func markupBudget(text string, entities []sidecar.MessageEntityRecord) int { + budget := addBudget(0, len(text), 6) + budget = addBudget(budget, len(entities), 256) + for _, entity := range entities { + if entity.Value != nil { + budget = addBudget(budget, len(*entity.Value), 6) + } + } + return budget +} +func addBudget(budget, size, factor int) int { + maximum := int(^uint(0) >> 1) + if size > (maximum-budget)/factor { + return maximum + } + return budget + size*factor +} diff --git a/pkg/sidecar/client.go b/pkg/sidecar/client.go index 6f9b12d..a7d00bf 100644 --- a/pkg/sidecar/client.go +++ b/pkg/sidecar/client.go @@ -27,6 +27,9 @@ type Client struct { // requested lossless event history. var ErrEventReplayUnavailable = errors.New("Inline sidecar event replay is unavailable") +// ErrEventGenerationRequired prevents acknowledging a cursor without its lineage. +var ErrEventGenerationRequired = errors.New("Inline sidecar acknowledgement requires an event generation") + type EventReplayUnavailableError struct { RequestedAfterSequence *uint64 `json:"requested_after_sequence,omitempty"` OldestRetainedSequence *uint64 `json:"oldest_retained_sequence,omitempty"` @@ -57,6 +60,22 @@ func NewClient(baseURL string) *Client { } } +// Compatibility probes protocol metadata without selecting or constructing an account. +func (c *Client) Compatibility(ctx context.Context) (*ProtocolInfo, error) { + var info ProtocolInfo + req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.BaseURL+"/compatibility", nil) + if err != nil { + return nil, err + } + if err := c.doRequest(req, &info, false); err != nil { + return nil, err + } + if info.ProtocolVersion != ProtocolVersion { + return nil, fmt.Errorf("unsupported Inline sidecar protocol %d, expected %d", info.ProtocolVersion, ProtocolVersion) + } + return &info, nil +} + func (c *Client) WithSessionNamespace(namespace string) *Client { copy := *c copy.SessionNamespace = strings.TrimSpace(namespace) @@ -303,6 +322,9 @@ func (c *Client) EventsAfter(ctx context.Context, namespace string, afterSequenc // EventsAfterGeneration opens the event stream within one durable adapter // event-log generation. func (c *Client) EventsAfterGeneration(ctx context.Context, namespace, generation string, afterSequence uint64) (*EventStream, error) { + if _, err := c.Compatibility(ctx); err != nil { + return nil, err + } query := url.Values{} query.Set("after_sequence", strconv.FormatUint(afterSequence, 10)) if strings.TrimSpace(namespace) != "" { @@ -312,9 +334,9 @@ func (c *Client) EventsAfterGeneration(ctx context.Context, namespace, generatio query.Set("generation", generation) } eventsURL := websocketURL(c.BaseURL, "/ws/events?"+query.Encode()) - options := &websocket.DialOptions{} + options := &websocket.DialOptions{HTTPHeader: http.Header{ProtocolVersionHeader: []string{strconv.Itoa(ProtocolVersion)}}, HTTPClient: c.HTTP} if strings.TrimSpace(c.SessionNamespace) != "" { - options.HTTPHeader = http.Header{"X-Inline-Session-Namespace": []string{c.SessionNamespace}} + options.HTTPHeader.Set("X-Inline-Session-Namespace", c.SessionNamespace) } conn, response, err := websocket.Dial(ctx, eventsURL, options) if err != nil { @@ -331,16 +353,32 @@ func (c *Client) EventsAfterGeneration(ctx context.Context, namespace, generatio return &EventStream{conn: conn}, nil } -// AckEvents acknowledges durable processing through sequence. +// AckEvents cannot safely acknowledge a cursor without its event-log generation. +// Deprecated: use AckEventsGeneration with the generation persisted with the cursor. func (c *Client) AckEvents(ctx context.Context, namespace string, sequence uint64) error { return c.AckEventsGeneration(ctx, namespace, "", sequence) } // AckEventsGeneration acknowledges durable processing within one event-log generation. func (c *Client) AckEventsGeneration(ctx context.Context, namespace, generation string, sequence uint64) error { + if strings.TrimSpace(generation) == "" { + return ErrEventGenerationRequired + } request := EventAckRequest{SessionNamespace: namespace, Generation: generation, Sequence: sequence} - var response EventAckResponse - return c.do(ctx, http.MethodPost, "/rpc/events/ack", request, &response) + // Pointers distinguish a missing/null confirmation from an explicit sequence + // zero. Keep the public response DTO convenient for server-side producers. + var response struct { + Generation *string `json:"generation"` + AcknowledgedSequence *uint64 `json:"acknowledged_sequence"` + } + if err := c.do(ctx, http.MethodPost, "/rpc/events/ack", request, &response); err != nil { + return err + } + if response.Generation == nil || *response.Generation != generation || + response.AcknowledgedSequence == nil || *response.AcknowledgedSequence != sequence { + return errors.New("Inline sidecar acknowledgement did not confirm the requested generation and sequence") + } + return nil } type EventStream struct { @@ -441,7 +479,7 @@ func (c *Client) do(ctx context.Context, method, path string, body any, out any) req.Header.Set("Content-Type", "application/json") } - return c.doRequest(req, out) + return c.doRequest(req, out, true) } func (c *Client) doRaw(ctx context.Context, method, path, contentType string, body io.Reader, out any) error { @@ -452,11 +490,19 @@ func (c *Client) doRaw(ctx context.Context, method, path, contentType string, bo if contentType != "" { req.Header.Set("Content-Type", contentType) } - return c.doRequest(req, out) + return c.doRequest(req, out, true) } -func (c *Client) doRequest(req *http.Request, out any) error { - if strings.TrimSpace(c.SessionNamespace) != "" { +func (c *Client) doRequest(req *http.Request, out any, accountScoped bool) error { + // Do not cache the probe: even a direct call after an adapter downgrade must + // reject before the old adapter's account extractors can execute a mutation. + if accountScoped { + if _, err := c.Compatibility(req.Context()); err != nil { + return err + } + } + req.Header.Set(ProtocolVersionHeader, strconv.Itoa(ProtocolVersion)) + if accountScoped && strings.TrimSpace(c.SessionNamespace) != "" { req.Header.Set("X-Inline-Session-Namespace", c.SessionNamespace) } httpClient := c.HTTP @@ -470,6 +516,16 @@ func (c *Client) doRequest(req *http.Request, out any) error { defer resp.Body.Close() if resp.StatusCode < 200 || resp.StatusCode >= 300 { + if resp.StatusCode == http.StatusGone { + var replayErr struct { + EventReplayUnavailableError + Code string `json:"error"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, 64*1024)).Decode(&replayErr); err == nil && + (replayErr.Code == "sidecar_event_generation_reset" || replayErr.Code == "sidecar_event_replay_unavailable") { + return &replayErr.EventReplayUnavailableError + } + } return fmt.Errorf("Inline sidecar %s %s returned HTTP %d", req.Method, req.URL.Path, resp.StatusCode) } if out == nil { diff --git a/pkg/sidecar/client_test.go b/pkg/sidecar/client_test.go index 28cea80..cd7aaef 100644 --- a/pkg/sidecar/client_test.go +++ b/pkg/sidecar/client_test.go @@ -15,7 +15,7 @@ import ( ) func TestClientStatusDecodesVersionedResponse(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := compatibleTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/status" { t.Fatalf("unexpected path %s", r.URL.Path) } @@ -46,7 +46,7 @@ func TestClientStatusDecodesVersionedResponse(t *testing.T) { } func TestClientConnectReturnsSidecarError(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := compatibleTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/rpc/connect" { t.Fatalf("unexpected path %s", r.URL.Path) } @@ -78,7 +78,7 @@ func TestClientConnectReturnsSidecarError(t *testing.T) { } func TestClientAuthStartAndVerify(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := compatibleTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/rpc/auth/start": var request AuthStartRequest @@ -141,7 +141,7 @@ func TestClientAuthStartAndVerify(t *testing.T) { } func TestClientResumeDecodesStatus(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := compatibleTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/rpc/resume" { t.Fatalf("unexpected path %s", r.URL.Path) } @@ -162,7 +162,7 @@ func TestClientResumeDecodesStatus(t *testing.T) { } func TestClientDialogsAndHistoryDecodeVersionedResponses(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := compatibleTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/rpc/dialogs": synced := int64(11) @@ -238,7 +238,7 @@ func TestClientDialogsAndHistoryDecodeVersionedResponses(t *testing.T) { } func TestClientParticipantsAndChatCreation(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := compatibleTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/rpc/chat/participants": var request ChatParticipantsRequest @@ -401,7 +401,7 @@ func TestClientParticipantsAndChatCreation(t *testing.T) { } func TestClientUploadSendsMultipartMetadataAndFile(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := compatibleTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/rpc/upload" { t.Fatalf("unexpected path %s", r.URL.Path) } @@ -579,7 +579,7 @@ func TestEventEnvelopeDecodesSpaceMemberChanged(t *testing.T) { } func TestEventsAfterRequestsNamespaceAndCursor(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := compatibleTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/ws/events" { t.Fatalf("unexpected path %s", r.URL.Path) } @@ -623,7 +623,7 @@ func TestEventsAfterRequestsNamespaceAndCursor(t *testing.T) { } func TestEventsAfterMapsGoneToReplayUnavailable(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + server := compatibleTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusGone) _, _ = w.Write([]byte(`{"error":"sidecar_event_replay_unavailable","requested_after_sequence":3,"oldest_retained_sequence":8,"latest_sequence":12,"event_generation":"generation-2"}`)) @@ -641,7 +641,7 @@ func TestEventsAfterMapsGoneToReplayUnavailable(t *testing.T) { } func TestAckEventsPostsDurableCursor(t *testing.T) { - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server := compatibleTestServer(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.URL.Path != "/rpc/events/ack" { t.Fatalf("unexpected path %s", r.URL.Path) } @@ -712,3 +712,18 @@ func stringPtr(value string) *string { func boolPtr(value bool) *bool { return &value } + +// Compatible fixtures expose the same static, account-free preflight as the adapter. +func compatibleTestServer(t *testing.T, handler http.Handler) *httptest.Server { + t.Helper() + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/compatibility" { + writeJSON(t, w, ProtocolInfo{ProtocolVersion: ProtocolVersion}) + return + } + if r.Header.Get(ProtocolVersionHeader) != "5" { + t.Errorf("missing protocol header on %s", r.URL.Path) + } + handler.ServeHTTP(w, r) + })) +} diff --git a/pkg/sidecar/protocol_gate_test.go b/pkg/sidecar/protocol_gate_test.go new file mode 100644 index 0000000..dfb7684 --- /dev/null +++ b/pkg/sidecar/protocol_gate_test.go @@ -0,0 +1,351 @@ +package sidecar + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "reflect" + "runtime/debug" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/coder/websocket" + "hegel.dev/go/hegel" +) + +// Isolate the former stack-overflow regression: a bad preflight must fail this +// test rather than take down the entire package's test process. +func TestClientSubpathPreflightRegression(t *testing.T) { + if os.Getenv("INLINE_PROTOCOL_SUBPATH_FIXTURE") != "1" { + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + cmd := exec.CommandContext(ctx, binary, "-test.run=^TestClientSubpathPreflightRegression$") + cmd.Env = append(os.Environ(), "INLINE_PROTOCOL_SUBPATH_FIXTURE=1") + output, err := cmd.CombinedOutput() + if err != nil { + if len(output) > 3000 { + output = output[:3000] + } + t.Fatalf("subpath fixture failed: %v\n%s", err, output) + } + return + } + debug.SetMaxStack(1 << 20) + var probes, operations atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get(ProtocolVersionHeader) != "5" { + t.Error("fixture request omitted the version header") + } + if strings.HasSuffix(r.URL.Path, "/compatibility") { + if r.Header.Get("X-Inline-Session-Namespace") != "" { + t.Error("compatibility selected an account") + } + probes.Add(1) + writeJSON(t, w, ProtocolInfo{ProtocolVersion: ProtocolVersion}) + return + } + if r.Header.Get("X-Inline-Session-Namespace") != "fixture" { + t.Error("account operation lost its namespace") + } + operations.Add(1) + switch { + case strings.HasSuffix(r.URL.Path, "/ws/events"): + conn, err := websocket.Accept(w, r, nil) + if err != nil { + t.Error(err) + return + } + defer conn.CloseNow() + _ = conn.Close(websocket.StatusNormalClosure, "fixture complete") + case strings.HasSuffix(r.URL.Path, "/rpc/upload"): + writeRPCResult(t, w, "message", MessageMutation{}) + default: + writeRPCResult(t, w, "empty", struct{}{}) + } + })) + defer server.Close() + hegel.Test(t, func(ht *hegel.T) { + // Vary the endpoint configuration over the full numeric segment domain; + // the path deliberately has more than one segment. + segment := hegel.Draw(ht, hegel.Integers(uint64(0), uint64(math.MaxUint64))) + client := NewClient(server.URL + fmt.Sprintf("/fixture-%d/nested", segment)).WithSessionNamespace("fixture") + beforeProbes, beforeOperations := probes.Load(), operations.Load() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if _, err := client.Compatibility(ctx); err != nil { + ht.Fatal(err) + } + if err := client.Read(ctx, ReadRequest{}); err != nil { + ht.Fatal(err) + } + if _, err := client.Upload(ctx, UploadRequest{}, []byte("fixture")); err != nil { + ht.Fatal(err) + } + stream, err := client.Events(ctx) + if err != nil { + ht.Fatal(err) + } + _ = stream.Close(websocket.StatusNormalClosure, "fixture complete") + if probes.Load()-beforeProbes != 4 || operations.Load()-beforeOperations != 3 { + ht.Fatalf("probe/operation counts = %d/%d, want 4/3", probes.Load()-beforeProbes, operations.Load()-beforeOperations) + } + }, hegel.WithTestCases(1000), hegel.WithDatabase(t.TempDir())) +} + +type protocolFixtureTransport func(*http.Request) (*http.Response, error) + +func (fn protocolFixtureTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return fn(req) +} + +// Run exported acknowledgement methods against controlled HTTP responses, +// including exact uint64 values that must never pass through float64. +func ackFixtureClient(payload string, status int, calls *atomic.Int64) *Client { + client := NewClient("http://fixture.invalid").WithSessionNamespace("fixture") + client.HTTP = &http.Client{Transport: protocolFixtureTransport(func(req *http.Request) (*http.Response, error) { + body, code := payload, status + if req.URL.Path == "/compatibility" { + body, code = `{"protocol_version":5}`, http.StatusOK + } else { + calls.Add(1) + } + return &http.Response{StatusCode: code, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(body)), Request: req}, nil + })} + return client +} + +func TestAckGenerationResetMetadataProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + latest := hegel.Draw(ht, hegel.Integers(uint64(0), uint64(math.MaxUint64))) + generation := fmt.Sprintf("generation-%d", latest) + payload := fmt.Sprintf(`{"error":"sidecar_event_generation_reset","latest_sequence":%d,"event_generation":%q,"reset_required":true}`, latest, generation) + var calls atomic.Int64 + client := ackFixtureClient(payload, http.StatusGone, &calls) + err := client.AckEventsGeneration(context.Background(), "fixture", "old-generation", latest) + var reset *EventReplayUnavailableError + if !errors.Is(err, ErrEventReplayUnavailable) || !errors.As(err, &reset) { + ht.Fatalf("ack reset error = %v, want typed replay-unavailable", err) + } + if !reset.ResetRequired || reset.EventGeneration != generation || reset.LatestSequence == nil || *reset.LatestSequence != latest { + ht.Fatalf("reset metadata changed: %#v", reset) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func TestAckConfirmationProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + sequence := hegel.Draw(ht, hegel.Integers(uint64(0), uint64(math.MaxUint64))) + generation := fmt.Sprintf("generation-%d", hegel.Draw(ht, hegel.Integers(uint64(0), uint64(math.MaxUint64)))) + variant := hegel.Draw(ht, hegel.Integers(0, 8)) + payload := fmt.Sprintf(`{"generation":%q,"acknowledged_sequence":%d}`, generation, sequence) + switch variant { + case 1: + payload = `{}` + case 2: + payload = `null` + case 3: + payload = fmt.Sprintf(`{"acknowledged_sequence":%d}`, sequence) + case 4: + payload = fmt.Sprintf(`{"generation":null,"acknowledged_sequence":%d}`, sequence) + case 5: + payload = fmt.Sprintf(`{"generation":%q}`, generation) + case 6: + payload = fmt.Sprintf(`{"generation":%q,"acknowledged_sequence":null}`, generation) + case 7: + payload = fmt.Sprintf(`{"generation":%q,"acknowledged_sequence":%d}`, generation+"-other", sequence) + case 8: + payload = fmt.Sprintf(`{"generation":%q,"acknowledged_sequence":%d}`, generation, sequence^1) + } + var calls atomic.Int64 + err := ackFixtureClient(payload, http.StatusOK, &calls).AckEventsGeneration(context.Background(), "fixture", generation, sequence) + if (err == nil) != (variant == 0) { + ht.Fatalf("confirmation %s: error=%v, valid=%v", payload, err, variant == 0) + } + }, hegel.WithTestCases(2000), hegel.WithDatabase(t.TempDir())) +} + +func TestAckMissingConfirmationDoesNotMeanZeroRegression(t *testing.T) { + for _, payload := range []string{`{}`, `null`, `{"generation":"G"}`, `{"generation":"G","acknowledged_sequence":null}`} { + var calls atomic.Int64 + if err := ackFixtureClient(payload, http.StatusOK, &calls).AckEventsGeneration(context.Background(), "fixture", "G", 0); err == nil { + t.Fatalf("missing zero confirmation accepted: %s", payload) + } + } + var calls atomic.Int64 + if err := ackFixtureClient(`{"generation":"G","acknowledged_sequence":0}`, http.StatusOK, &calls).AckEventsGeneration(context.Background(), "fixture", "G", 0); err != nil { + t.Fatalf("explicit zero confirmation rejected: %v", err) + } +} + +func TestAckRequiresGenerationBeforeTransport(t *testing.T) { + for _, generation := range []string{"", " ", "\t\n"} { + var calls atomic.Int64 + client := ackFixtureClient(`{}`, http.StatusOK, &calls) + if err := client.AckEventsGeneration(context.Background(), "fixture", generation, 1); err == nil { + t.Fatalf("missing generation %q accepted", generation) + } + if calls.Load() != 0 { + t.Fatal("missing generation reached account transport") + } + } + var calls atomic.Int64 + if err := ackFixtureClient(`{}`, http.StatusOK, &calls).AckEvents(context.Background(), "fixture", 1); err == nil || calls.Load() != 0 { + t.Fatalf("legacy ack without lineage must fail before transport: err=%v calls=%d", err, calls.Load()) + } +} + +func TestAckHTTPFailureIsNotAlwaysAnEventReset(t *testing.T) { + for _, fixture := range []struct { + status int + body string + }{ + {http.StatusGone, `{"error":"unrelated_resource_gone"}`}, + {http.StatusGone, `not json`}, + {http.StatusConflict, `{"error":"ProtocolMismatch"}`}, + {http.StatusServiceUnavailable, `{"error":"unavailable"}`}, + } { + var calls atomic.Int64 + err := ackFixtureClient(fixture.body, fixture.status, &calls).AckEventsGeneration(context.Background(), "fixture", "G", 1) + if err == nil || errors.Is(err, ErrEventReplayUnavailable) { + t.Fatalf("HTTP %d %s incorrectly mapped: %v", fixture.status, fixture.body, err) + } + } +} + +func TestCompatibilityRejectsBeforeDirectMutation(t *testing.T) { + for _, version := range []int{0, 4, 6} { + t.Run(strconv.Itoa(version), func(t *testing.T) { + var effects atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/compatibility" { + if r.Header.Get("X-Inline-Session-Namespace") != "" { + t.Error("compatibility probe must not select an account") + } + _ = json.NewEncoder(w).Encode(ProtocolInfo{ProtocolVersion: version}) + return + } + effects.Add(1) + writeRPCResult(t, w, "empty", struct{}{}) + })) + defer server.Close() + err := NewClient(server.URL).WithSessionNamespace("fixture").Read(context.Background(), ReadRequest{}) + if err == nil { + t.Error("incompatible adapter accepted mutation") + } + if got := effects.Load(); got != 0 { + t.Errorf("incompatible adapter executed %d mutations", got) + } + }) + } +} + +func TestEveryMutationCarriesVersionHeader(t *testing.T) { + var probes, effects atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/compatibility" { + probes.Add(1) + _ = json.NewEncoder(w).Encode(ProtocolInfo{ProtocolVersion: ProtocolVersion}) + return + } + if got := r.Header.Get("X-Inline-Protocol-Version"); got != "5" { + t.Errorf("request version header = %q, want 5", got) + http.Error(w, "protocol mismatch", http.StatusConflict) + return + } + effects.Add(1) + writeRPCResult(t, w, "empty", struct{}{}) + })) + defer server.Close() + if err := NewClient(server.URL).Read(context.Background(), ReadRequest{}); err != nil { + t.Fatal(err) + } + if probes.Load() != 1 || effects.Load() != 1 { + t.Fatalf("probes=%d effects=%d", probes.Load(), effects.Load()) + } +} + +func TestCompleteStructureSurvivesJSONWithExplicitClears(t *testing.T) { + raw := []byte(`{"chat_id":7,"structure":{"kind":"channel","peer_user_id":null,"space_id":null,"parent_chat_id":null,"parent_message_id":null}}`) + var dialog DialogRecord + if err := json.Unmarshal(raw, &dialog); err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(dialog) + if err != nil { + t.Fatal(err) + } + var result map[string]json.RawMessage + if err := json.Unmarshal(encoded, &result); err != nil { + t.Fatal(err) + } + if string(result["structure"]) != `{"kind":"channel","peer_user_id":null,"space_id":null,"parent_chat_id":null,"parent_message_id":null}` { + t.Fatalf("complete structure lost explicit clear semantics: %s", encoded) + } +} + +func TestStructuralAuthorityRoundTripAcrossIDDomain(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + remoteID := hegel.Draw(ht, hegel.Integers(int64(math.MinInt64), int64(math.MaxInt64))) + optionalID := func() *int64 { + if !hegel.Draw(ht, hegel.Booleans()) { + return nil + } + value := hegel.Draw(ht, hegel.Integers(int64(math.MinInt64), int64(math.MaxInt64))) + return &value + } + original := DialogRecord{ChatID: remoteID, Structure: &DialogStructure{ + Kind: hegel.Draw(ht, hegel.SampledFrom([]string{"direct", "channel", "reply_child"})), + PeerUserID: optionalID(), SpaceID: optionalID(), ParentChatID: optionalID(), ParentMessageID: optionalID(), + }} + raw, err := json.Marshal(original) + if err != nil { + ht.Fatal(err) + } + var decoded DialogRecord + if err := json.Unmarshal(raw, &decoded); err != nil { + ht.Fatal(err) + } + if !reflect.DeepEqual(decoded, original) { + ht.Fatalf("structural authority changed across JSON: %s", raw) + } + + }) +} + +func TestPartialStructureCannotMasqueradeAsCompleteDetach(t *testing.T) { + for _, field := range []string{"kind", "peer_user_id", "space_id", "parent_chat_id", "parent_message_id"} { + t.Run(field, func(t *testing.T) { + descriptor := map[string]any{"kind": "channel", "peer_user_id": nil, "space_id": nil, "parent_chat_id": nil, "parent_message_id": nil} + delete(descriptor, field) + raw, _ := json.Marshal(map[string]any{"chat_id": 7, "structure": descriptor}) + var dialog DialogRecord + if err := json.Unmarshal(raw, &dialog); err == nil { + t.Fatalf("partial descriptor accepted as authoritative: %s", raw) + } + }) + } +} + +func TestLegacyDescriptorAbsenceStaysUnknown(t *testing.T) { + var unknown DialogRecord + if err := json.Unmarshal([]byte(`{"chat_id":1}`), &unknown); err != nil { + t.Fatal(err) + } + if unknown.Structure != nil { + t.Fatal("legacy absence invented an authoritative descriptor") + } +} diff --git a/pkg/sidecar/types.go b/pkg/sidecar/types.go index 4ce40cc..623c405 100644 --- a/pkg/sidecar/types.go +++ b/pkg/sidecar/types.go @@ -8,7 +8,10 @@ import ( const ( // ProtocolVersion is the sidecar command/event protocol version expected by // this bridge. - ProtocolVersion = 4 + ProtocolVersion = 5 + + // ProtocolVersionHeader is checked before any account factory or handler. + ProtocolVersionHeader = "X-Inline-Protocol-Version" // DefaultBaseURL is the loopback sidecar endpoint. DefaultBaseURL = "http://127.0.0.1:29342" @@ -139,25 +142,66 @@ type DialogsRequest struct { const DialogsOrderStableChatID = "stable_chat_id" +// DialogStructure is a complete source-backed structural observation only. +// A nil descriptor is unknown; nullable fields here explicitly clear old values. +type DialogStructure struct { + Kind string `json:"kind"` + PeerUserID *int64 `json:"peer_user_id"` + SpaceID *int64 `json:"space_id"` + ParentChatID *int64 `json:"parent_chat_id"` + ParentMessageID *int64 `json:"parent_message_id"` +} + +// UnmarshalJSON requires explicit nullable members: absent keys are patches, +// not evidence that an observed workspace or parent was detached. +func (structure *DialogStructure) UnmarshalJSON(data []byte) error { + var fields map[string]json.RawMessage + if err := json.Unmarshal(data, &fields); err != nil { + return err + } + for _, key := range []string{"kind", "peer_user_id", "space_id", "parent_chat_id", "parent_message_id"} { + if _, ok := fields[key]; !ok { + return fmt.Errorf("incomplete Inline structural descriptor: missing %s", key) + } + } + type wireStructure DialogStructure + var decoded wireStructure + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + switch decoded.Kind { + case "direct", "channel", "reply_child": + default: + return fmt.Errorf("unsupported Inline structural kind %q", decoded.Kind) + } + *structure = DialogStructure(decoded) + return nil +} + type DialogRecord struct { - ChatID int64 `json:"chat_id"` - PeerUserID *int64 `json:"peer_user_id,omitempty"` - Title *string `json:"title,omitempty"` - Emoji *string `json:"emoji,omitempty"` - LastMessageID *int64 `json:"last_message_id,omitempty"` - SyncedThroughMessageID *int64 `json:"synced_through_message_id,omitempty"` - UnreadCount *uint32 `json:"unread_count,omitempty"` - SpaceID *int64 `json:"space_id,omitempty"` - IsPublic *bool `json:"is_public,omitempty"` - Archived *bool `json:"archived,omitempty"` - Pinned *bool `json:"pinned,omitempty"` - Open *bool `json:"open,omitempty"` - ChatListHidden *bool `json:"chat_list_hidden,omitempty"` - Order *string `json:"order,omitempty"` - PinnedOrder *string `json:"pinned_order,omitempty"` - NotificationMode *string `json:"notification_mode,omitempty"` - FollowMode *string `json:"follow_mode,omitempty"` - PinnedMessageIDs []int64 `json:"pinned_message_ids,omitempty"` + Description *string `json:"description,omitempty"` + CreatedBy *int64 `json:"created_by,omitempty"` + Structure *DialogStructure `json:"structure,omitempty"` + ParentChatID *int64 `json:"parent_chat_id,omitempty"` + ParentMessageID *int64 `json:"parent_message_id,omitempty"` + ChatID int64 `json:"chat_id"` + PeerUserID *int64 `json:"peer_user_id,omitempty"` + Title *string `json:"title,omitempty"` + Emoji *string `json:"emoji,omitempty"` + LastMessageID *int64 `json:"last_message_id,omitempty"` + SyncedThroughMessageID *int64 `json:"synced_through_message_id,omitempty"` + UnreadCount *uint32 `json:"unread_count,omitempty"` + SpaceID *int64 `json:"space_id,omitempty"` + IsPublic *bool `json:"is_public,omitempty"` + Archived *bool `json:"archived,omitempty"` + Pinned *bool `json:"pinned,omitempty"` + Open *bool `json:"open,omitempty"` + ChatListHidden *bool `json:"chat_list_hidden,omitempty"` + Order *string `json:"order,omitempty"` + PinnedOrder *string `json:"pinned_order,omitempty"` + NotificationMode *string `json:"notification_mode,omitempty"` + FollowMode *string `json:"follow_mode,omitempty"` + PinnedMessageIDs []int64 `json:"pinned_message_ids,omitempty"` } type DialogsPage struct { @@ -166,8 +210,36 @@ type DialogsPage struct { NextCursor string `json:"next_cursor,omitempty"` } +type SpaceRecord struct { + SpaceID int64 `json:"space_id"` + Name string `json:"name"` + Creator bool `json:"creator"` + Date int64 `json:"date"` + IsPublic *bool `json:"is_public"` + Seq *int32 `json:"seq,omitempty"` + GridEnabled *bool `json:"grid_enabled,omitempty"` +} + +type SpaceMemberRecord struct { + SpaceID int64 `json:"space_id"` + UserID int64 `json:"user_id"` + Role *string `json:"role"` + Date int64 `json:"date"` + CanAccessPublicChats bool `json:"can_access_public_chats"` +} + +type SpaceAccessRecord struct { + SpaceID int64 `json:"space_id"` + Status string `json:"status"` +} + type AccountStateSnapshot struct { - DeletedChatIDs []int64 `json:"deleted_chat_ids"` + Spaces []SpaceRecord `json:"spaces"` + SpaceMembers []SpaceMemberRecord `json:"space_members"` + Users []UserRecord `json:"users"` + SpaceAccess []SpaceAccessRecord `json:"space_access"` + SpaceAccessComplete bool `json:"space_access_complete"` + DeletedChatIDs []int64 `json:"deleted_chat_ids"` } type ChatStateRequest struct { @@ -291,7 +363,52 @@ type CreatedChat struct { ParentMessageID *int64 `json:"parent_message_id,omitempty"` } +type MessageEntityRecord struct { + Kind string `json:"kind"` + Offset int64 `json:"offset"` + Length int64 `json:"length"` + UserID *int64 `json:"user_id"` + AgentID *int64 `json:"agent_id"` + GroupID *int64 `json:"group_id"` + ChatID *int64 `json:"chat_id"` + Value *string `json:"value"` +} + +type MessageAttachmentRecord struct { + AttachmentID int64 `json:"attachment_id"` + Kind string `json:"kind"` + Title *string `json:"title"` + URL *string `json:"url"` + Provider *string `json:"provider"` +} + +type MessageActionRecord struct { + ActionID string `json:"action_id"` + Label string `json:"label"` + Kind string `json:"kind"` +} + +type AgentSessionMessageMetadata struct { + AgentSessionID int64 `json:"agent_session_id"` + Provider int32 `json:"provider"` + Role int32 `json:"role"` + Relation int32 `json:"relation"` +} + +type MessageMetadata struct { + Mentioned *bool `json:"mentioned"` + EditTimestamp *int64 `json:"edit_timestamp"` + Revision *int64 `json:"revision"` + SenderIsBot *bool `json:"sender_is_bot"` + IsService bool `json:"is_service"` + Entities []MessageEntityRecord `json:"entities"` + Attachments []MessageAttachmentRecord `json:"attachments"` + Actions []MessageActionRecord `json:"actions"` + AgentSession *AgentSessionMessageMetadata `json:"agent_session"` +} + type MessageRecord struct { + Metadata MessageMetadata `json:"metadata"` ChatID int64 `json:"chat_id"` MessageID int64 `json:"message_id"` SenderID int64 `json:"sender_id"` @@ -339,33 +456,39 @@ type ExternalID struct { } type SendTextRequest struct { - Peer Peer `json:"peer"` - Text string `json:"text"` - ExternalID *ExternalID `json:"external_id,omitempty"` - RandomID *int64 `json:"random_id,omitempty"` - ReplyToMessageID *int64 `json:"reply_to_message_id,omitempty"` + ParseMarkdown bool `json:"parse_markdown,omitempty"` + NotificationMode string `json:"notification_mode,omitempty"` + Entities []MessageEntityRecord `json:"entities,omitempty"` + Peer Peer `json:"peer"` + Text string `json:"text"` + ExternalID *ExternalID `json:"external_id,omitempty"` + RandomID *int64 `json:"random_id,omitempty"` + ReplyToMessageID *int64 `json:"reply_to_message_id,omitempty"` } type UploadRequest struct { - Peer Peer `json:"peer"` - Kind string `json:"kind"` - FileName *string `json:"file_name,omitempty"` - MimeType *string `json:"mime_type,omitempty"` - SizeBytes *uint64 `json:"size_bytes,omitempty"` - Caption *string `json:"caption,omitempty"` - Width *uint32 `json:"width,omitempty"` - Height *uint32 `json:"height,omitempty"` - DurationMS *uint64 `json:"duration_ms,omitempty"` - ExternalID *ExternalID `json:"external_id,omitempty"` - RandomID *int64 `json:"random_id,omitempty"` - ReplyToMessageID *int64 `json:"reply_to_message_id,omitempty"` + Entities []MessageEntityRecord `json:"entities,omitempty"` + Peer Peer `json:"peer"` + Kind string `json:"kind"` + FileName *string `json:"file_name,omitempty"` + MimeType *string `json:"mime_type,omitempty"` + SizeBytes *uint64 `json:"size_bytes,omitempty"` + Caption *string `json:"caption,omitempty"` + Width *uint32 `json:"width,omitempty"` + Height *uint32 `json:"height,omitempty"` + DurationMS *uint64 `json:"duration_ms,omitempty"` + ExternalID *ExternalID `json:"external_id,omitempty"` + RandomID *int64 `json:"random_id,omitempty"` + ReplyToMessageID *int64 `json:"reply_to_message_id,omitempty"` } type EditMessageRequest struct { - ChatID int64 `json:"chat_id"` - MessageID int64 `json:"message_id"` - Text string `json:"text"` - ExternalID *ExternalID `json:"external_id,omitempty"` + ParseMarkdown bool `json:"parse_markdown,omitempty"` + Entities []MessageEntityRecord `json:"entities,omitempty"` + ChatID int64 `json:"chat_id"` + MessageID int64 `json:"message_id"` + Text string `json:"text"` + ExternalID *ExternalID `json:"external_id,omitempty"` } type DeleteMessageRequest struct { diff --git a/scripts/check.sh b/scripts/check.sh index 71984b2..8040ef8 100755 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -28,8 +28,10 @@ bash -n scripts/check.sh scripts/smoke-local.sh scripts/e2e-local.sh docker-run. echo "==> Compose config" docker compose config >/dev/null -docker compose config | grep -q 'INLINE_API_BASE_URL:' -docker compose config | grep -q 'INLINE_REALTIME_URL:' +# Read the full output: grep -q closes the pipe early and can make Compose +# exit with a broken pipe under pipefail even when the expected key exists. +docker compose config | grep 'INLINE_API_BASE_URL:' >/dev/null +docker compose config | grep 'INLINE_REALTIME_URL:' >/dev/null echo "==> Go tests" go test -tags goolm ./... diff --git a/scripts/e2e-local.sh b/scripts/e2e-local.sh index 21f6f7e..84a4ca9 100755 --- a/scripts/e2e-local.sh +++ b/scripts/e2e-local.sh @@ -117,6 +117,8 @@ services: image: ${SYNAPSE_IMAGE} environment: SYNAPSE_CONFIG_PATH: /data/homeserver.yaml + UID: "$(id -u)" + GID: "$(id -g)" ports: - "127.0.0.1:${SYNAPSE_PORT}:8008" volumes: @@ -200,6 +202,8 @@ function generate_synapse_config { echo "==> Generating Synapse config with ${SYNAPSE_IMAGE}" docker run --rm \ -v "${SYNAPSE_DATA}:/data" \ + -e "UID=$(id -u)" \ + -e "GID=$(id -g)" \ -e "SYNAPSE_SERVER_NAME=${SERVER_NAME}" \ -e "SYNAPSE_REPORT_STATS=no" \ "${SYNAPSE_IMAGE}" generate @@ -225,9 +229,13 @@ function prepare { function wait_for_http { local name=$1 url=$2 + local -a protocol_headers=() + if [[ "${url}" == "${SIDECAR_URL}/"* ]]; then + protocol_headers=(-H "X-Inline-Protocol-Version: 5") + fi echo "==> Waiting for ${name}: ${url}" for _ in $(seq 1 90); do - if curl -fsS "${url}" >/dev/null 2>&1; then + if curl -fsS "${protocol_headers[@]}" "${url}" >/dev/null 2>&1; then return 0 fi sleep 1 @@ -397,8 +405,8 @@ function matrix_login_token { response=$(curl -fsS \ -H "Content-Type: application/json" \ -d "${body}" \ - "${HOMESERVER_ADDRESS}/_matrix/client/v3/login") - jq -r '.access_token' <<<"${response}" + "${HOMESERVER_ADDRESS}/_matrix/client/v3/login") || return + jq -er '.access_token | select(type == "string" and length > 0)' <<<"${response}" } function matrix_auth_json { @@ -469,10 +477,11 @@ function matrix_join_room { function wait_for_room_message_body { local token=$1 room_id=$2 pattern=$3 - local encoded_room response + local encoded_room response filter encoded_room=$(uri_encode "${room_id}") + filter=$(uri_encode '{"types":["m.room.message"]}') for _ in $(seq 1 60); do - response=$(matrix_auth_json "${token}" GET "/_matrix/client/v3/rooms/${encoded_room}/messages?dir=b&limit=50") + response=$(matrix_auth_json "${token}" GET "/_matrix/client/v3/rooms/${encoded_room}/messages?dir=b&limit=50&filter=${filter}") if jq -e --arg pattern "${pattern}" ' ($pattern | ascii_downcase) as $needle | @@ -492,13 +501,13 @@ function wait_for_room_message_body { function sidecar_post_json { local path=$1 body=${2:-} if [[ -n "${body}" ]]; then - curl -fsS \ + curl -fsS -H "X-Inline-Protocol-Version: 5" \ -X POST \ -H "Content-Type: application/json" \ -d "${body}" \ "${SIDECAR_URL}${path}" else - curl -fsS -X POST "${SIDECAR_URL}${path}" + curl -fsS -H "X-Inline-Protocol-Version: 5" -X POST "${SIDECAR_URL}${path}" fi } @@ -618,11 +627,11 @@ function wait_for_portal_room { } function fixture_sent_count { - curl -fsS "${SIDECAR_URL}/fixture/sent" | jq -r '.sent_texts | length' + curl -fsS -H "X-Inline-Protocol-Version: 5" "${SIDECAR_URL}/fixture/sent" | jq -r '.sent_texts | length' } function fixture_event_client_count { - curl -fsS "${SIDECAR_URL}/fixture/status" | jq -r '.event_clients' + curl -fsS -H "X-Inline-Protocol-Version: 5" "${SIDECAR_URL}/fixture/status" | jq -r '.event_clients' } function wait_for_fixture_sent_count { @@ -726,15 +735,253 @@ function fixture_prepare_logged_in_portals { wait_for_room_message_body "${token}" "${group_room}" "fixture group hello" FIXTURE_DM_ROOM="${dm_room}" FIXTURE_GROUP_ROOM="${group_room}" + FIXTURE_MGMT_ROOM="${room_id}" + FIXTURE_SOURCE_CALLS=$(fixture_source_calls) +} + +function fixture_status_json { + curl -fsS -H "X-Inline-Protocol-Version: 5" "${SIDECAR_URL}/fixture/status" +} + +function fixture_source_calls { + fixture_status_json | jq -c '[.source_calls["/rpc/read"] // 0, .source_calls["/rpc/marked-unread"] // 0, .source_calls["/rpc/dialog/notifications"] // 0]' +} + +function fixture_assert_no_source_calls { + [[ "$(fixture_source_calls)" == "${FIXTURE_SOURCE_CALLS}" ]] +} + +function fixture_presentation { + sidecar_post_json /fixture/presentation "$1" >/dev/null +} + +function fixture_room_state { + local token=$1 room=$2 + matrix_auth_json "${token}" GET "/_matrix/client/v3/rooms/$(uri_encode "${room}")/state" +} + +# Inspect emitted Matrix state as well as the bridge's persisted identity. +function fixture_wait_hierarchy { + local token=$1 child=$2 desired=$3 old=${4:-} + local state parent_state old_state + for _ in $(seq 1 60); do + state=$(fixture_room_state "${token}" "${child}") + if ! jq -e --arg parent "${desired}" ' + [.[] | select(.type == "m.space.parent" and ((.content.via // []) | length > 0))] as $parents + | if $parent == "" then ($parents | length == 0) + else ($parents | length == 1 and .[0].state_key == $parent and .[0].content.canonical == true) end + ' >/dev/null <<<"${state}"; then + sleep 1 + continue + fi + if [[ -n "${desired}" ]]; then + parent_state=$(fixture_room_state "${token}" "${desired}") + if ! jq -e --arg child "${child}" 'any(.[]; .type == "m.space.child" and .state_key == $child and ((.content.via // []) | length > 0))' >/dev/null <<<"${parent_state}"; then + sleep 1 + continue + fi + fi + if [[ -n "${old}" && "${old}" != "${desired}" ]]; then + old_state=$(fixture_room_state "${token}" "${old}") + if ! jq -e --arg child "${child}" 'all(.[]; .type != "m.space.child" or .state_key != $child or ((.content.via // []) | length == 0))' >/dev/null <<<"${old_state}"; then + sleep 1 + continue + fi + fi + return 0 + done + echo "Fixture hierarchy failed for ${child}, expected parent ${desired:-none}" >&2 + return 1 +} + +function fixture_assert_room_identity { + local token=$1 room=$2 kind=$3 state + state=$(fixture_room_state "${token}" "${room}") + jq -e --arg kind "${kind}" ' + [.[] | select(.type == "m.bridge" or .type == "uk.half-shot.bridge")] as $info + | ($info | length == 2) + and all($info[]; (.content["com.beeper.room_type.v2"] // "") == $kind + and (if $kind == "dm" then .content["com.beeper.room_type"] == "dm" else (.content["com.beeper.room_type"] // "") == "" end)) + ' >/dev/null <<<"${state}" +} + +function fixture_assert_presentation { + local token=$1 group=$2 dm=$3 space_a space_b hidden + echo "==> Checking emitted channel/DM identity, exclusions, and workspace links" + fixture_assert_room_identity "${token}" "${group}" "" + fixture_assert_room_identity "${token}" "${dm}" dm + hidden=$(bridge_db_scalar "SELECT COUNT(*) FROM portal WHERE id IN ('404','405') AND mxid IS NOT NULL AND mxid <> '';") + [[ "${hidden}" == 0 ]] + space_a=$(wait_for_portal_room space:301) + matrix_join_room "${token}" "${space_a}" + fixture_wait_hierarchy "${token}" "${group}" "${space_a}" + + fixture_presentation '{"action":"move","chat_id":202,"space_id":302}' + space_b=$(wait_for_portal_room space:302) + matrix_join_room "${token}" "${space_b}" + fixture_wait_hierarchy "${token}" "${group}" "${space_b}" "${space_a}" + fixture_presentation '{"action":"detach","chat_id":202}' + fixture_wait_hierarchy "${token}" "${group}" "" "${space_b}" + fixture_presentation '{"action":"move","chat_id":202,"space_id":301}' + fixture_wait_hierarchy "${token}" "${group}" "${space_a}" + fixture_presentation '{"action":"access_removed","space_id":301}' + fixture_wait_hierarchy "${token}" "${group}" "" "${space_a}" + fixture_presentation '{"action":"access_restored","space_id":301}' + fixture_wait_hierarchy "${token}" "${group}" "${space_a}" + [[ "$(wait_for_portal_room 202)" == "${group}" ]] + fixture_assert_no_source_calls +} + +function fixture_wait_read_applied { + local chat=$1 cursor=$2 applied + for _ in $(seq 1 60); do + applied=$(bridge_db_scalar "SELECT COALESCE(json_extract(state,'$.read_applied'),0) FROM inline_projection WHERE chat='${chat}';") + if [[ "${applied}" == "${cursor}" ]]; then + return 0 + fi + sleep 1 + done + echo "Fixture source cursor ${cursor} was not projected for chat ${chat}" >&2 + return 1 +} + +function fixture_assert_read_state { + local token=$1 bot_mxid=$2 room chat encoded_room encoded_user event10 event20 target state baseline_reads + echo "==> Checking exact read prefix, private receipt echo, and manual unread" + chat=$(( $(date +%s) * 100000 + RANDOM )) + fixture_presentation "$(jq -nc --argjson chat "${chat}" '{action:"receipt_case",chat_id:$chat}')" + room=$(wait_for_portal_room "${chat}") + FIXTURE_RECEIPT_CHAT="${chat}" + FIXTURE_RECEIPT_ROOM="${room}" + matrix_join_room "${token}" "${room}" + wait_for_room_message_body "${token}" "${room}" "fixture source unread 20" + wait_for_room_message_body "${token}" "${room}" "fixture source read 10" + encoded_room=$(uri_encode "${room}") + encoded_user=$(uri_encode "$(test_user_mxid)") + # Both messages are present, but cursor10 cannot cross the preceding unread20. + [[ "$(bridge_db_scalar "SELECT COALESCE(json_extract(state,'$.read_applied'),0) FROM inline_projection WHERE chat='${chat}';")" == 0 ]] + matrix_auth_json "${token}" PUT "/_matrix/client/v3/user/${encoded_user}/rooms/${encoded_room}/account_data/m.marked_unread" '{"unread":true}' >/dev/null + send_matrix_text "${token}" "${FIXTURE_MGMT_ROOM}" "!inline login-matrix ${token}" + wait_for_bot_message "${token}" "${FIXTURE_MGMT_ROOM}" "${bot_mxid}" "Successfully switched puppets" + event10=$(bridge_db_scalar "SELECT mxid FROM message WHERE id='${chat}/10' LIMIT 1;") + event20=$(bridge_db_scalar "SELECT mxid FROM message WHERE id='${chat}/20' LIMIT 1;") + [[ -n "${event10}" && -n "${event20}" ]] + fixture_presentation "$(jq -nc --argjson chat "${chat}" '{action:"read",chat_id:$chat,read_max_id:20,marked_unread:true}')" + fixture_wait_read_applied "${chat}" 20 + target=$(matrix_auth_json "${token}" GET "/_matrix/client/v3/user/${encoded_user}/rooms/${encoded_room}/account_data/m.fully_read" | jq -r '.event_id') + [[ "${target}" == "${event10}" ]] + matrix_auth_json "${token}" GET "/_matrix/client/v3/user/${encoded_user}/rooms/${encoded_room}/account_data/m.marked_unread" | jq -e '.unread == true' >/dev/null + fixture_wait_private_receipt "${token}" "${room}" "${event10}" + fixture_assert_no_source_calls + + # A late lower ID is still source-read, even after the desire was applied. + sidecar_post_json /fixture/push-message "$(jq -nc --argjson chat "${chat}" '{chat_id:$chat,message_id:15,timestamp:300,text:"fixture late read 15"}')" >/dev/null + wait_for_room_message_body "${token}" "${room}" "fixture late read 15" + fixture_wait_receipt_target "${token}" "${chat}" "${room}" 15 20 + # Persist a cursor whose target has not arrived yet, then deliver it. + fixture_presentation "$(jq -nc --argjson chat "${chat}" '{action:"read",chat_id:$chat,read_max_id:30,marked_unread:true}')" + fixture_wait_read_pending "${chat}" 30 + sidecar_post_json /fixture/push-message "$(jq -nc --argjson chat "${chat}" '{chat_id:$chat,message_id:30,timestamp:400,text:"fixture read before delivery 30"}')" >/dev/null + wait_for_room_message_body "${token}" "${room}" "fixture read before delivery 30" + fixture_wait_receipt_target "${token}" "${chat}" "${room}" 30 30 + fixture_assert_no_source_calls + + # A genuine public user receipt must still reach Inline exactly once. + baseline_reads=$(jq -r '.[0]' <<<"${FIXTURE_SOURCE_CALLS}") + matrix_auth_json "${token}" POST "/_matrix/client/v3/rooms/${encoded_room}/receipt/m.read/$(uri_encode "${event20}")" '{}' >/dev/null + for _ in $(seq 1 60); do + if fixture_status_json | jq -e --argjson expected "$((baseline_reads+1))" '(.source_calls["/rpc/read"] // 0) == $expected' >/dev/null; then + fixture_assert_receipt_barrier "${token}" "${chat}" "${room}" + return 0 + fi + sleep 1 + done + echo "Genuine Matrix user read receipt did not reach Inline exactly once" >&2 + return 1 +} + +function fixture_assert_receipt_barrier { + local token=$1 chat=$2 room=$3 encoded_room before after + echo "==> Checking arbitrary Matrix content blocks source read projection" + encoded_room=$(uri_encode "${room}") + before=$(matrix_auth_json "${token}" GET "/_matrix/client/v3/user/$(uri_encode "$(test_user_mxid)")/rooms/${encoded_room}/account_data/m.fully_read" | jq -r '.event_id') + matrix_auth_json "${token}" PUT "/_matrix/client/v3/rooms/${encoded_room}/send/m.sticker/fixture-barrier-${chat}" '{"body":"fixture unread sticker","url":"mxc://localhost/fixture-sticker","info":{"mimetype":"image/png","w":1,"h":1}}' >/dev/null + sidecar_post_json /fixture/push-message "$(jq -nc --argjson chat "${chat}" '{chat_id:$chat,message_id:40,timestamp:500,text:"fixture read after unknown sticker 40"}')" >/dev/null + wait_for_room_message_body "${token}" "${room}" "fixture read after unknown sticker 40" + fixture_presentation "$(jq -nc --argjson chat "${chat}" '{action:"read",chat_id:$chat,read_max_id:40,marked_unread:true}')" + fixture_wait_read_pending "${chat}" 40 + [[ "$(bridge_db_scalar "SELECT COALESCE(json_extract(state,'$.read_applied'),0) FROM inline_projection WHERE chat='${chat}';")" == 30 ]] + after=$(matrix_auth_json "${token}" GET "/_matrix/client/v3/user/$(uri_encode "$(test_user_mxid)")/rooms/${encoded_room}/account_data/m.fully_read" | jq -r '.event_id') + [[ "${after}" == "${before}" ]] +} + +function fixture_wait_read_pending { + local chat=$1 cursor=$2 + for _ in $(seq 1 60); do + if [[ "$(bridge_db_scalar "SELECT json_extract(state,'$.read_max_id') || ':' || json_extract(state,'$.read_pending') FROM inline_projection WHERE chat='${chat}';")" == "${cursor}:1" ]]; then + return 0 + fi + sleep 1 + done + echo "Missing pending source cursor ${cursor} for ${chat}" >&2 + return 1 +} + +function fixture_wait_private_receipt { + local token=$1 room=$2 target=$3 filter state + filter=$(jq -nc --arg room "${room}" '{room:{rooms:[$room],timeline:{limit:1},ephemeral:{types:["m.receipt"]}}}' | jq -sr '.[0] | tojson | @uri') + for _ in $(seq 1 60); do + state=$(matrix_auth_json "${token}" GET "/_matrix/client/v3/sync?timeout=0&filter=${filter}") + if jq -e --arg room "${room}" --arg event "${target}" --arg user "$(test_user_mxid)" 'any(.rooms.join[$room].ephemeral.events[]?; .type == "m.receipt" and .content[$event]["m.read.private"][$user] != null)' >/dev/null <<<"${state}"; then + return 0 + fi + sleep 1 + done + echo "Private receipt for the exact fixture target was not visible" >&2 + return 1 +} + +function fixture_wait_receipt_target { + local token=$1 chat=$2 room=$3 source=$4 coverage=$5 expected actual journal + for _ in $(seq 1 60); do + expected=$(bridge_db_scalar "SELECT mxid FROM message WHERE id='${chat}/${source}' LIMIT 1;") + journal=$(bridge_db_scalar "SELECT COALESCE(json_extract(state,'$.read_applied'),0) || ':' || json_extract(state,'$.read_pending') FROM inline_projection WHERE chat='${chat}';") + actual=$(matrix_auth_json "${token}" GET "/_matrix/client/v3/user/$(uri_encode "$(test_user_mxid)")/rooms/$(uri_encode "${room}")/account_data/m.fully_read" | jq -r '.event_id') + if [[ -n "${expected}" && "${actual}" == "${expected}" && "${journal}" == "${coverage}:0" ]]; then + return 0 + fi + sleep 1 + done + echo "Receipt did not retarget to source ${source} with coverage ${coverage}" >&2 + return 1 +} + +function fixture_assert_receipt_recovery { + local token=$1 chat=$2 room=$3 expected actual journal + echo "==> Checking persisted receipt barrier after restart" + expected=$(bridge_db_scalar "SELECT mxid FROM message WHERE id='${chat}/30' LIMIT 1;") + [[ -n "${expected}" ]] + for _ in $(seq 1 60); do + journal=$(bridge_db_scalar "SELECT json_extract(state,'$.read_max_id') || ':' || COALESCE(json_extract(state,'$.read_applied'),0) || ':' || json_extract(state,'$.read_pending') || ':' || (COALESCE(json_extract(state,'$.receipt_sync_token'),'') != '') FROM inline_projection WHERE chat='${chat}';") + actual=$(matrix_auth_json "${token}" GET "/_matrix/client/v3/user/$(uri_encode "$(test_user_mxid)")/rooms/$(uri_encode "${room}")/account_data/m.fully_read" | jq -r '.event_id') + if [[ "${journal}" == '40:30:1:1' && "${actual}" == "${expected}" ]]; then + matrix_auth_json "${token}" GET "/_matrix/client/v3/user/$(uri_encode "$(test_user_mxid)")/rooms/$(uri_encode "${room}")/account_data/m.marked_unread" | jq -e '.unread == true' >/dev/null + fixture_assert_no_source_calls + return 0 + fi + sleep 1 + done + echo "Receipt barrier or sync cursor was lost after fixture restart" >&2 + return 1 } function smoke { start echo "==> Adapter status" - curl -fsS "${SIDECAR_URL}/status" | jq . + curl -fsS -H "X-Inline-Protocol-Version: 5" "${SIDECAR_URL}/status" | jq . echo "==> Adapter resume" - curl -fsS -X POST "${SIDECAR_URL}/rpc/resume" | jq . + curl -fsS -H "X-Inline-Protocol-Version: 5" -X POST "${SIDECAR_URL}/rpc/resume" | jq . local token bot_localpart bot_mxid room_id token=$(matrix_login_token) @@ -871,6 +1118,8 @@ function fixture_check { fixture_prepare_logged_in_portals "${token}" "${bot_mxid}" dm_room="${FIXTURE_DM_ROOM}" + fixture_assert_presentation "${token}" "${FIXTURE_GROUP_ROOM}" "${dm_room}" + fixture_assert_read_state "${token}" "${bot_mxid}" echo "==> Sending Matrix outbound message through DM portal" initial_sent_count=$(fixture_sent_count) @@ -892,21 +1141,31 @@ function fixture_restart_check { require_live_commands start - local token bot_localpart bot_mxid dm_room catchup_text realtime_text + local token bot_localpart bot_mxid dm_room catchup_text realtime_text space_room token=$(matrix_login_token) bot_localpart=$(bridge_config_value appservice.bot.username) bot_mxid="@${bot_localpart}:${SERVER_NAME}" fixture_prepare_logged_in_portals "${token}" "${bot_mxid}" dm_room="${FIXTURE_DM_ROOM}" + fixture_assert_presentation "${token}" "${FIXTURE_GROUP_ROOM}" "${dm_room}" + fixture_assert_read_state "${token}" "${bot_mxid}" + FIXTURE_SOURCE_CALLS=$(fixture_source_calls) + space_room=$(wait_for_portal_room space:301) echo "==> Stopping Go bridge and queuing fixture message while bridge is down" stop_pid "bridge" "${RUN_DIR}/bridge.pid" catchup_text="fixture catchup after bridge restart $(date +%s)" fixture_push_message 101 "${catchup_text}" + fixture_presentation '{"action":"detach","chat_id":202}' echo "==> Restarting Go bridge and waiting for missed message catch-up" start_bridge wait_for_room_message_body "${token}" "${dm_room}" "${catchup_text}" + fixture_wait_hierarchy "${token}" "${FIXTURE_GROUP_ROOM}" "" "${space_room}" + [[ "$(wait_for_portal_room 202)" == "${FIXTURE_GROUP_ROOM}" ]] + fixture_assert_receipt_recovery "${token}" "${FIXTURE_RECEIPT_CHAT}" "${FIXTURE_RECEIPT_ROOM}" + fixture_presentation '{"action":"move","chat_id":202,"space_id":301}' + fixture_wait_hierarchy "${token}" "${FIXTURE_GROUP_ROOM}" "${space_room}" echo "==> Restarting bridge and fixture sidecar, then verifying realtime resumes" stop_pid "bridge" "${RUN_DIR}/bridge.pid" @@ -918,6 +1177,8 @@ function fixture_restart_check { wait_for_fixture_event_clients 1 fixture_push_message 101 "${realtime_text}" wait_for_room_message_body "${token}" "${dm_room}" "${realtime_text}" + fixture_wait_hierarchy "${token}" "${FIXTURE_GROUP_ROOM}" "${space_room}" + fixture_assert_receipt_recovery "${token}" "${FIXTURE_RECEIPT_CHAT}" "${FIXTURE_RECEIPT_ROOM}" echo "Fixture restart recovery and catch-up check passed." } @@ -932,7 +1193,7 @@ function status { else echo "Synapse: not reachable at ${HOMESERVER_ADDRESS}" fi - if curl -fsS "${SIDECAR_URL}/health" >/dev/null 2>&1; then + if curl -fsS -H "X-Inline-Protocol-Version: 5" "${SIDECAR_URL}/health" >/dev/null 2>&1; then echo "Adapter: reachable at ${SIDECAR_URL}" else echo "Adapter: not reachable at ${SIDECAR_URL}" diff --git a/scripts/e2econfig/docker_run_test.go b/scripts/e2econfig/docker_run_test.go new file mode 100644 index 0000000..a77f889 --- /dev/null +++ b/scripts/e2econfig/docker_run_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "context" + "math" + "net/http" + "net/http/httptest" + "net/url" + "os" + "os/exec" + "path/filepath" + "sync/atomic" + "testing" + "time" + + "hegel.dev/go/hegel" +) + +// Execute the real entrypoint readiness helper against a protocol-enforcing +// HTTP server. No config, account store, service, or container is created. +func checkEntrypointReadiness(t *testing.T, failures uint64) (bool, int64) { + t.Helper() + var calls atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + n := calls.Add(1) + if r.URL.Path != "/health" || r.Header.Get("X-Inline-Protocol-Version") != "5" { + http.Error(w, "protocol mismatch", 409) + return + } + if uint64(n) <= failures { + http.Error(w, "starting", 503) + return + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + address, err := url.Parse(server.URL) + if err != nil { + t.Fatal(err) + } + bash, err := exec.LookPath("bash") + if err != nil { + t.Fatal(err) + } + curl, err := exec.LookPath("curl") + if err != nil { + t.Fatal(err) + } + script, err := filepath.Abs("../../docker-run.sh") + if err != nil { + t.Fatal(err) + } + temp := t.TempDir() + bin := filepath.Join(temp, "bin") + if err = os.Mkdir(bin, 0700); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(bin, "curl"), []byte("#!/bin/sh\nexec \"$CURL_BINARY\" -A 'OpenAI File Downloader, XaiImageApiFetch/1.0' \"$@\"\n"), 0700); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, bash, "-c", `source "$1"; sidecar_pid=$$; sleep() { :; }; wait_for_sidecar`, "fixture", script) + cmd.Env = []string{"PATH=" + bin + ":/usr/bin:/bin", "CURL_BINARY=" + curl, "HOME=" + temp, "DATA_DIR=" + filepath.Join(temp, "must-not-create"), "INLINE_SIDECAR_BIND=" + address.Host} + output, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("readiness timed out: %s", output) + } + if _, statErr := os.Stat(filepath.Join(temp, "must-not-create")); !os.IsNotExist(statErr) { + t.Fatal("sourcing readiness helper had filesystem effects") + } + return err == nil, calls.Load() +} +func TestDockerEntrypointProtocolReadiness(t *testing.T) { + ok, calls := checkEntrypointReadiness(t, 0) + if !ok || calls != 2 { + t.Fatalf("healthy protocol5 sidecar rejected: ok%t calls%d", ok, calls) + } +} +func TestDockerEntrypointReadinessRetryBudgetProperty(t *testing.T) { + hegel.Test(t, func(ht *hegel.T) { + failures := hegel.Draw(ht, hegel.Integers[uint64](0, math.MaxUint64)) + if failures > 61 { + failures = 61 + } + ok, calls := checkEntrypointReadiness(t, failures) + expected := int64(failures) + 2 + if expected > 61 { + expected = 61 + } + if ok != (failures <= 60) || calls != expected { + ht.Fatalf("startupfailures%d: ok%t calls%d wantok%t calls%d", failures, ok, calls, failures <= 60, expected) + } + }, hegel.WithTestCases(30), hegel.WithDatabase(t.TempDir())) +} diff --git a/scripts/e2econfig/main.go b/scripts/e2econfig/main.go index f86e5fc..50c6d37 100644 --- a/scripts/e2econfig/main.go +++ b/scripts/e2econfig/main.go @@ -158,6 +158,8 @@ func patchBridgeConfig(doc map[string]any, patch bridgePatch) { patch.HomeserverDomain: "user", adminMXID: "admin", }) + // Cached source history uses the framework's historical-delivery path. + setPath(doc, "backfill.enabled", true) } type synapsePatch struct { @@ -174,6 +176,12 @@ func patchSynapseConfig(doc map[string]any, patch synapsePatch) { setPath(doc, "app_service_config_files", []any{patch.RegistrationPath}) setPath(doc, "report_stats", false) setPath(doc, "suppress_key_server_warning", true) + // Repeated fixture runs log in and create/join several synthetic rooms. + // Keep the local server's limits above that deterministic test workload. + for _, path := range []string{"rc_login.address", "rc_login.account", "rc_room_creation", "rc_joins.local", "rc_message"} { + setPath(doc, path+".per_second", 10) + setPath(doc, path+".burst_count", 100) + } } func readYAMLMap(path string) (map[string]any, error) { diff --git a/scripts/e2econfig/main_test.go b/scripts/e2econfig/main_test.go index 908c298..6412871 100644 --- a/scripts/e2econfig/main_test.go +++ b/scripts/e2econfig/main_test.go @@ -41,6 +41,7 @@ func TestPatchBridgeConfigSetsLocalE2EFields(t *testing.T) { assertPath(t, doc, "appservice.bot.avatar", "") assertPath(t, doc, "bridge.permissions.localhost", "user") assertPath(t, doc, "bridge.permissions.@alice:localhost", "admin") + assertPath(t, doc, "backfill.enabled", "true") } func TestPatchSynapseConfigSetsRegistrationAndLocalRegistration(t *testing.T) { @@ -58,6 +59,8 @@ func TestPatchSynapseConfigSetsRegistrationAndLocalRegistration(t *testing.T) { assertPath(t, doc, "registration_shared_secret", "local-secret") assertPath(t, doc, "report_stats", "false") assertPath(t, doc, "suppress_key_server_warning", "true") + assertPath(t, doc, "rc_login.address.burst_count", "100") + assertPath(t, doc, "rc_login.account.per_second", "10") files, ok := doc["app_service_config_files"].([]any) if !ok || len(files) != 1 || files[0] != "/data/matrix-inline-registration.yaml" { diff --git a/scripts/e2efixture/main.go b/scripts/e2efixture/main.go index 29a7695..423a8de 100644 --- a/scripts/e2efixture/main.go +++ b/scripts/e2efixture/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "crypto/rand" "encoding/json" "errors" "flag" @@ -32,6 +33,12 @@ const ( type fixtureServer struct { mu sync.Mutex + saveMu sync.Mutex + generation string + spaces []sidecar.SpaceRecord + access []sidecar.SpaceAccessRecord + readStates map[int64]sidecar.ReadStateRecord + sourceCalls map[string]int statePath string loggedIn bool nextMessageID int64 @@ -45,13 +52,22 @@ type fixtureServer struct { } type fixtureEvent struct { - sequence uint64 - payload []byte + Sequence uint64 `json:"sequence"` + Payload json.RawMessage `json:"payload"` } type storedState struct { - LoggedIn bool `json:"logged_in"` - NextMessageID int64 `json:"next_message_id"` + LoggedIn bool `json:"logged_in"` + NextMessageID int64 `json:"next_message_id"` + Generation string `json:"generation"` + Sequence uint64 `json:"sequence"` + Events []fixtureEvent `json:"events"` + Dialogs []sidecar.DialogRecord `json:"dialogs"` + Spaces []sidecar.SpaceRecord `json:"spaces"` + Access []sidecar.SpaceAccessRecord `json:"access"` + ReadStates map[int64]sidecar.ReadStateRecord `json:"read_states"` + History map[int64][]sidecar.MessageRecord `json:"history"` + SourceCalls map[string]int `json:"source_calls"` } type sentText struct { @@ -94,6 +110,11 @@ func newFixtureServer(statePath string) *fixtureServer { groupLast := int64(6001) return &fixtureServer{ statePath: statePath, + generation: rand.Text(), + spaces: []sidecar.SpaceRecord{{SpaceID: 301, Name: "Fixture Workspace A"}, {SpaceID: 302, Name: "Fixture Workspace B"}}, + access: []sidecar.SpaceAccessRecord{{SpaceID: 301, Status: "accessible"}, {SpaceID: 302, Status: "accessible"}}, + readStates: map[int64]sidecar.ReadStateRecord{101: {ChatID: 101}, 202: {ChatID: 202, ReadMaxID: int64Ptr(6000), MarkedUnread: true}, 208: {ChatID: 208, ReadMaxID: int64Ptr(10), MarkedUnread: true}}, + sourceCalls: make(map[string]int), nextMessageID: 7000, users: []sidecar.UserRecord{ {UserID: accountID, DisplayName: stringPtr("Fixture Self"), Username: stringPtr("fixture_self")}, @@ -101,10 +122,17 @@ func newFixtureServer(statePath string) *fixtureServer { {UserID: groupUserID, DisplayName: stringPtr("Lin Fixture"), Username: stringPtr("lin_fixture")}, }, dialogs: []sidecar.DialogRecord{ - {ChatID: dmChatID, PeerUserID: int64Ptr(peerUserID), Title: &dmTitle, LastMessageID: &dmLast}, - {ChatID: groupChatID, Title: &groupTitle, LastMessageID: &groupLast}, + {Structure: &sidecar.DialogStructure{Kind: "direct", PeerUserID: int64Ptr(peerUserID)}, ChatID: dmChatID, PeerUserID: int64Ptr(peerUserID), Title: &dmTitle, LastMessageID: &dmLast}, + {Structure: &sidecar.DialogStructure{Kind: "channel", SpaceID: int64Ptr(301)}, SpaceID: int64Ptr(301), NotificationMode: stringPtr("mentions"), IsPublic: boolPtr(false), ChatID: groupChatID, Title: &groupTitle, LastMessageID: &groupLast}, + {ChatID: 203, Title: stringPtr("Fixture Public"), Structure: &sidecar.DialogStructure{Kind: "channel", SpaceID: int64Ptr(301)}, SpaceID: int64Ptr(301), IsPublic: boolPtr(true), NotificationMode: stringPtr("all")}, + {ChatID: 208, Title: stringPtr("Fixture Receipt Order"), Structure: &sidecar.DialogStructure{Kind: "channel", SpaceID: int64Ptr(301)}, SpaceID: int64Ptr(301), LastMessageID: int64Ptr(20)}, + {ChatID: 404, Title: stringPtr("Fixture Hidden"), Structure: &sidecar.DialogStructure{Kind: "channel"}, ChatListHidden: boolPtr(true)}, + {ChatID: 405, Title: stringPtr("Fixture Child"), Structure: &sidecar.DialogStructure{Kind: "reply_child", ParentChatID: int64Ptr(202), ParentMessageID: int64Ptr(6001)}, ParentChatID: int64Ptr(202), ParentMessageID: int64Ptr(6001), ChatListHidden: boolPtr(true)}, + {ChatID: 406, Title: stringPtr("Fixture Unknown")}, + {ChatID: 407, Title: stringPtr("Fixture Archived"), Structure: &sidecar.DialogStructure{Kind: "channel"}, Archived: boolPtr(true), NotificationMode: stringPtr("none")}, }, history: map[int64][]sidecar.MessageRecord{ + 208: {{ChatID: 208, MessageID: 20, SenderID: groupUserID, Timestamp: 100, Content: sidecar.MessageContent{Type: "text", Text: "fixture source unread 20"}}, {ChatID: 208, MessageID: 10, SenderID: groupUserID, Timestamp: 200, Content: sidecar.MessageContent{Type: "text", Text: "fixture source read 10"}}}, dmChatID: {{ ChatID: dmChatID, MessageID: dmLast, @@ -125,6 +153,7 @@ func newFixtureServer(statePath string) *fixtureServer { } func (srv *fixtureServer) register(mux *http.ServeMux) { + mux.HandleFunc("/compatibility", func(w http.ResponseWriter, r *http.Request) { srv.writeJSON(w, srv.protocol()) }) mux.HandleFunc("/health", srv.handleHealth) mux.HandleFunc("/status", srv.handleStatus) mux.HandleFunc("/rpc/resume", srv.handleResume) @@ -135,18 +164,22 @@ func (srv *fixtureServer) register(mux *http.ServeMux) { mux.HandleFunc("/rpc/dialogs", srv.handleDialogs) mux.HandleFunc("/rpc/state/dialogs", srv.handleDialogs) mux.HandleFunc("/rpc/history", srv.handleHistory) + mux.HandleFunc("/rpc/state/chat/history", srv.handleHistory) + mux.HandleFunc("/rpc/state/account", srv.handleAccountState) + mux.HandleFunc("/rpc/state/chat", srv.handleChatState) mux.HandleFunc("/rpc/chat/participants", srv.handleParticipants) mux.HandleFunc("/rpc/chat/participants/add", srv.handleEmpty) mux.HandleFunc("/rpc/chat/participants/remove", srv.handleEmpty) mux.HandleFunc("/rpc/chat/info", srv.handleEmpty) mux.HandleFunc("/rpc/chat/delete", srv.handleEmpty) mux.HandleFunc("/rpc/send", srv.handleSend) - mux.HandleFunc("/rpc/read", srv.handleEmpty) + mux.HandleFunc("/rpc/read", srv.handleRead) mux.HandleFunc("/rpc/marked-unread", srv.handleEmpty) mux.HandleFunc("/rpc/dialog/notifications", srv.handleEmpty) mux.HandleFunc("/rpc/typing", srv.handleEmpty) mux.HandleFunc("/rpc/events/ack", srv.handleEventAck) mux.HandleFunc("/ws/events", srv.handleEvents) + mux.HandleFunc("/fixture/presentation", srv.handlePresentation) mux.HandleFunc("/fixture/sent", srv.handleFixtureSent) mux.HandleFunc("/fixture/status", srv.handleFixtureStatus) mux.HandleFunc("/fixture/push-message", srv.handleFixturePushMessage) @@ -154,9 +187,10 @@ func (srv *fixtureServer) register(mux *http.ServeMux) { func (srv *fixtureServer) handleHealth(w http.ResponseWriter, r *http.Request) { srv.writeJSON(w, sidecar.Health{ - OK: true, - Protocol: srv.protocol(), - Status: srv.status().Status, + OK: true, + EventGeneration: srv.generation, + Protocol: srv.protocol(), + Status: srv.status().Status, }) } @@ -230,6 +264,8 @@ func (srv *fixtureServer) handleDialogs(w http.ResponseWriter, r *http.Request) if !decodeRequest(w, r, &req) || !srv.requireLogin(w) { return } + srv.mu.Lock() + defer srv.mu.Unlock() limit := len(srv.dialogs) if req.Limit != nil && int(*req.Limit) < limit { limit = int(*req.Limit) @@ -245,6 +281,8 @@ func (srv *fixtureServer) handleHistory(w http.ResponseWriter, r *http.Request) if !decodeRequest(w, r, &req) || !srv.requireLogin(w) { return } + srv.mu.Lock() + defer srv.mu.Unlock() messages := append([]sidecar.MessageRecord(nil), srv.history[req.ChatID]...) messages = filterHistory(messages, req.BeforeMessageID, req.AfterMessageID) if req.Limit != nil && int(*req.Limit) < len(messages) { @@ -317,6 +355,12 @@ func (srv *fixtureServer) handleSend(w http.ResponseWriter, r *http.Request) { } func (srv *fixtureServer) handleEmpty(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/rpc/marked-unread" || r.URL.Path == "/rpc/dialog/notifications" { + srv.mu.Lock() + srv.sourceCalls[r.URL.Path]++ + srv.mu.Unlock() + _ = srv.saveState() + } if !srv.requireLogin(w) { return } @@ -331,11 +375,11 @@ func (srv *fixtureServer) handleEvents(w http.ResponseWriter, r *http.Request) { afterSequence, _ := strconv.ParseUint(r.URL.Query().Get("after_sequence"), 10, 64) srv.mu.Lock() for _, event := range srv.events { - if event.sequence <= afterSequence { + if event.Sequence <= afterSequence { continue } ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) - err := conn.Write(ctx, websocket.MessageText, event.payload) + err := conn.Write(ctx, websocket.MessageText, event.Payload) cancel() if err != nil { srv.mu.Unlock() @@ -363,7 +407,7 @@ func (srv *fixtureServer) handleEventAck(w http.ResponseWriter, r *http.Request) srv.writeError(w, err) return } - srv.writeJSON(w, sidecar.EventAckResponse{AcknowledgedSequence: request.Sequence}) + srv.writeJSON(w, sidecar.EventAckResponse{Generation: request.Generation, AcknowledgedSequence: request.Sequence}) } func (srv *fixtureServer) handleFixtureSent(w http.ResponseWriter, r *http.Request) { @@ -380,6 +424,10 @@ func (srv *fixtureServer) handleFixtureStatus(w http.ResponseWriter, r *http.Req "event_clients": len(srv.clients), "sent_texts": len(srv.sent), "next_message_id": srv.nextMessageID, + "source_calls": srv.sourceCalls, + "read_states": srv.readStates, + "generation": srv.generation, + "sequence": srv.sequence, }) } @@ -389,6 +437,7 @@ func (srv *fixtureServer) handleFixturePushMessage(w http.ResponseWriter, r *htt SenderID int64 `json:"sender_id"` MessageID int64 `json:"message_id"` Text string `json:"text"` + Timestamp int64 `json:"timestamp"` } if !decodeRequest(w, r, &req) || !srv.requireLogin(w) { return @@ -412,7 +461,7 @@ func (srv *fixtureServer) handleFixturePushMessage(w http.ResponseWriter, r *htt ChatID: req.ChatID, MessageID: req.MessageID, SenderID: req.SenderID, - Timestamp: time.Now().Unix(), + Timestamp: req.Timestamp, Content: sidecar.MessageContent{Type: "text", Text: req.Text}, } srv.history[req.ChatID] = append(srv.history[req.ChatID], record) @@ -464,6 +513,9 @@ func (srv *fixtureServer) requireLogin(w http.ResponseWriter) bool { func (srv *fixtureServer) setLastMessageIDLocked(chatID, messageID int64) { for idx := range srv.dialogs { if srv.dialogs[idx].ChatID == chatID { + if srv.dialogs[idx].LastMessageID != nil && *srv.dialogs[idx].LastMessageID > messageID { + return + } srv.dialogs[idx].LastMessageID = &messageID return } @@ -483,6 +535,7 @@ func (srv *fixtureServer) broadcast(event map[string]any) error { "protocol_version": sidecar.ProtocolVersion, "session_namespace": strconv.FormatInt(accountID, 10), "sequence": sequence, + "generation": srv.generation, "reliability": sidecar.EventLossless, "event": event, } @@ -491,8 +544,11 @@ func (srv *fixtureServer) broadcast(event map[string]any) error { return err } srv.mu.Lock() - srv.events = append(srv.events, fixtureEvent{sequence: sequence, payload: data}) + srv.events = append(srv.events, fixtureEvent{Sequence: sequence, Payload: data}) srv.mu.Unlock() + if err := srv.saveState(); err != nil { + return err + } for _, conn := range clients { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) err := conn.Write(ctx, websocket.MessageText, data) @@ -516,6 +572,29 @@ func (srv *fixtureServer) loadState() error { return fmt.Errorf("parse fixture state: %w", err) } srv.loggedIn = state.LoggedIn + if state.Generation != "" { + srv.generation = state.Generation + } + srv.sequence = state.Sequence + srv.events = state.Events + if state.Dialogs != nil { + srv.dialogs = state.Dialogs + } + if state.Spaces != nil { + srv.spaces = state.Spaces + } + if state.Access != nil { + srv.access = state.Access + } + if state.ReadStates != nil { + srv.readStates = state.ReadStates + } + if state.History != nil { + srv.history = state.History + } + if state.SourceCalls != nil { + srv.sourceCalls = state.SourceCalls + } if state.NextMessageID > 0 { srv.nextMessageID = state.NextMessageID } @@ -523,17 +602,23 @@ func (srv *fixtureServer) loadState() error { } func (srv *fixtureServer) saveState() error { + srv.saveMu.Lock() + defer srv.saveMu.Unlock() srv.mu.Lock() - state := storedState{LoggedIn: srv.loggedIn, NextMessageID: srv.nextMessageID} + state := storedState{LoggedIn: srv.loggedIn, NextMessageID: srv.nextMessageID, Generation: srv.generation, Sequence: srv.sequence, Events: srv.events, Dialogs: srv.dialogs, Spaces: srv.spaces, Access: srv.access, ReadStates: srv.readStates, History: srv.history, SourceCalls: srv.sourceCalls} + data, err := json.Marshal(state) srv.mu.Unlock() - if err := os.MkdirAll(filepath.Dir(srv.statePath), 0o700); err != nil { - return fmt.Errorf("create fixture state dir: %w", err) - } - data, err := json.MarshalIndent(state, "", "\t") if err != nil { return err } - return os.WriteFile(srv.statePath, data, 0o600) + if err = os.MkdirAll(filepath.Dir(srv.statePath), 0o700); err != nil { + return err + } + tmp := srv.statePath + ".tmp" + if err = os.WriteFile(tmp, data, 0o600); err != nil { + return err + } + return os.Rename(tmp, srv.statePath) } func (srv *fixtureServer) writeResult(w http.ResponseWriter, resultType string, data any) { @@ -612,3 +697,147 @@ func stringPtr(value string) *string { func int64Ptr(value int64) *int64 { return &value } + +func boolPtr(v bool) *bool { return &v } +func (srv *fixtureServer) handleAccountState(w http.ResponseWriter, r *http.Request) { + if !srv.requireLogin(w) { + return + } + srv.mu.Lock() + defer srv.mu.Unlock() + srv.writeResult(w, "account_state", sidecar.AccountStateSnapshot{Spaces: srv.spaces, SpaceAccess: srv.access, SpaceAccessComplete: true, Users: srv.users, DeletedChatIDs: []int64{}}) +} +func (srv *fixtureServer) handleChatState(w http.ResponseWriter, r *http.Request) { + var req sidecar.ChatStateRequest + if !decodeRequest(w, r, &req) || !srv.requireLogin(w) { + return + } + srv.mu.Lock() + defer srv.mu.Unlock() + state := sidecar.ChatStateSnapshot{ChatID: req.ChatID, DeletedMessageIDs: []int64{}, ReactionSnapshotMessageIDs: []int64{}} + for _, d := range srv.dialogs { + if d.ChatID == req.ChatID { + copy := d + state.Dialog = © + break + } + } + if read, ok := srv.readStates[req.ChatID]; ok { + state.ReadState = &read + } + srv.writeResult(w, "chat_state", state) +} +func (srv *fixtureServer) handleRead(w http.ResponseWriter, r *http.Request) { + var req sidecar.ReadRequest + if !decodeRequest(w, r, &req) || !srv.requireLogin(w) { + return + } + srv.mu.Lock() + srv.sourceCalls["/rpc/read"]++ + state := srv.readStates[req.ChatID] + state.ChatID = req.ChatID + state.ReadMaxID = req.MaxMessageID + state.MarkedUnread = false + srv.readStates[req.ChatID] = state + srv.mu.Unlock() + _ = srv.saveState() + srv.writeResult(w, "empty", struct{}{}) +} +func (srv *fixtureServer) handlePresentation(w http.ResponseWriter, r *http.Request) { + var req struct { + Action string `json:"action"` + ChatID int64 `json:"chat_id"` + SpaceID int64 `json:"space_id"` + ReadMaxID int64 `json:"read_max_id"` + MarkedUnread bool `json:"marked_unread"` + } + if !decodeRequest(w, r, &req) { + return + } + if req.ChatID == 0 { + req.ChatID = 202 + } + var evt map[string]any + srv.mu.Lock() + switch req.Action { + case "receipt_case": + if req.ChatID <= 0 { + srv.mu.Unlock() + http.Error(w, "receipt case requires a positive chat ID", http.StatusBadRequest) + return + } + for _, d := range srv.dialogs { + if d.ChatID == req.ChatID { + srv.mu.Unlock() + http.Error(w, "receipt case already exists", http.StatusConflict) + return + } + } + srv.dialogs = append(srv.dialogs, sidecar.DialogRecord{ChatID: req.ChatID, Title: stringPtr("Fixture Receipt Order"), Structure: &sidecar.DialogStructure{Kind: "channel", SpaceID: int64Ptr(301)}, SpaceID: int64Ptr(301), LastMessageID: int64Ptr(20)}) + srv.history[req.ChatID] = []sidecar.MessageRecord{ + {ChatID: req.ChatID, MessageID: 20, SenderID: groupUserID, Timestamp: 100, Content: sidecar.MessageContent{Type: "text", Text: "fixture source unread 20"}}, + {ChatID: req.ChatID, MessageID: 10, SenderID: groupUserID, Timestamp: 200, Content: sidecar.MessageContent{Type: "text", Text: "fixture source read 10"}}, + } + srv.readStates[req.ChatID] = sidecar.ReadStateRecord{ChatID: req.ChatID, ReadMaxID: int64Ptr(10), MarkedUnread: true} + evt = map[string]any{"ChatUpserted": map[string]any{"chat_id": req.ChatID}} + case "read": + srv.readStates[req.ChatID] = sidecar.ReadStateRecord{ChatID: req.ChatID, ReadMaxID: int64Ptr(req.ReadMaxID), MarkedUnread: req.MarkedUnread} + evt = map[string]any{"ReadStateChanged": map[string]any{"chat_id": req.ChatID}} + case "move", "detach": + found := false + for i := range srv.dialogs { + if srv.dialogs[i].ChatID == req.ChatID { + d := &srv.dialogs[i] + s := &sidecar.DialogStructure{Kind: "channel"} + if req.Action == "move" { + s.SpaceID = int64Ptr(req.SpaceID) + } + d.Structure = s + d.SpaceID = s.SpaceID + d.PeerUserID = nil + d.ParentChatID = nil + d.ParentMessageID = nil + found = true + break + } + } + if !found { + srv.mu.Unlock() + http.Error(w, "unknown fixture chat", 400) + return + } + evt = map[string]any{"ChatUpserted": map[string]any{"chat_id": req.ChatID}} + case "access_removed", "access_restored": + status := "removed" + if req.Action == "access_restored" { + status = "accessible" + } + for i := range srv.access { + if srv.access[i].SpaceID == req.SpaceID { + srv.access[i].Status = status + } + } + evt = map[string]any{"SpaceMemberChanged": map[string]any{"space_id": req.SpaceID, "user_id": accountID}} + case "rename": + for i := range srv.spaces { + if srv.spaces[i].SpaceID == req.SpaceID { + srv.spaces[i].Name = "Fixture Renamed Workspace" + } + } + evt = map[string]any{"SpaceUpserted": map[string]any{"space_id": req.SpaceID}} + default: + srv.mu.Unlock() + http.Error(w, "unknown fixture action", 400) + return + } + srv.mu.Unlock() + if err := srv.saveState(); err != nil { + srv.writeError(w, err) + return + } + if err := srv.broadcast(evt); err != nil { + srv.writeError(w, err) + return + } + srv.writeJSON(w, map[string]any{"ok": true}) +} diff --git a/scripts/e2efixture/main_test.go b/scripts/e2efixture/main_test.go new file mode 100644 index 0000000..8037ce0 --- /dev/null +++ b/scripts/e2efixture/main_test.go @@ -0,0 +1,97 @@ +package main + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + + "github.com/inline-chat/matrix-inline/pkg/sidecar" +) + +func TestFixturePresentationControlsSurviveRestart(t *testing.T) { + path := filepath.Join(t.TempDir(), "fixture.json") + srv := newFixtureServer(path) + srv.loggedIn = true + mux := http.NewServeMux() + srv.register(mux) + post := func(raw string) { + t.Helper() + w := httptest.NewRecorder() + r := httptest.NewRequest("POST", "/fixture/presentation", strings.NewReader(raw)) + mux.ServeHTTP(w, r) + if w.Code != 200 { + t.Fatalf("control %s: %d %s", raw, w.Code, w.Body.String()) + } + } + post(`{"action":"move","chat_id":202,"space_id":302}`) + post(`{"action":"read","chat_id":202,"read_max_id":10,"marked_unread":true}`) + restarted := newFixtureServer(path) + if err := restarted.loadState(); err != nil { + t.Fatal(err) + } + if restarted.dialogs[1].Structure.SpaceID == nil || *restarted.dialogs[1].Structure.SpaceID != 302 { + t.Fatal("move lost after fixture restart") + } + if r := restarted.readStates[202]; r.ReadMaxID == nil || *r.ReadMaxID != 10 || !r.MarkedUnread { + t.Fatal("read state lost after fixture restart") + } + if restarted.sequence == 0 || restarted.generation == "" { + t.Fatal("fixture replay lineage lost") + } +} +func TestFixtureCompatibilityNeverNeedsLogin(t *testing.T) { + srv := newFixtureServer(filepath.Join(t.TempDir(), "fixture.json")) + mux := http.NewServeMux() + srv.register(mux) + w := httptest.NewRecorder() + mux.ServeHTTP(w, httptest.NewRequest("GET", "/compatibility", nil)) + var p sidecar.ProtocolInfo + if err := json.Unmarshal(w.Body.Bytes(), &p); err != nil || p.ProtocolVersion != sidecar.ProtocolVersion || srv.loggedIn { + t.Fatal("static compatibility contract failed") + } +} + +func TestFixtureAckConfirmsGenerationAndSequence(t *testing.T) { + srv := newFixtureServer(filepath.Join(t.TempDir(), "fixture.json")) + mux := http.NewServeMux() + srv.register(mux) + w := httptest.NewRecorder() + mux.ServeHTTP(w, httptest.NewRequest("POST", "/rpc/events/ack", strings.NewReader(`{"generation":"fixture-generation","sequence":0}`))) + var ack sidecar.EventAckResponse + if err := json.Unmarshal(w.Body.Bytes(), &ack); err != nil || ack.Generation != "fixture-generation" || ack.AcknowledgedSequence != 0 { + t.Fatalf("ack failed confirmation: %+v %v", ack, err) + } +} + +func TestFixtureReceiptCaseIsFreshAndDurable(t *testing.T) { + path := filepath.Join(t.TempDir(), "fixture.json") + srv := newFixtureServer(path) + mux := http.NewServeMux() + srv.register(mux) + request := `{"action":"receipt_case","chat_id":10001}` + w := httptest.NewRecorder() + mux.ServeHTTP(w, httptest.NewRequest("POST", "/fixture/presentation", strings.NewReader(request))) + if w.Code != 200 { + t.Fatalf("receipt case: %d %s", w.Code, w.Body.String()) + } + restarted := newFixtureServer(path) + if err := restarted.loadState(); err != nil { + t.Fatal(err) + } + messages := restarted.history[10001] + if len(messages) != 2 || messages[0].MessageID != 20 || messages[0].Timestamp != 100 || messages[1].MessageID != 10 || messages[1].Timestamp != 200 { + t.Fatalf("receipt order lost: %+v", messages) + } + read := restarted.readStates[10001] + if read.ReadMaxID == nil || *read.ReadMaxID != 10 || !read.MarkedUnread { + t.Fatal("initial source read state lost") + } + w = httptest.NewRecorder() + mux.ServeHTTP(w, httptest.NewRequest("POST", "/fixture/presentation", strings.NewReader(request))) + if w.Code != http.StatusConflict { + t.Fatal("receipt case overwrote existing source chat") + } +} diff --git a/scripts/smoke-local.sh b/scripts/smoke-local.sh index 8b243be..95bae61 100755 --- a/scripts/smoke-local.sh +++ b/scripts/smoke-local.sh @@ -33,18 +33,18 @@ fi echo "==> Waiting for adapter health at ${URL}/health" for _ in $(seq 1 60); do - if curl -fsS "${URL}/health" >/dev/null 2>&1; then + if curl -fsS -H "X-Inline-Protocol-Version: 5" "${URL}/health" >/dev/null 2>&1; then break fi sleep 1 done -curl -fsS "${URL}/health" | jq . +curl -fsS -H "X-Inline-Protocol-Version: 5" "${URL}/health" | jq . echo "==> Adapter status" -curl -fsS "${URL}/status" | jq . +curl -fsS -H "X-Inline-Protocol-Version: 5" "${URL}/status" | jq . echo "==> Adapter resume" -curl -fsS -X POST "${URL}/rpc/resume" | jq . +curl -fsS -H "X-Inline-Protocol-Version: 5" -X POST "${URL}/rpc/resume" | jq . echo "Smoke check passed."