From 8b14b4aeb7625ad47ba4ecca94808c5d82ab4e1f Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Fri, 20 Nov 2026 08:18:49 +0100 Subject: [PATCH 1/5] feat: add LiveKit client spike (webrtc-livekit) Joins a room over LiveKit's signaling with a publisher and a subscriber peer, publishes and subscribes to tracks and exchanges data packets. No simulcast, dynacast or reconnect yet. --- pom.xml | 1 + webrtc-livekit/pom.xml | 83 ++ .../onvoid/webrtc/livekit/CandidateJson.java | 209 ++++ .../onvoid/webrtc/livekit/DataMessage.java | 76 ++ .../dev/onvoid/webrtc/livekit/LiveKit.java | 123 ++ .../webrtc/livekit/LiveKitException.java | 35 + .../onvoid/webrtc/livekit/Participant.java | 105 ++ .../onvoid/webrtc/livekit/Publication.java | 65 ++ .../java/dev/onvoid/webrtc/livekit/Room.java | 136 +++ .../dev/onvoid/webrtc/livekit/RoomImpl.java | 921 +++++++++++++++ .../dev/onvoid/webrtc/livekit/RoomState.java | 33 + .../onvoid/webrtc/livekit/SignalClient.java | 256 +++++ .../onvoid/webrtc/livekit/TrackSource.java | 52 + webrtc-livekit/src/main/java/module-info.java | 30 + .../src/main/proto/livekit_metrics.proto | 108 ++ .../src/main/proto/livekit_models.proto | 1009 +++++++++++++++++ .../src/main/proto/livekit_rtc.proto | 683 +++++++++++ .../src/main/proto/logger/options.proto | 33 + .../webrtc/livekit/RoomIntegrationTest.java | 200 ++++ .../webrtc/livekit/SignalingFormatTests.java | 99 ++ .../dev/onvoid/webrtc/livekit/TestTokens.java | 63 + 21 files changed, 4320 insertions(+) create mode 100644 webrtc-livekit/pom.xml create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/CandidateJson.java create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/DataMessage.java create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/LiveKit.java create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/LiveKitException.java create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Participant.java create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Publication.java create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomState.java create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/SignalClient.java create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/TrackSource.java create mode 100644 webrtc-livekit/src/main/java/module-info.java create mode 100644 webrtc-livekit/src/main/proto/livekit_metrics.proto create mode 100644 webrtc-livekit/src/main/proto/livekit_models.proto create mode 100644 webrtc-livekit/src/main/proto/livekit_rtc.proto create mode 100644 webrtc-livekit/src/main/proto/logger/options.proto create mode 100644 webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/RoomIntegrationTest.java create mode 100644 webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/SignalingFormatTests.java create mode 100644 webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/TestTokens.java diff --git a/pom.xml b/pom.xml index b4750648..3af3fcc8 100644 --- a/pom.xml +++ b/pom.xml @@ -58,6 +58,7 @@ webrtc-media webrtc-swing webrtc-javafx + webrtc-livekit webrtc-examples diff --git a/webrtc-livekit/pom.xml b/webrtc-livekit/pom.xml new file mode 100644 index 00000000..773a0fa9 --- /dev/null +++ b/webrtc-livekit/pom.xml @@ -0,0 +1,83 @@ + + + 4.0.0 + + + dev.onvoid.webrtc + webrtc-java-parent + 1.0.2-SNAPSHOT + + + webrtc-java-livekit + + webrtc-java-livekit + + LiveKit client for webrtc-java. Joins LiveKit rooms, publishes and subscribes to tracks and + exchanges data over the client API. + + + + + 4.36.2 + + v1.54.1 + + + + + + + + + kr.motd.maven + os-maven-plugin + 1.7.1 + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + 0.6.1 + + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} + + + + + compile + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + ${project.build.directory}/natives + + + ${livekit.url} + + + + + + + + + dev.onvoid.webrtc + webrtc-java + ${project.version} + + + com.google.protobuf + protobuf-java + ${protobuf.version} + + + diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/CandidateJson.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/CandidateJson.java new file mode 100644 index 00000000..9f6514a1 --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/CandidateJson.java @@ -0,0 +1,209 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import java.util.HashMap; +import java.util.Map; + +import dev.onvoid.webrtc.RTCIceCandidate; + +/** + * The JSON form of an RTCIceCandidateInit, which LiveKit's trickle messages + * carry as a string. Reads only flat objects of strings, numbers, booleans and + * nulls, which is all a candidate init holds. + */ +final class CandidateJson { + + private CandidateJson() { + } + + static String encode(RTCIceCandidate candidate) { + StringBuilder json = new StringBuilder(candidate.sdp.length() + 64); + json.append("{\"candidate\":"); + quote(json, candidate.sdp); + json.append(",\"sdpMid\":"); + if (candidate.sdpMid == null) { + json.append("null"); + } + else { + quote(json, candidate.sdpMid); + } + json.append(",\"sdpMLineIndex\":").append(candidate.sdpMLineIndex).append('}'); + return json.toString(); + } + + static RTCIceCandidate decode(String json) { + Map fields = parse(json); + Object candidate = fields.get("candidate"); + Object mid = fields.get("sdpMid"); + Object index = fields.get("sdpMLineIndex"); + + if (!(candidate instanceof String)) { + throw new IllegalArgumentException("No candidate in " + json); + } + + return new RTCIceCandidate(mid instanceof String ? (String) mid : null, + index instanceof Number ? ((Number) index).intValue() : 0, (String) candidate); + } + + private static void quote(StringBuilder json, String value) { + json.append('"'); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + switch (c) { + case '"': json.append("\\\""); break; + case '\\': json.append("\\\\"); break; + case '\n': json.append("\\n"); break; + case '\r': json.append("\\r"); break; + case '\t': json.append("\\t"); break; + default: + if (c < 0x20) { + json.append(String.format("\\u%04x", (int) c)); + } + else { + json.append(c); + } + } + } + json.append('"'); + } + + private static Map parse(String json) { + Reader reader = new Reader(json); + Map fields = new HashMap<>(); + + reader.expect('{'); + if (!reader.consume('}')) { + do { + String key = reader.string(); + reader.expect(':'); + fields.put(key, reader.value()); + } + while (reader.consume(',')); + reader.expect('}'); + } + + return fields; + } + + + + private static final class Reader { + + private final String text; + + private int pos; + + + Reader(String text) { + this.text = text; + } + + void skipSpace() { + while (pos < text.length() && Character.isWhitespace(text.charAt(pos))) { + pos++; + } + } + + boolean consume(char c) { + skipSpace(); + if (pos < text.length() && text.charAt(pos) == c) { + pos++; + return true; + } + return false; + } + + void expect(char c) { + if (!consume(c)) { + throw new IllegalArgumentException("Expected '" + c + "' at " + pos + " in " + text); + } + } + + Object value() { + skipSpace(); + if (pos >= text.length()) { + throw new IllegalArgumentException("Unexpected end of " + text); + } + + char c = text.charAt(pos); + if (c == '"') { + return string(); + } + if (text.startsWith("null", pos)) { + pos += 4; + return null; + } + if (text.startsWith("true", pos)) { + pos += 4; + return Boolean.TRUE; + } + if (text.startsWith("false", pos)) { + pos += 5; + return Boolean.FALSE; + } + + int start = pos; + while (pos < text.length() && "+-0123456789.eE".indexOf(text.charAt(pos)) >= 0) { + pos++; + } + if (start == pos) { + throw new IllegalArgumentException("Unexpected '" + c + "' at " + pos + " in " + text); + } + return Double.parseDouble(text.substring(start, pos)); + } + + String string() { + expect('"'); + StringBuilder value = new StringBuilder(); + + while (pos < text.length()) { + char c = text.charAt(pos++); + if (c == '"') { + return value.toString(); + } + if (c != '\\') { + value.append(c); + continue; + } + if (pos >= text.length()) { + break; + } + + char escaped = text.charAt(pos++); + switch (escaped) { + case 'n': value.append('\n'); break; + case 'r': value.append('\r'); break; + case 't': value.append('\t'); break; + case 'b': value.append('\b'); break; + case 'f': value.append('\f'); break; + case 'u': + if (pos + 4 > text.length()) { + throw new IllegalArgumentException("Bad escape in " + text); + } + value.append((char) Integer.parseInt(text.substring(pos, pos + 4), 16)); + pos += 4; + break; + default: value.append(escaped); + } + } + + throw new IllegalArgumentException("Unterminated string in " + text); + } + } + +} diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/DataMessage.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/DataMessage.java new file mode 100644 index 00000000..0f084496 --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/DataMessage.java @@ -0,0 +1,76 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import java.util.Optional; + +/** + * Data another participant published to the room. + */ +public final class DataMessage { + + private final String senderIdentity; + + private final String topic; + + private final byte[] payload; + + private final boolean reliable; + + + DataMessage(String senderIdentity, String topic, byte[] payload, boolean reliable) { + this.senderIdentity = senderIdentity; + this.topic = topic; + this.payload = payload; + this.reliable = reliable; + } + + /** + * @return the identity of the sender, empty for data the server sent. + */ + public String senderIdentity() { + return senderIdentity; + } + + /** + * @return the topic the sender gave, if any. + */ + public Optional topic() { + return Optional.ofNullable(topic); + } + + /** + * @return a copy of the payload. + */ + public byte[] payload() { + return payload.clone(); + } + + /** + * @return true if it came over the reliable channel, false if over the + * lossy one. + */ + public boolean isReliable() { + return reliable; + } + + @Override + public String toString() { + return "DataMessage[from " + senderIdentity + ", topic " + topic + ", " + payload.length + " bytes]"; + } + +} diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/LiveKit.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/LiveKit.java new file mode 100644 index 00000000..27ef79ab --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/LiveKit.java @@ -0,0 +1,123 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import static java.util.Objects.requireNonNull; + +import java.net.URI; +import java.time.Duration; +import java.util.concurrent.CompletableFuture; + +import dev.onvoid.webrtc.client.WebRtc; + +/** + * Joins LiveKit rooms over the peers of a {@link WebRtc} runtime. + *
{@code
+ * Room room = LiveKit.room(rtc, URI.create("wss://live.example.com"), token)
+ *         .connect()
+ *         .get();
+ * }
+ * The token comes from the application's backend, which signs it with the + * server's API secret; this module creates none. + */ +public final class LiveKit { + + /** The version this client reports to the server. */ + static final String VERSION = "0.1.0"; + + /** + * The signaling protocol version this client speaks, with a publisher and + * a subscriber connection. + */ + static final int PROTOCOL = 16; + + private LiveKit() { + } + + /** + * @param rtc the runtime that owns the peers and the tracks. + * @param url the server's URL, such as wss://live.example.com. + * @param token the access token for the room. + * + * @return a builder of the connection. + */ + public static Builder room(WebRtc rtc, URI url, String token) { + return new Builder(rtc, url, token); + } + + + + /** + * Builds the connection to a room. + */ + public static final class Builder { + + private final WebRtc rtc; + + private final URI url; + + private final String token; + + private boolean autoSubscribe = true; + + private Duration timeout = Duration.ofSeconds(10); + + + private Builder(WebRtc rtc, URI url, String token) { + this.rtc = requireNonNull(rtc, "rtc"); + this.url = requireNonNull(url, "url"); + this.token = requireNonNull(token, "token"); + } + + /** + * @param autoSubscribe true, the default, to receive every track + * published in the room. + * + * @return this builder. + */ + public Builder autoSubscribe(boolean autoSubscribe) { + this.autoSubscribe = autoSubscribe; + return this; + } + + /** + * @param timeout how long joining and each request to the server may + * take, ten seconds by default. + * + * @return this builder. + */ + public Builder timeout(Duration timeout) { + requireNonNull(timeout, "timeout"); + if (timeout.isNegative() || timeout.isZero()) { + throw new IllegalArgumentException("The timeout must be positive"); + } + this.timeout = timeout; + return this; + } + + /** + * Opens the signaling connection and joins the room. + * + * @return a future with the room once joined, which fails if the + * server refuses the token or does not answer in time. + */ + public CompletableFuture connect() { + return RoomImpl.connect(rtc, url, token, autoSubscribe, timeout); + } + } + +} diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/LiveKitException.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/LiveKitException.java new file mode 100644 index 00000000..41b0f254 --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/LiveKitException.java @@ -0,0 +1,35 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +/** + * A failure of the LiveKit connection: the server refused the join, a request + * timed out, or the room was left while an operation was pending. + */ +public class LiveKitException extends RuntimeException { + + private static final long serialVersionUID = 1L; + + public LiveKitException(String message) { + super(message); + } + + public LiveKitException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Participant.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Participant.java new file mode 100644 index 00000000..521f68c7 --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Participant.java @@ -0,0 +1,105 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.ParticipantInfo; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.TrackInfo; + +/** + * A participant of a {@link Room}, the local one or a remote one. Its name, + * metadata and attributes follow the updates of the server. + */ +public final class Participant { + + private final String sid; + + private final String identity; + + private volatile String name = ""; + + private volatile String metadata = ""; + + private volatile Map attributes = Collections.emptyMap(); + + private volatile Set trackSids = Collections.emptySet(); + + + Participant(ParticipantInfo info) { + this.sid = info.getSid(); + this.identity = info.getIdentity(); + + update(info); + } + + /** + * @return the server's id of this participant in this session. + */ + public String sid() { + return sid; + } + + /** + * @return the identity the token names. + */ + public String identity() { + return identity; + } + + /** + * @return the display name, empty if none. + */ + public String name() { + return name; + } + + /** + * @return the metadata, empty if none. + */ + public String metadata() { + return metadata; + } + + /** + * @return the attributes, as of the last update. + */ + public Map attributes() { + return attributes; + } + + @Override + public String toString() { + return "Participant[" + identity + ", " + sid + "]"; + } + + void update(ParticipantInfo info) { + name = info.getName(); + metadata = info.getMetadata(); + attributes = Collections.unmodifiableMap(new HashMap<>(info.getAttributesMap())); + trackSids = info.getTracksList().stream().map(TrackInfo::getSid).collect(Collectors.toUnmodifiableSet()); + } + + boolean publishes(String trackSid) { + return trackSids.contains(trackSid); + } + +} diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Publication.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Publication.java new file mode 100644 index 00000000..7ee3d9cc --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Publication.java @@ -0,0 +1,65 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import dev.onvoid.webrtc.client.LocalTrack; + +/** + * A local track the server accepted and that is negotiated to it. + */ +public final class Publication { + + private final String sid; + + private final TrackSource source; + + private final LocalTrack track; + + + Publication(String sid, TrackSource source, LocalTrack track) { + this.sid = sid; + this.source = source; + this.track = track; + } + + /** + * @return the server's id of the track, as other participants see it. + */ + public String sid() { + return sid; + } + + /** + * @return where the track comes from. + */ + public TrackSource source() { + return source; + } + + /** + * @return the published track, which stays the application's. + */ + public LocalTrack track() { + return track; + } + + @Override + public String toString() { + return "Publication[" + sid + ", " + source + ", " + track.id() + "]"; + } + +} diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java new file mode 100644 index 00000000..79fce90f --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java @@ -0,0 +1,136 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +import dev.onvoid.webrtc.client.LocalTrack; +import dev.onvoid.webrtc.client.RemoteTrack; +import dev.onvoid.webrtc.client.Subscription; + +/** + * A joined LiveKit room. It holds two peers of the runtime, one that publishes + * and one that subscribes, and the signaling connection that negotiates them. + *

+ * Handlers run on the room's own thread, one at a time and in the order the + * server sent what they report. A handler that blocks holds up the room, so + * long work belongs on another thread. The futures complete on that thread + * too. + */ +public interface Room extends AutoCloseable { + + /** + * @return the name of the room. + */ + String name(); + + /** + * @return the connection state. + */ + RoomState state(); + + /** + * @return this client's participant. + */ + Participant localParticipant(); + + /** + * @return the other participants, as of the last update of the server. + */ + List remoteParticipants(); + + /** + * Publishes a local track: the server accepts it, then the publisher peer + * negotiates it. + * + * @param track the track, which stays the application's to close. + * @param source where the track comes from. + * + * @return a future with the publication, which fails if the server + * refuses the track or does not answer in time. + */ + CompletableFuture publish(LocalTrack track, TrackSource source); + + /** + * Sends data to every other participant of the room. + * + * @param payload the data. + * @param topic a topic the receivers can tell messages apart by, or + * null. + * @param reliable true to send it ordered and retransmitted, false to + * send it once, for data that is stale soon, such as a + * pointer position. + * + * @return a future that completes once the data is handed to the channel. + */ + CompletableFuture publishData(byte[] payload, String topic, boolean reliable); + + /** + * @param handler gets each participant that joins after this client. + * + * @return removes the handler. + */ + Subscription onParticipantJoined(Consumer handler); + + /** + * @param handler gets each participant that leaves. + * + * @return removes the handler. + */ + Subscription onParticipantLeft(Consumer handler); + + /** + * @param handler gets each remote track this client receives, with the + * participant that publishes it. + * + * @return removes the handler. + */ + Subscription onTrackSubscribed(BiConsumer handler); + + /** + * @param handler gets the data other participants publish. + * + * @return removes the handler. + */ + Subscription onData(Consumer handler); + + /** + * @param handler gets the reason once the room is left, for whatever + * reason, the application's close included. + * + * @return removes the handler. + */ + Subscription onDisconnected(Consumer handler); + + /** + * Leaves the room and closes its peers. The tracks stay the + * application's. + * + * @return a future that completes once the peers are closed. + */ + CompletableFuture closeAsync(); + + /** + * Leaves the room and waits for its peers to close. + */ + @Override + void close(); + +} diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java new file mode 100644 index 00000000..71507815 --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java @@ -0,0 +1,921 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import static java.util.Objects.requireNonNull; + +import java.lang.System.Logger.Level; +import java.net.URI; +import java.net.http.HttpClient; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executor; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.function.BiConsumer; +import java.util.function.Consumer; + +import com.google.protobuf.ByteString; +import com.google.protobuf.InvalidProtocolBufferException; + +import dev.onvoid.webrtc.RTCIceCandidate; +import dev.onvoid.webrtc.RTCSdpType; +import dev.onvoid.webrtc.RTCSessionDescription; +import dev.onvoid.webrtc.client.ChannelOptions; +import dev.onvoid.webrtc.client.DataChannel; +import dev.onvoid.webrtc.client.LocalTrack; +import dev.onvoid.webrtc.client.MediaKind; +import dev.onvoid.webrtc.client.Negotiation; +import dev.onvoid.webrtc.client.Peer; +import dev.onvoid.webrtc.client.PeerOptions; +import dev.onvoid.webrtc.client.RemoteTrack; +import dev.onvoid.webrtc.client.SignalMessage; +import dev.onvoid.webrtc.client.Subscription; +import dev.onvoid.webrtc.client.WebRtc; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.DataPacket; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.DisconnectReason; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.ParticipantInfo; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.TrackType; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.UserPacket; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.AddTrackRequest; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.ICEServer; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.JoinResponse; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.LeaveRequest; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.Ping; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.RequestResponse; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SessionDescription; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SignalRequest; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SignalResponse; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SignalTarget; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.TrackPublishedResponse; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.TrickleRequest; + +/** + * The room: the state LiveKit's signaling describes, and the publisher and + * subscriber peers it negotiates. Everything but the handler lists is + * confined to the room's single thread, which also runs the handlers. + */ +final class RoomImpl implements Room, SignalClient.Handler { + + private static final System.Logger LOG = System.getLogger(Room.class.getName()); + + private static final String RELIABLE_LABEL = "_reliable"; + + private static final String LOSSY_LABEL = "_lossy"; + + private final WebRtc rtc; + + private final Duration timeout; + + private final ScheduledExecutorService thread; + + /** Runs on the room's thread, and drops what comes after the room is gone. */ + private final Executor executor; + + private final SignalClient signal; + + private final CompletableFuture joined = new CompletableFuture<>(); + + private final CompletableFuture closed = new CompletableFuture<>(); + + private final List> joinedHandlers = new CopyOnWriteArrayList<>(); + + private final List> leftHandlers = new CopyOnWriteArrayList<>(); + + private final List> trackHandlers = new CopyOnWriteArrayList<>(); + + private final List> dataHandlers = new CopyOnWriteArrayList<>(); + + private final List> disconnectHandlers = new CopyOnWriteArrayList<>(); + + // Room thread only, unless volatile. + + private volatile RoomState state = RoomState.CONNECTING; + + private volatile String name = ""; + + private volatile Participant local; + + /** By sid, in join order. Written on the room thread, read anywhere. */ + private volatile Map participants = Map.of(); + + /** Tracks that arrived before the participant that publishes them, by participant sid. */ + private final Map> orphanTracks = new HashMap<>(); + + private final Map pendingPublishes = new HashMap<>(); + + private Peer publisher; + + private Peer subscriber; + + private DataChannel reliable; + + private DataChannel lossy; + + private boolean negotiating; + + private boolean renegotiate; + + private boolean publisherNegotiated; + + /** Wait for the offer that is out. */ + private List> negotiationWaiters = new ArrayList<>(); + + /** Wait for the offer after the one that is out. */ + private List> nextNegotiationWaiters = new ArrayList<>(); + + private int offerId; + + private int dataSequence = 1; + + private ScheduledFuture pingTask; + + private long lastPongMillis; + + + private RoomImpl(WebRtc rtc, Duration timeout) { + this.rtc = rtc; + this.timeout = timeout; + this.thread = Executors.newSingleThreadScheduledExecutor(task -> { + Thread t = new Thread(task, "livekit-room"); + t.setDaemon(true); + return t; + }); + this.executor = task -> { + try { + thread.execute(task); + } + catch (RejectedExecutionException e) { + // The room is closed and every future of it completed. + } + }; + this.signal = new SignalClient(executor, this); + } + + static CompletableFuture connect(WebRtc rtc, URI url, String token, boolean autoSubscribe, + Duration timeout) { + URI uri = SignalClient.rtcUri(url, LiveKit.PROTOCOL, autoSubscribe); + RoomImpl room = new RoomImpl(rtc, timeout); + + room.executor.execute(() -> { + ScheduledFuture deadline = room.thread.schedule( + () -> room.failJoin(new LiveKitException("No answer to the join within " + timeout)), + timeout.toMillis(), TimeUnit.MILLISECONDS); + room.joined.whenComplete((r, e) -> deadline.cancel(false)); + + room.signal.open(HttpClient.newHttpClient(), uri, token, timeout) + .whenCompleteAsync((v, e) -> { + if (e != null) { + room.failJoin(e instanceof CompletionException && e.getCause() != null + ? e.getCause() + : e); + } + }, room.executor); + }); + + return room.joined.copy(); + } + + @Override + public String name() { + return name; + } + + @Override + public RoomState state() { + return state; + } + + @Override + public Participant localParticipant() { + return local; + } + + @Override + public List remoteParticipants() { + return List.copyOf(participants.values()); + } + + @Override + public CompletableFuture publish(LocalTrack track, TrackSource source) { + requireNonNull(track, "track"); + requireNonNull(source, "source"); + + CompletableFuture result = new CompletableFuture<>(); + result.orTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS); + + post(() -> startPublish(track, source, result), result); + + return result; + } + + @Override + public CompletableFuture publishData(byte[] payload, String topic, boolean reliable) { + requireNonNull(payload, "payload"); + + byte[] data = payload.clone(); + CompletableFuture result = new CompletableFuture<>(); + result.orTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS); + + post(() -> sendData(data, topic, reliable, result), result); + + return result; + } + + @Override + public Subscription onParticipantJoined(Consumer handler) { + return subscribe(joinedHandlers, handler); + } + + @Override + public Subscription onParticipantLeft(Consumer handler) { + return subscribe(leftHandlers, handler); + } + + @Override + public Subscription onTrackSubscribed(BiConsumer handler) { + return subscribe(trackHandlers, handler); + } + + @Override + public Subscription onData(Consumer handler) { + return subscribe(dataHandlers, handler); + } + + @Override + public Subscription onDisconnected(Consumer handler) { + return subscribe(disconnectHandlers, handler); + } + + @Override + public CompletableFuture closeAsync() { + executor.execute(() -> disconnect("The application left the room", true)); + return closed.copy(); + } + + @Override + public void close() { + try { + closeAsync().get(timeout.toMillis() * 2, TimeUnit.MILLISECONDS); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + catch (Exception e) { + LOG.log(Level.WARNING, "Closing the room did not finish", e); + } + } + + // Signaling, on the room thread. + + @Override + public void onSignal(SignalResponse response) { + if (state == RoomState.DISCONNECTED) { + return; + } + + switch (response.getMessageCase()) { + case JOIN: + onJoin(response.getJoin()); + break; + case ANSWER: + onAnswer(response.getAnswer()); + break; + case OFFER: + onOffer(response.getOffer()); + break; + case TRICKLE: + onTrickle(response.getTrickle()); + break; + case UPDATE: + onParticipants(response.getUpdate().getParticipantsList()); + break; + case TRACK_PUBLISHED: + onTrackPublished(response.getTrackPublished()); + break; + case LEAVE: + LeaveRequest leave = response.getLeave(); + disconnect("The server ended the session (" + leave.getReason() + ", " + leave.getAction() + ")", + false); + break; + case PONG_RESP: + lastPongMillis = System.currentTimeMillis(); + break; + case REQUEST_RESPONSE: + RequestResponse answer = response.getRequestResponse(); + if (answer.getReason() != RequestResponse.Reason.OK) { + LOG.log(Level.WARNING, "The server refused request " + answer.getRequestId() + ": " + + answer.getReason() + " " + answer.getMessage()); + } + break; + default: + LOG.log(Level.DEBUG, () -> "Ignored " + response.getMessageCase()); + } + } + + @Override + public void onSignalClosed(String reason) { + if (!joined.isDone()) { + failJoin(new LiveKitException(reason)); + } + else { + disconnect(reason, false); + } + } + + private void onJoin(JoinResponse join) { + if (joined.isDone()) { + return; + } + + name = join.getRoom().getName(); + local = new Participant(join.getParticipant()); + + Map others = new LinkedHashMap<>(); + for (ParticipantInfo info : join.getOtherParticipantsList()) { + others.put(info.getSid(), new Participant(info)); + } + participants = others; + + try { + publisher = createPeer(SignalTarget.PUBLISHER, join.getIceServersList()); + subscriber = createPeer(SignalTarget.SUBSCRIBER, join.getIceServersList()); + + // The channels go into the first offer of the publisher. + reliable = publisher.createChannel(RELIABLE_LABEL, ChannelOptions.builder().ordered(true).build()); + lossy = publisher.createChannel(LOSSY_LABEL, + ChannelOptions.builder().ordered(false).maxRetransmits(0).build()); + } + catch (RuntimeException e) { + failJoin(e); + return; + } + + receiveData(reliable); + receiveData(lossy); + + subscriber.onTrack(track -> executor.execute(() -> onRemoteTrack(track))); + subscriber.onChannel(channel -> executor.execute(() -> receiveData(channel))); + + startPing(join.getPingInterval(), join.getPingTimeout()); + + state = RoomState.CONNECTED; + + if (!join.getSubscriberPrimary() || join.getFastPublish()) { + negotiatePublisher(); + } + + joined.complete(this); + } + + private Peer createPeer(SignalTarget target, List servers) { + PeerOptions.Builder options = PeerOptions.builder() + .name("livekit-" + target.name().toLowerCase()) + .negotiation(Negotiation.MANUAL) + .signaling(message -> { + // A manual peer signals only its candidates. + if (message.type() == SignalMessage.Type.CANDIDATE) { + RTCIceCandidate candidate = message.candidate(); + executor.execute(() -> sendTrickle(candidate, target)); + } + }); + + for (ICEServer server : servers) { + for (String url : server.getUrlsList()) { + if (server.getUsername().isEmpty()) { + options.iceServer(url); + } + else { + options.iceServer(url, server.getUsername(), server.getCredential()); + } + } + } + + Peer peer = rtc.createPeer(options.build()); + peer.onError(e -> LOG.log(Level.WARNING, "The " + target + " peer failed", e)); + + return peer; + } + + private void sendTrickle(RTCIceCandidate candidate, SignalTarget target) { + if (state == RoomState.DISCONNECTED) { + return; + } + + signal.send(SignalRequest.newBuilder() + .setTrickle(TrickleRequest.newBuilder() + .setCandidateInit(CandidateJson.encode(candidate)) + .setTarget(target)) + .build()); + } + + private void onTrickle(TrickleRequest trickle) { + Peer peer = trickle.getTarget() == SignalTarget.PUBLISHER ? publisher : subscriber; + if (peer == null) { + return; + } + + RTCIceCandidate candidate; + try { + candidate = CandidateJson.decode(trickle.getCandidateInit()); + } + catch (IllegalArgumentException e) { + LOG.log(Level.WARNING, "Dropped a candidate that does not parse", e); + return; + } + + peer.negotiator().addCandidate(candidate).exceptionally(e -> { + LOG.log(Level.DEBUG, () -> "A remote candidate was refused: " + e); + return null; + }); + } + + // The publisher offers, the server answers. + + private CompletableFuture negotiatePublisher() { + CompletableFuture negotiated = new CompletableFuture<>(); + + if (negotiating) { + // The offer that is out may predate what the caller changed. + nextNegotiationWaiters.add(negotiated); + renegotiate = true; + } + else { + negotiationWaiters.add(negotiated); + sendOffer(); + } + + return negotiated; + } + + private void sendOffer() { + negotiating = true; + + publisher.negotiator().createOffer().whenCompleteAsync((offer, error) -> { + if (error != null) { + negotiationDone(error); + return; + } + if (state == RoomState.DISCONNECTED) { + return; + } + + signal.send(SignalRequest.newBuilder() + .setOffer(SessionDescription.newBuilder() + .setType("offer") + .setSdp(offer.sdp) + .setId(++offerId)) + .build()); + }, executor); + } + + private void onAnswer(SessionDescription answer) { + if (publisher == null || !negotiating) { + return; + } + + publisher.negotiator().applyAnswer(new RTCSessionDescription(RTCSdpType.ANSWER, answer.getSdp())) + .whenCompleteAsync((v, error) -> negotiationDone(error), executor); + } + + private void negotiationDone(Throwable error) { + negotiating = false; + + if (error == null) { + publisherNegotiated = true; + } + else { + LOG.log(Level.WARNING, "Negotiating the publisher failed", error); + } + + List> waiters = negotiationWaiters; + negotiationWaiters = nextNegotiationWaiters; + nextNegotiationWaiters = new ArrayList<>(); + + if (renegotiate && state != RoomState.DISCONNECTED) { + renegotiate = false; + sendOffer(); + } + + for (CompletableFuture waiter : waiters) { + if (error == null) { + waiter.complete(null); + } + else { + waiter.completeExceptionally(error); + } + } + } + + // The server offers, the subscriber answers. + + private void onOffer(SessionDescription offer) { + if (subscriber == null) { + return; + } + + subscriber.negotiator().answer(new RTCSessionDescription(RTCSdpType.OFFER, offer.getSdp())) + .whenCompleteAsync((answer, error) -> { + if (error != null) { + LOG.log(Level.WARNING, "Answering the server's offer failed", error); + return; + } + if (state == RoomState.DISCONNECTED) { + return; + } + + signal.send(SignalRequest.newBuilder() + .setAnswer(SessionDescription.newBuilder() + .setType("answer") + .setSdp(answer.sdp) + .setId(offer.getId())) + .build()); + }, executor); + } + + // Publishing. + + private void startPublish(LocalTrack track, TrackSource source, CompletableFuture result) { + if (state != RoomState.CONNECTED) { + result.completeExceptionally(new LiveKitException("The room is " + state)); + return; + } + + // The server knows the track by this id until it names its own. + String cid = track.id(); + if (pendingPublishes.containsKey(cid)) { + result.completeExceptionally(new LiveKitException("The track " + cid + " is being published")); + return; + } + + pendingPublishes.put(cid, new PendingPublish(track, source, result)); + result.whenCompleteAsync((p, e) -> pendingPublishes.remove(cid), executor); + + signal.send(SignalRequest.newBuilder() + .setAddTrack(AddTrackRequest.newBuilder() + .setCid(cid) + .setName(track.id()) + .setType(track.kind() == MediaKind.VIDEO ? TrackType.VIDEO : TrackType.AUDIO) + .setSource(source.proto())) + .build()); + } + + private void onTrackPublished(TrackPublishedResponse published) { + PendingPublish pending = pendingPublishes.remove(published.getCid()); + if (pending == null || pending.result.isDone()) { + return; + } + + try { + publisher.addTrack(pending.track); + } + catch (RuntimeException e) { + pending.result.completeExceptionally(e); + return; + } + + Publication publication = new Publication(published.getTrack().getSid(), pending.source, pending.track); + + negotiatePublisher().whenComplete((v, error) -> { + if (error != null) { + pending.result.completeExceptionally(error); + } + else { + pending.result.complete(publication); + } + }); + } + + // Data. + + private void sendData(byte[] payload, String topic, boolean reliable, CompletableFuture result) { + if (state != RoomState.CONNECTED) { + result.completeExceptionally(new LiveKitException("The room is " + state)); + return; + } + + UserPacket.Builder user = UserPacket.newBuilder().setPayload(ByteString.copyFrom(payload)); + if (topic != null) { + user.setTopic(topic); + } + + DataPacket.Builder packet = DataPacket.newBuilder() + .setParticipantIdentity(local.identity()) + .setUser(user); + if (reliable) { + packet.setSequence(dataSequence++); + } + byte[] bytes = packet.build().toByteArray(); + + DataChannel channel = reliable ? this.reliable : this.lossy; + CompletableFuture ready = publisherNegotiated || negotiating + ? CompletableFuture.completedFuture(null) + : negotiatePublisher(); + + ready.thenCompose(v -> channel.whenOpen()) + .thenCompose(open -> open.send(bytes)) + .whenComplete((v, error) -> { + if (error != null) { + result.completeExceptionally(error); + } + else { + result.complete(null); + } + }); + } + + private void receiveData(DataChannel channel) { + boolean reliableChannel = !LOSSY_LABEL.equals(channel.label()); + + channel.onMessage(message -> { + if (message.isText()) { + return; + } + byte[] bytes = message.bytes(); + executor.execute(() -> onDataPacket(bytes, reliableChannel)); + }); + } + + private void onDataPacket(byte[] bytes, boolean reliableChannel) { + DataPacket packet; + try { + packet = DataPacket.parseFrom(bytes); + } + catch (InvalidProtocolBufferException e) { + LOG.log(Level.WARNING, "Dropped a data packet that does not parse", e); + return; + } + + if (!packet.hasUser()) { + return; + } + + UserPacket user = packet.getUser(); + DataMessage message = new DataMessage(packet.getParticipantIdentity(), + user.hasTopic() ? user.getTopic() : null, user.getPayload().toByteArray(), reliableChannel); + + for (Consumer handler : dataHandlers) { + call(() -> handler.accept(message)); + } + } + + // Participants and their tracks. + + private void onParticipants(List infos) { + Map next = new LinkedHashMap<>(participants); + List joinedNow = new ArrayList<>(); + List leftNow = new ArrayList<>(); + + for (ParticipantInfo info : infos) { + if (local != null && info.getSid().equals(local.sid())) { + local.update(info); + continue; + } + + Participant participant = next.get(info.getSid()); + + if (info.getState() == ParticipantInfo.State.DISCONNECTED) { + if (participant != null) { + next.remove(info.getSid()); + orphanTracks.remove(info.getSid()); + leftNow.add(participant); + } + } + else if (participant == null) { + participant = new Participant(info); + next.put(info.getSid(), participant); + joinedNow.add(participant); + } + else { + participant.update(info); + } + } + + participants = next; + + for (Participant participant : joinedNow) { + for (Consumer handler : joinedHandlers) { + call(() -> handler.accept(participant)); + } + + List tracks = orphanTracks.remove(participant.sid()); + if (tracks != null) { + tracks.forEach(track -> trackSubscribed(participant, track)); + } + } + for (Participant participant : leftNow) { + for (Consumer handler : leftHandlers) { + call(() -> handler.accept(participant)); + } + } + } + + private void onRemoteTrack(RemoteTrack track) { + if (state == RoomState.DISCONNECTED) { + return; + } + + // LiveKit's stream ids are "|". + String participantSid = null; + for (String streamId : track.streamIds()) { + int separator = streamId.indexOf('|'); + if (separator > 0) { + participantSid = streamId.substring(0, separator); + break; + } + } + + Participant participant = participantSid == null ? null : participants.get(participantSid); + if (participant == null) { + for (Participant candidate : participants.values()) { + if (candidate.publishes(track.id())) { + participant = candidate; + break; + } + } + } + + if (participant == null) { + if (participantSid != null) { + orphanTracks.computeIfAbsent(participantSid, sid -> new ArrayList<>()).add(track); + } + else { + LOG.log(Level.WARNING, "Dropped a track of an unknown participant: " + track.id()); + } + return; + } + + trackSubscribed(participant, track); + } + + private void trackSubscribed(Participant participant, RemoteTrack track) { + for (BiConsumer handler : trackHandlers) { + call(() -> handler.accept(participant, track)); + } + } + + // Keepalive. + + private void startPing(int intervalSeconds, int timeoutSeconds) { + if (intervalSeconds <= 0) { + return; + } + + long timeoutMillis = TimeUnit.SECONDS.toMillis(timeoutSeconds); + lastPongMillis = System.currentTimeMillis(); + + pingTask = thread.scheduleAtFixedRate(() -> { + long now = System.currentTimeMillis(); + + if (timeoutMillis > 0 && now - lastPongMillis > timeoutMillis) { + disconnect("The server did not answer pings for " + timeoutSeconds + " s", false); + return; + } + + signal.send(SignalRequest.newBuilder() + .setPingReq(Ping.newBuilder().setTimestamp(now)) + .build()); + }, intervalSeconds, intervalSeconds, TimeUnit.SECONDS); + } + + // Leaving. + + private void failJoin(Throwable error) { + if (joined.isDone()) { + return; + } + + joined.completeExceptionally(error); + disconnect(String.valueOf(error.getMessage()), false); + } + + private void disconnect(String reason, boolean sendLeave) { + if (state == RoomState.DISCONNECTED) { + return; + } + state = RoomState.DISCONNECTED; + + if (pingTask != null) { + pingTask.cancel(false); + } + if (sendLeave) { + signal.send(SignalRequest.newBuilder() + .setLeave(LeaveRequest.newBuilder() + .setReason(DisconnectReason.CLIENT_INITIATED) + .setAction(LeaveRequest.Action.DISCONNECT)) + .build()); + } + signal.close(); + + LiveKitException error = new LiveKitException("The room was left: " + reason); + joined.completeExceptionally(error); + + List publishes = new ArrayList<>(pendingPublishes.values()); + pendingPublishes.clear(); + publishes.forEach(pending -> pending.result.completeExceptionally(error)); + + List> waiters = new ArrayList<>(negotiationWaiters); + waiters.addAll(nextNegotiationWaiters); + negotiationWaiters.clear(); + nextNegotiationWaiters.clear(); + waiters.forEach(waiter -> waiter.completeExceptionally(error)); + + orphanTracks.clear(); + + CompletableFuture publisherClosed = publisher != null + ? publisher.closeAsync() + : CompletableFuture.completedFuture(null); + CompletableFuture subscriberClosed = subscriber != null + ? subscriber.closeAsync() + : CompletableFuture.completedFuture(null); + + CompletableFuture.allOf(publisherClosed, subscriberClosed) + .orTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS) + .whenComplete((v, closeError) -> executor.execute(() -> { + for (Consumer handler : disconnectHandlers) { + call(() -> handler.accept(reason)); + } + + if (closeError != null) { + closed.completeExceptionally(closeError instanceof TimeoutException + ? new LiveKitException("The peers did not close in time", closeError) + : closeError); + } + else { + closed.complete(null); + } + + thread.shutdown(); + })); + } + + // Helpers. + + private void post(Runnable task, CompletableFuture future) { + try { + thread.execute(task); + } + catch (RejectedExecutionException e) { + future.completeExceptionally(new LiveKitException("The room is closed")); + } + } + + private static Subscription subscribe(List handlers, T handler) { + requireNonNull(handler, "handler"); + handlers.add(handler); + return () -> handlers.remove(handler); + } + + private static void call(Runnable handler) { + try { + handler.run(); + } + catch (RuntimeException e) { + LOG.log(Level.WARNING, "A room handler failed", e); + } + } + + + + private static final class PendingPublish { + + final LocalTrack track; + + final TrackSource source; + + final CompletableFuture result; + + + PendingPublish(LocalTrack track, TrackSource source, CompletableFuture result) { + this.track = track; + this.source = source; + this.result = result; + } + } + +} diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomState.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomState.java new file mode 100644 index 00000000..f732bf9e --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomState.java @@ -0,0 +1,33 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +/** + * The connection state of a {@link Room}. + */ +public enum RoomState { + + /** The signaling connection is being opened and the room joined. */ + CONNECTING, + + /** The room is joined. */ + CONNECTED, + + /** The room is left, by the application, the server or a failure. Final. */ + DISCONNECTED + +} diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/SignalClient.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/SignalClient.java new file mode 100644 index 00000000..b5426147 --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/SignalClient.java @@ -0,0 +1,256 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import java.io.ByteArrayOutputStream; +import java.net.URI; +import java.net.URLEncoder; +import java.net.http.HttpClient; +import java.net.http.WebSocket; +import java.net.http.WebSocketHandshakeException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionStage; +import java.util.concurrent.Executor; +import java.util.concurrent.TimeUnit; + +import com.google.protobuf.InvalidProtocolBufferException; + +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SignalRequest; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SignalResponse; + +/** + * The signaling connection to a LiveKit server: protobuf messages in binary + * WebSocket frames. Incoming messages and the end of the connection are handed + * to the room's executor, in order. + */ +final class SignalClient implements WebSocket.Listener { + + interface Handler { + + /** A message of the server, on the room's executor. */ + void onSignal(SignalResponse response); + + /** The connection ended without the room closing it, on the room's executor. */ + void onSignalClosed(String reason); + + } + + + private static final System.Logger LOG = System.getLogger(SignalClient.class.getName()); + + private static final Duration CLOSE_TIMEOUT = Duration.ofSeconds(2); + + private final Executor executor; + + private final Handler handler; + + /** Frames of the message being received. Only on WebSocket's listener thread. */ + private final ByteArrayOutputStream frames = new ByteArrayOutputStream(); + + private volatile WebSocket socket; + + /** Sends one at a time, as WebSocket requires. Only on the room's executor. */ + private CompletableFuture sending = CompletableFuture.completedFuture(null); + + private volatile boolean closed; + + + SignalClient(Executor executor, Handler handler) { + this.executor = executor; + this.handler = handler; + } + + /** + * The URL of the signaling endpoint for a server URL as LiveKit hands it + * out, such as wss://example.com, with https and http taken as wss and ws. + */ + static URI rtcUri(URI url, int protocol, boolean autoSubscribe) { + String scheme = url.getScheme() == null ? "" : url.getScheme().toLowerCase(Locale.ROOT); + switch (scheme) { + case "https": scheme = "wss"; break; + case "http": scheme = "ws"; break; + case "wss": + case "ws": break; + default: + throw new IllegalArgumentException("Not a LiveKit URL: " + url); + } + + String path = url.getRawPath() == null ? "" : url.getRawPath(); + if (path.endsWith("/")) { + path = path.substring(0, path.length() - 1); + } + + String query = "protocol=" + protocol + + "&sdk=" + encode("java") + + "&version=" + encode(LiveKit.VERSION) + + "&os=" + encode(System.getProperty("os.name", "")) + + "&auto_subscribe=" + (autoSubscribe ? 1 : 0); + + String authority = url.getRawAuthority(); + return URI.create(scheme + "://" + authority + path + "/rtc?" + query); + } + + /** + * Opens the connection. The token goes in the Authorization header, not in + * the URL, which proxies and servers may log. + */ + CompletableFuture open(HttpClient client, URI uri, String token, Duration timeout) { + return client.newWebSocketBuilder() + .header("Authorization", "Bearer " + token) + .connectTimeout(timeout) + .buildAsync(uri, this) + .handle((webSocket, error) -> { + if (error != null) { + throw new LiveKitException("Could not open the signaling connection to " + + uri.getHost() + describe(error), error); + } + if (closed) { + webSocket.abort(); + throw new LiveKitException("The room was closed"); + } + socket = webSocket; + return null; + }); + } + + /** + * Sends a request after those sent before. Only on the room's executor. + */ + void send(SignalRequest request) { + WebSocket webSocket = socket; + + if (closed || webSocket == null) { + return; + } + + ByteBuffer data = ByteBuffer.wrap(request.toByteArray()); + + sending = sending.handle((r, e) -> null) + .thenCompose(v -> webSocket.sendBinary(data, true)) + .whenComplete((r, e) -> { + if (e != null && !closed) { + LOG.log(System.Logger.Level.WARNING, "Sending " + request.getMessageCase() + " failed", e); + } + }); + } + + /** + * Closes the connection once what was sent before is out, and cuts it off + * if the server does not answer the close. Only on the room's executor. + */ + void close() { + if (closed) { + return; + } + closed = true; + + WebSocket webSocket = socket; + if (webSocket == null) { + return; + } + + sending.handle((r, e) -> null) + .thenCompose(v -> webSocket.sendClose(WebSocket.NORMAL_CLOSURE, "")) + .whenComplete((r, e) -> CompletableFuture.delayedExecutor(CLOSE_TIMEOUT.toMillis(), + TimeUnit.MILLISECONDS).execute(webSocket::abort)); + } + + @Override + public void onOpen(WebSocket webSocket) { + webSocket.request(1); + } + + @Override + public CompletionStage onBinary(WebSocket webSocket, ByteBuffer data, boolean last) { + byte[] chunk = new byte[data.remaining()]; + data.get(chunk); + frames.write(chunk, 0, chunk.length); + + if (last) { + byte[] message = frames.toByteArray(); + frames.reset(); + + try { + SignalResponse response = SignalResponse.parseFrom(message); + post(() -> handler.onSignal(response)); + } + catch (InvalidProtocolBufferException e) { + LOG.log(System.Logger.Level.WARNING, "Dropped a signal message that does not parse", e); + } + } + + webSocket.request(1); + return null; + } + + @Override + public CompletionStage onText(WebSocket webSocket, CharSequence data, boolean last) { + webSocket.request(1); + return null; + } + + @Override + public CompletionStage onClose(WebSocket webSocket, int statusCode, String reason) { + if (!closed) { + post(() -> handler.onSignalClosed("The server closed the signaling connection (" + statusCode + + (reason.isEmpty() ? "" : ", " + reason) + ")")); + } + return null; + } + + @Override + public void onError(WebSocket webSocket, Throwable error) { + if (!closed) { + post(() -> handler.onSignalClosed("The signaling connection failed: " + error)); + } + } + + private void post(Runnable task) { + try { + executor.execute(task); + } + catch (RuntimeException e) { + // The room is gone. + } + } + + private static String describe(Throwable error) { + Throwable cause = error; + while (cause.getCause() != null && !(cause instanceof WebSocketHandshakeException)) { + cause = cause.getCause(); + } + if (cause instanceof WebSocketHandshakeException) { + var response = ((WebSocketHandshakeException) cause).getResponse(); + Object body = response.body(); + if (body instanceof byte[]) { + body = new String((byte[]) body, StandardCharsets.UTF_8); + } + String text = body == null ? "" : body.toString().trim(); + return ": HTTP " + response.statusCode() + (text.isEmpty() ? "" : " " + text); + } + return ": " + cause; + } + + private static String encode(String value) { + return URLEncoder.encode(value, StandardCharsets.UTF_8); + } + +} diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/TrackSource.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/TrackSource.java new file mode 100644 index 00000000..d0ea35c3 --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/TrackSource.java @@ -0,0 +1,52 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels; + +/** + * Where a published track comes from. Other participants see it, and the + * server checks it against the sources the token may publish. + */ +public enum TrackSource { + + UNKNOWN(LivekitModels.TrackSource.UNKNOWN), + CAMERA(LivekitModels.TrackSource.CAMERA), + MICROPHONE(LivekitModels.TrackSource.MICROPHONE), + SCREEN_SHARE(LivekitModels.TrackSource.SCREEN_SHARE), + SCREEN_SHARE_AUDIO(LivekitModels.TrackSource.SCREEN_SHARE_AUDIO); + + private final LivekitModels.TrackSource proto; + + TrackSource(LivekitModels.TrackSource proto) { + this.proto = proto; + } + + LivekitModels.TrackSource proto() { + return proto; + } + + static TrackSource of(LivekitModels.TrackSource proto) { + for (TrackSource source : values()) { + if (source.proto == proto) { + return source; + } + } + return UNKNOWN; + } + +} diff --git a/webrtc-livekit/src/main/java/module-info.java b/webrtc-livekit/src/main/java/module-info.java new file mode 100644 index 00000000..4fd40ab9 --- /dev/null +++ b/webrtc-livekit/src/main/java/module-info.java @@ -0,0 +1,30 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module webrtc.java.livekit { + + requires transitive webrtc.java; + requires java.net.http; + requires com.google.protobuf; + + exports dev.onvoid.webrtc.livekit; + + // The generated protocol classes stay internal. Protobuf reaches them by + // reflection for its descriptor-based operations, such as toString(). + opens dev.onvoid.webrtc.livekit.internal.proto to com.google.protobuf; + opens dev.onvoid.webrtc.livekit.internal.proto.logger to com.google.protobuf; + +} diff --git a/webrtc-livekit/src/main/proto/livekit_metrics.proto b/webrtc-livekit/src/main/proto/livekit_metrics.proto new file mode 100644 index 00000000..09c2dfde --- /dev/null +++ b/webrtc-livekit/src/main/proto/livekit_metrics.proto @@ -0,0 +1,108 @@ +syntax = "proto3"; + +package livekit; +option java_package = "dev.onvoid.webrtc.livekit.internal.proto"; +option go_package = "github.com/livekit/protocol/livekit"; +option csharp_namespace = "LiveKit.Proto"; +option ruby_package = "LiveKit::Proto"; + +import "google/protobuf/timestamp.proto"; +import "logger/options.proto"; + + +/* + Protocol used to record metrics for a specific session. + + Clients send their timestamp in their own monotonically increasing time (e.g `performance.now` on JS). + These timestamps are then augmented by the SFU to its time base. + + A metric can be linked to a specific track by setting `track_sid`. +*/ + + +// index from [0: MAX_LABEL_PREDEFINED_MAX_VALUE) are for predefined labels (`MetricLabel`) +enum MetricLabel { + AGENTS_LLM_TTFT = 0; // time to first token from LLM + AGENTS_STT_TTFT = 1; // time to final transcription + AGENTS_TTS_TTFB = 2; // time to first byte + + CLIENT_VIDEO_SUBSCRIBER_FREEZE_COUNT = 3; // Number of video freezes + CLIENT_VIDEO_SUBSCRIBER_TOTAL_FREEZE_DURATION = 4; // total duration of freezes + CLIENT_VIDEO_SUBSCRIBER_PAUSE_COUNT = 5; // number of video pauses + CLIENT_VIDEO_SUBSCRIBER_TOTAL_PAUSES_DURATION = 6; // total duration of pauses + CLIENT_AUDIO_SUBSCRIBER_CONCEALED_SAMPLES = 7; // number of concealed (synthesized) audio samples + CLIENT_AUDIO_SUBSCRIBER_SILENT_CONCEALED_SAMPLES = 8; // number of silent concealed samples + CLIENT_AUDIO_SUBSCRIBER_CONCEALMENT_EVENTS = 9; // number of concealment events + CLIENT_AUDIO_SUBSCRIBER_INTERRUPTION_COUNT = 10; // number of interruptions + CLIENT_AUDIO_SUBSCRIBER_TOTAL_INTERRUPTION_DURATION = 11; // total duration of interruptions + CLIENT_SUBSCRIBER_JITTER_BUFFER_DELAY = 12; // total time spent in jitter buffer + CLIENT_SUBSCRIBER_JITTER_BUFFER_EMITTED_COUNT = 13; // total time spent in jitter buffer + CLIENT_VIDEO_PUBLISHER_QUALITY_LIMITATION_DURATION_BANDWIDTH = 14; // total duration spent in bandwidth quality limitation + CLIENT_VIDEO_PUBLISHER_QUALITY_LIMITATION_DURATION_CPU = 15; // total duration spent in cpu quality limitation + CLIENT_VIDEO_PUBLISHER_QUALITY_LIMITATION_DURATION_OTHER = 16; // total duration spent in other quality limitation + + PUBLISHER_RTT = 17; // Publisher RTT (participant -> server) + SERVER_MESH_RTT = 18; // RTT between publisher node and subscriber node (could involve intermedia node(s)) + SUBSCRIBER_RTT = 19; // Subscribe RTT (server -> participant) + + METRIC_LABEL_PREDEFINED_MAX_VALUE = 4096; +} + +message MetricsBatch { + int64 timestamp_ms = 1; // time at which this batch is sent based on a monotonic clock (millisecond resolution) + google.protobuf.Timestamp normalized_timestamp = 2; + // To avoid repeating string values, we store them in a separate list and reference them by index + // This is useful for storing participant identities, track names, etc. + // There is also a predefined list of labels that can be used to reference common metrics. + // They have reserved indices from 0 to (METRIC_LABEL_PREDEFINED_MAX_VALUE - 1). + // Indexes pointing at str_data should start from METRIC_LABEL_PREDEFINED_MAX_VALUE, + // such that str_data[0] == index of METRIC_LABEL_PREDEFINED_MAX_VALUE. + repeated string str_data = 3; + repeated TimeSeriesMetric time_series = 4; + repeated EventMetric events = 5; +} + +message TimeSeriesMetric { + // Metric name e.g "speech_probablity". The string value is not directly stored in the message, but referenced by index + // in the `str_data` field of `MetricsBatch` + uint32 label = 1; + uint32 participant_identity = 2; // index into `str_data` + uint32 track_sid = 3; // index into `str_data` + repeated MetricSample samples = 4; + uint32 rid = 5; // index into 'str_data' +} + +message MetricSample { + int64 timestamp_ms = 1; // time of metric based on a monotonic clock (in milliseconds) + google.protobuf.Timestamp normalized_timestamp = 2; + float value = 3; +} + +message EventMetric { + uint32 label = 1; + uint32 participant_identity = 2; // index into `str_data` + uint32 track_sid = 3; // index into `str_data` + int64 start_timestamp_ms = 4; // start time of event based on a monotonic clock (in milliseconds) + optional int64 end_timestamp_ms = 5; // end time of event based on a monotonic clock (in milliseconds), if needed + google.protobuf.Timestamp normalized_start_timestamp = 6; + optional google.protobuf.Timestamp normalized_end_timestamp = 7; + string metadata = 8; + uint32 rid = 9; // index into 'str_data' +} + +message MetricsRecordingHeader { + reserved 2; // was observability field, removed in #1294 + string room_id = 1 [(logger.name) = "roomID"]; + uint64 duration = 3; // milliseconds + google.protobuf.Timestamp start_time = 4; + map room_tags = 5; + string room_name = 6; + google.protobuf.Timestamp room_start_time = 7; + string job_id = 8; + bool simulated = 9; // session is a simulation; the collector skips PII redaction for it unless redaction_enabled is set + bool redaction_enabled = 10; // force PII redaction on for this session (only ever enables, never disables) + // JSON object of session facts for agent evals, at most 16 KiB. Its schema is + // evalschema/session-summary.schema.json in livekit/cloud-protocol. The collector + // stores it without reading individual keys. + string session_summary = 11; +} diff --git a/webrtc-livekit/src/main/proto/livekit_models.proto b/webrtc-livekit/src/main/proto/livekit_models.proto new file mode 100644 index 00000000..99484cdf --- /dev/null +++ b/webrtc-livekit/src/main/proto/livekit_models.proto @@ -0,0 +1,1009 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package livekit; +option java_package = "dev.onvoid.webrtc.livekit.internal.proto"; + +import "google/protobuf/timestamp.proto"; +import "livekit_metrics.proto"; +import "logger/options.proto"; + +option csharp_namespace = "LiveKit.Proto"; +option go_package = "github.com/livekit/protocol/livekit"; +option ruby_package = "LiveKit::Proto"; + +message Pagination { + string after_id = 1 [(logger.name) = "afterID"]; // list entities which IDs are greater + int32 limit = 2; +} + +message TokenPagination { + string token = 1; +} + +// ListUpdate is used for updated APIs where 'repeated string' field is modified. +message ListUpdate { + repeated string set = 1; // set the field to a new list + repeated string add = 2; // append items to a list, avoiding duplicates + repeated string remove = 3; // delete items from a list + bool clear = 4; // sets the list to an empty list +} + +message Room { + string sid = 1; + string name = 2; + uint32 empty_timeout = 3; + uint32 departure_timeout = 14; + uint32 max_participants = 4; + int64 creation_time = 5; + int64 creation_time_ms = 15; + string turn_password = 6 [(logger.sensitivity) = SENSITIVITY_SECRET]; + repeated Codec enabled_codecs = 7; + string metadata = 8 [ + (logger.sensitivity) = SENSITIVITY_PII, + (logger.redact_format) = "" + ]; + uint32 num_participants = 9; + uint32 num_publishers = 11; + bool active_recording = 10; + TimedVersion version = 13; + + // NEXT-ID: 16 +} + +message Codec { + string mime = 1; + string fmtp_line = 2; +} + +enum AudioCodec { + DEFAULT_AC = 0; + OPUS = 1; + AAC = 2; + AC_MP3 = 3; +} + +enum VideoCodec { + DEFAULT_VC = 0; + H264_BASELINE = 1; + H264_MAIN = 2; + H264_HIGH = 3; + VP8 = 4; +} + +enum ImageCodec { + IC_DEFAULT = 0; + IC_JPEG = 1; +} + +// Policy for publisher to handle subscribers that are unable to support the primary codec of a track +enum BackupCodecPolicy { + // default behavior, the track prefer to regress to backup codec and all subscribers will receive the backup codec, + // the sfu will try to regress codec if possible but not assured. + PREFER_REGRESSION = 0; + // encoding/send the primary and backup codec simultaneously + SIMULCAST = 1; + // force the track to regress to backup codec, this option can be used in video conference or the publisher has limited bandwidth/encoding power + REGRESSION = 2; +} + +message PlayoutDelay { + bool enabled = 1; + uint32 min = 2; + uint32 max = 3; +} + +message ParticipantPermission { + // allow participant to subscribe to other tracks in the room + bool can_subscribe = 1; + // allow participant to publish new tracks to room + bool can_publish = 2; + // allow participant to publish data + bool can_publish_data = 3; + // sources that are allowed to be published + repeated TrackSource can_publish_sources = 9; + // indicates that it's hidden to others + bool hidden = 7; + // indicates it's a recorder instance + // deprecated: use ParticipantInfo.kind instead + bool recorder = 8 [deprecated = true]; + // indicates that participant can update own metadata and attributes + bool can_update_metadata = 10; + // indicates that participant is an agent + // deprecated: use ParticipantInfo.kind instead + bool agent = 11 [deprecated = true]; + // if a participant can subscribe to metrics + bool can_subscribe_metrics = 12; + // if a participant can manage an agent session via RemoteSession (control and access state) + bool can_manage_agent_session = 13; + + // NEXT_ID: 14 +} + +message ParticipantInfo { + enum State { + // websocket' connected, but not offered yet + JOINING = 0; + // server received client offer + JOINED = 1; + // ICE connectivity established + ACTIVE = 2; + // WS disconnected + DISCONNECTED = 3; + } + enum Kind { + // standard participants, e.g. web clients + STANDARD = 0; + // only ingests streams + INGRESS = 1; + // only consumes streams + EGRESS = 2; + // SIP participants + SIP = 3; + // LiveKit agents + AGENT = 4; + // Connectors participants + CONNECTOR = 7; + // Bridge participants + BRIDGE = 8; + // NEXT_ID: 9 + } + enum KindDetail { + CLOUD_AGENT = 0; + FORWARDED = 1; + CONNECTOR_WHATSAPP = 2; + CONNECTOR_TWILIO = 3; + BRIDGE_RTSP = 4; + SIMULATION = 5; + // NEXT_ID: 6 + } + string sid = 1; + string identity = 2; + State state = 3; + repeated TrackInfo tracks = 4; + string metadata = 5 [ + (logger.sensitivity) = SENSITIVITY_PII, + (logger.redact_format) = "" + ]; + // timestamp when participant joined room, in seconds + int64 joined_at = 6; + // timestamp when participant joined room, in milliseconds + int64 joined_at_ms = 17; + string name = 9 [(logger.sensitivity) = SENSITIVITY_PII]; + uint32 version = 10; + ParticipantPermission permission = 11; + string region = 12; + // indicates the participant has an active publisher connection + // and can publish to the server + bool is_publisher = 13; + Kind kind = 14; + map attributes = 15 [ + (logger.sensitivity) = SENSITIVITY_PII, + (logger.redact_format) = "" + ]; + DisconnectReason disconnect_reason = 16; + repeated KindDetail kind_details = 18; + repeated DataTrackInfo data_tracks = 19; + + // protocol version used for client feature compatibility + int32 client_protocol = 20; + + // capabilities the participant's client advertises, mirrored from ClientInfo. + // Lets other participants perform client-side feature detection. + repeated ClientInfo.Capability capabilities = 21; + + // NEXT_ID: 22 +} + +enum TrackType { + AUDIO = 0; + VIDEO = 1; + DATA = 2; +} + +enum TrackSource { + UNKNOWN = 0; + CAMERA = 1; + MICROPHONE = 2; + SCREEN_SHARE = 3; + SCREEN_SHARE_AUDIO = 4; +} + +message Encryption { + enum Type { + NONE = 0; + GCM = 1; + CUSTOM = 2; + } +} + +message SimulcastCodecInfo { + string mime_type = 1; + string mid = 2; + string cid = 3; + repeated VideoLayer layers = 4; + VideoLayer.Mode video_layer_mode = 5; + // cid (client side id for track) could be different between + // signalling (AddTrackRequest) and SDP offer. This field + // will be populated only if it is different to avoid + // duplication and keep the representation concise. + string sdp_cid = 6; +} + +message TrackInfo { + string sid = 1; + TrackType type = 2; + string name = 3 [(logger.sensitivity) = SENSITIVITY_PII]; + bool muted = 4; + // original width of video (unset for audio) + // clients may receive a lower resolution version with simulcast + uint32 width = 5; + // original height of video (unset for audio) + uint32 height = 6; + // true if track is simulcasted + bool simulcast = 7 [deprecated = true]; // see `video_layer_mode` in `codecs` + // true if DTX (Discontinuous Transmission) is disabled for audio + bool disable_dtx = 8 [deprecated = true]; // deprecated in favor of `audio_features` + // source of media + TrackSource source = 9; + repeated VideoLayer layers = 10 [deprecated = true]; // see `codecs` for layers of individual codec + // mime type of codec + string mime_type = 11; + string mid = 12; + repeated SimulcastCodecInfo codecs = 13; + bool stereo = 14 [deprecated = true]; // deprecated in favor of `audio_features` + // true if RED (Redundant Encoding) is disabled for audio + bool disable_red = 15; + Encryption.Type encryption = 16; + string stream = 17; + TimedVersion version = 18; + repeated AudioTrackFeature audio_features = 19; + BackupCodecPolicy backup_codec_policy = 20; + + repeated PacketTrailerFeature packet_trailer_features = 21; +} + +message DataTrackInfo { + // Client-assigned, 16-bit identifier that will be attached to packets sent by the publisher. + uint32 pub_handle = 1; + + // Server-assigned track identifier. + string sid = 2; + + // Human-readable identifier (e.g., `geoLocation`, `servoPosition.x`, etc.), unique per publisher. + string name = 3; + + // Method used for end-to-end encryption (E2EE) on packet payloads. + Encryption.Type encryption = 4; + + // Encoding for frame payloads on this track. If unspecified, the track is untyped. + optional DataTrackFrameEncoding frame_encoding = 5; + + // ID of the schema used by frames on this track if the track is typed. + optional DataTrackSchemaId schema = 6; +} + +// Encoding for frame payloads. +message DataTrackFrameEncoding { + + // Well-known encoding for frame payloads. + // + // Mirrors the well-known message encodings from the MCAP spec: + // https://mcap.dev/spec/registry#message-encodings + // + enum WellKnownFrameEncoding { + WELL_KNOWN_FRAME_ENCODING_UNSPECIFIED = 0; + // ROS 1: must be described by `ROS1_MSG` schema encoding. + WELL_KNOWN_FRAME_ENCODING_ROS1 = 1; + // CDR: must be described by `ROS2_MSG`, `ROS2_IDL`, or `OMG_IDL` schema encoding. + WELL_KNOWN_FRAME_ENCODING_CDR = 2; + // Protocol Buffer: must be described by `PROTOBUF` schema encoding. + WELL_KNOWN_FRAME_ENCODING_PROTOBUF = 3; + // FlatBuffer: must be described by `FLATBUFFER` schema encoding. + WELL_KNOWN_FRAME_ENCODING_FLATBUFFER = 4; + // CBOR: self-describing. + WELL_KNOWN_FRAME_ENCODING_CBOR = 5; + // MessagePack: self-describing. + WELL_KNOWN_FRAME_ENCODING_MSGPACK = 6; + // JSON: self-describing or described by `JSON_SCHEMA` schema encoding. + WELL_KNOWN_FRAME_ENCODING_JSON = 7; + } + + oneof value { + WellKnownFrameEncoding well_known = 1; + // Identifier of a custom encoding not covered by the well-known cases. + // This must be non-empty and no longer than 32 characters. + string custom = 2; + } +} + +// Encoding for schema definition. +message DataTrackSchemaEncoding { + // Well-known encoding for schema definition. + // + // Mirrors the well-known schema encodings from the MCAP spec: + // https://mcap.dev/spec/registry#schema-encodings + // + enum WellKnownSchemaEncoding { + WELL_KNOWN_SCHEMA_ENCODING_UNSPECIFIED = 0; + // Protocol Buffer IDL: describes `PROTOBUF` frame encoding. + WELL_KNOWN_SCHEMA_ENCODING_PROTOBUF = 1; + // FlatBuffer IDL: describes `FLATBUFFER` frame encoding. + WELL_KNOWN_SCHEMA_ENCODING_FLATBUFFER = 2; + // ROS 1 Message: describes `ROS1` frame encoding. + WELL_KNOWN_SCHEMA_ENCODING_ROS1_MSG = 3; + // ROS 2 Message: describes `CDR` frame encoding. + WELL_KNOWN_SCHEMA_ENCODING_ROS2_MSG = 4; + // ROS 2 IDL: describes `CDR` frame encoding. + WELL_KNOWN_SCHEMA_ENCODING_ROS2_IDL = 5; + // OMG IDL: describes `CDR` frame encoding. + WELL_KNOWN_SCHEMA_ENCODING_OMG_IDL = 6; + // JSON Schema: describes `JSON` frame encoding. + WELL_KNOWN_SCHEMA_ENCODING_JSON_SCHEMA = 7; + } + + oneof value { + WellKnownSchemaEncoding well_known = 1; + // Identifier of a custom encoding not covered by the well-known cases. + // This must be non-empty and no longer than 32 characters. + string custom = 2; + } +} + +// Identifier for a data track schema. +// +// Schemas with the same name but different encodings are distinct. +// +message DataTrackSchemaId { + // This must be non-empty and no longer than 256 characters. + string name = 1; + DataTrackSchemaEncoding encoding = 2; +} + +enum DataTrackExtensionID { + DTEI_INVALID = 0; + DTEI_E2EE = 1; + DTEI_USER_TIMESTAMP = 2; + DTEI_PARTICIPANT_SID = 255; +} + +message DataTrackExtensionParticipantSid { + DataTrackExtensionID id = 1; + string participant_sid = 2; +} + +message DataTrackSubscriptionOptions { + // Rate in frames per second (FPS) the subscriber wants to receive frames at. + // If omitted, the subscriber defaults to the publisher's fps + optional uint32 target_fps = 1; +} + +// Key used to uniquely identify a data blob for storage and retrieval. +message DataBlobKey { + oneof key { + // Generic string key, blob contains arbitrary data. + string generic = 1; + // Data track schema identifier, blob contains schema definition. + DataTrackSchemaId schema_id = 2 [(logger.name) = "schemaID"]; + } +} + +// A blob of data stored in a room identified by a unique key. +message DataBlob { + // Unique key the data blob is identified by. + DataBlobKey key = 1; + // Contents of the data blob. This must not exceed 50 KB. + bytes contents = 2 [ + (logger.sensitivity) = SENSITIVITY_PII, + (logger.redact_format) = "" + ]; +} + +enum VideoQuality { + LOW = 0; + MEDIUM = 1; + HIGH = 2; + OFF = 3; +} + +// provide information about available spatial layers +message VideoLayer { + enum Mode { + MODE_UNUSED = 0; + ONE_SPATIAL_LAYER_PER_STREAM = 1; + MULTIPLE_SPATIAL_LAYERS_PER_STREAM = 2; + ONE_SPATIAL_LAYER_PER_STREAM_INCOMPLETE_RTCP_SR = 3; + } + + // for tracks with a single layer, this should be HIGH + VideoQuality quality = 1; + uint32 width = 2; + uint32 height = 3; + // target bitrate in bit per second (bps), server will measure actual + uint32 bitrate = 4; + uint32 ssrc = 5; + int32 spatial_layer = 6; + string rid = 7; + + uint32 repair_ssrc = 8; +} + +// new DataPacket API +message DataPacket { + enum Kind { + RELIABLE = 0; + LOSSY = 1; + } + Kind kind = 1 [deprecated = true]; + // participant identity of user that sent the message + string participant_identity = 4; + // identities of participants who will receive the message (sent to all by default) + repeated string destination_identities = 5; + oneof value { + UserPacket user = 2; + ActiveSpeakerUpdate speaker = 3 [deprecated = true]; + SipDTMF sip_dtmf = 6; + Transcription transcription = 7; + MetricsBatch metrics = 8; + ChatMessage chat_message = 9; + RpcRequest rpc_request = 10; + RpcAck rpc_ack = 11; + RpcResponse rpc_response = 12; + DataStream.Header stream_header = 13; + DataStream.Chunk stream_chunk = 14; + DataStream.Trailer stream_trailer = 15; + EncryptedPacket encrypted_packet = 18; + } + // sequence number of reliable packet + uint32 sequence = 16; + // sid of the user that sent the message + string participant_sid = 17; + + // NEXT_ID: 19 +} + +message EncryptedPacket { + Encryption.Type encryption_type = 1; + bytes iv = 2; + uint32 key_index = 3; + bytes encrypted_value = 4; // This is an encrypted EncryptedPacketPayload message representation +} + +message EncryptedPacketPayload { + oneof value { + UserPacket user = 1; + ChatMessage chat_message = 3; + RpcRequest rpc_request = 4; + RpcAck rpc_ack = 5; + RpcResponse rpc_response = 6; + DataStream.Header stream_header = 7; + DataStream.Chunk stream_chunk = 8; + DataStream.Trailer stream_trailer = 9; + + // NEXT_ID: 10 + } +} + +message ActiveSpeakerUpdate { + option deprecated = true; + repeated SpeakerInfo speakers = 1; +} + +message SpeakerInfo { + string sid = 1; + // audio level, 0-1.0, 1 is loudest + float level = 2; + // true if speaker is currently active + bool active = 3; +} + +message UserPacket { + // participant ID of user that sent the message + string participant_sid = 1 [deprecated = true]; + string participant_identity = 5 [deprecated = true]; + // user defined payload + bytes payload = 2; + // the ID of the participants who will receive the message (sent to all by default) + repeated string destination_sids = 3 [deprecated = true]; + // identities of participants who will receive the message (sent to all by default) + repeated string destination_identities = 6 [deprecated = true]; + // topic under which the message was published + optional string topic = 4; + // Unique ID to identify the message + optional string id = 8; + // start and end time allow relating the message to specific media time + optional uint64 start_time = 9; + optional uint64 end_time = 10; + // added by SDK to enable de-duping of messages, for INTERNAL USE ONLY + bytes nonce = 11; + + // NEXT_ID: 12 +} + +message SipDTMF { + uint32 code = 3; + string digit = 4; +} + +message Transcription { + // Participant that got its speech transcribed + string transcribed_participant_identity = 2; + string track_id = 3 [(logger.name) = "trackID"]; + repeated TranscriptionSegment segments = 4; + + // NEXT_ID: 6 +} + +message TranscriptionSegment { + string id = 1; + string text = 2; + uint64 start_time = 3; + uint64 end_time = 4; + bool final = 5; + string language = 6; +} + +message ChatMessage { + string id = 1; // uuid + int64 timestamp = 2; + optional int64 edit_timestamp = 3; // populated only if the intent is to edit/update an existing message + string message = 4; + bool deleted = 5; // true to remove message + bool generated = 6; // true if the chat message has been generated by an agent from a participant's audio transcription +} + +message RpcRequest { + string id = 1; + string method = 2; + string payload = 3; + uint32 response_timeout_ms = 4; + uint32 version = 5; + // Compressed payload data. When set, this field is used instead of `payload`. + bytes compressed_payload = 6; +} + +message RpcAck { + string request_id = 1 [(logger.name) = "requestID"]; +} + +message RpcResponse { + string request_id = 1 [(logger.name) = "requestID"]; + oneof value { + string payload = 2; + RpcError error = 3; + // Compressed payload data. When set, this field is used instead of `payload`. + bytes compressed_payload = 4; + } +} + +message RpcError { + uint32 code = 1; + string message = 2; + string data = 3; +} + +enum ConnectionQuality { + POOR = 0; + GOOD = 1; + EXCELLENT = 2; + LOST = 3; +} + +message ParticipantTracks { + // participant ID of participant to whom the tracks belong + string participant_sid = 1; + repeated string track_sids = 2; +} + +// details about the server +message ServerInfo { + enum Edition { + Standard = 0; + Cloud = 1; + } + Edition edition = 1; + string version = 2; + int32 protocol = 3; + string region = 4; + string node_id = 5 [(logger.name) = "nodeID"]; + // additional debugging information. sent only if server is in development mode + string debug_info = 6; + int32 agent_protocol = 7; +} + +// details about the client +message ClientInfo { + enum SDK { + UNKNOWN = 0; + JS = 1; + SWIFT = 2; + ANDROID = 3; + FLUTTER = 4; + GO = 5; + UNITY = 6; + REACT_NATIVE = 7; + RUST = 8; + PYTHON = 9; + CPP = 10; + UNITY_WEB = 11; + NODE = 12; + UNREAL = 13; + ESP32 = 14; + DOTNET = 15; + } + + // Optional capabilities advertised by the client at connect time. The SFU + // uses these flags to decide whether to enable features that require + // client-side support (e.g. passing RTP packet trailers through to the + // subscriber instead of stripping them). + enum Capability { + CAP_UNUSED = 0; + CAP_PACKET_TRAILER = 1; + CAP_COMPRESSION_DEFLATE_RAW = 2; + } + + SDK sdk = 1; + string version = 2; + int32 protocol = 3; + string os = 4; + string os_version = 5; + string device_model = 6; + string browser = 7; + string browser_version = 8; + string address = 9; + // wifi, wired, cellular, vpn, empty if not known + string network = 10; + // comma separated list of additional LiveKit SDKs in use of this client, with versions + // e.g. "components-js:1.2.3,track-processors-js:1.2.3" + string other_sdks = 11; + // client protocol version + int32 client_protocol = 12; + // capabilities the client advertises. Populated automatically by each SDK; + // not a user-configurable setting. + repeated Capability capabilities = 13; +} + +// server provided client configuration +message ClientConfiguration { + VideoConfiguration video = 1; + VideoConfiguration screen = 2; + + ClientConfigSetting resume_connection = 3; + DisabledCodecs disabled_codecs = 4; + ClientConfigSetting force_relay = 5; +} + +enum ClientConfigSetting { + UNSET = 0; + DISABLED = 1; + ENABLED = 2; +} + +message VideoConfiguration { + ClientConfigSetting hardware_encoder = 1; +} + +message DisabledCodecs { + // disabled for both publish and subscribe + repeated Codec codecs = 1; + // only disable for publish + repeated Codec publish = 2; +} + +enum DisconnectReason { + UNKNOWN_REASON = 0; + // the client initiated the disconnect + CLIENT_INITIATED = 1; + // another participant with the same identity has joined the room + DUPLICATE_IDENTITY = 2; + // the server instance is shutting down + SERVER_SHUTDOWN = 3; + // RoomService.RemoveParticipant was called + PARTICIPANT_REMOVED = 4; + // RoomService.DeleteRoom was called + ROOM_DELETED = 5; + // the client is attempting to resume a session, but server is not aware of it + STATE_MISMATCH = 6; + // client was unable to connect fully + JOIN_FAILURE = 7; + // Cloud-only, the server requested Participant to migrate the connection elsewhere + MIGRATION = 8; + // the signal websocket was closed unexpectedly + SIGNAL_CLOSE = 9; + // the room was closed, due to all Standard and Ingress participants having left + ROOM_CLOSED = 10; + // SIP callee did not respond in time + USER_UNAVAILABLE = 11; + // SIP callee rejected the call (busy) + USER_REJECTED = 12; + // SIP protocol failure or unexpected response + SIP_TRUNK_FAILURE = 13; + // server timed out a participant session + CONNECTION_TIMEOUT = 14; + // media stream failure or media timeout + MEDIA_FAILURE = 15; + // agent encountered an error + AGENT_ERROR = 16; +} + +// why a room ended, reported on room-ended telemetry +enum RoomEndReason { + ROOM_END_UNKNOWN = 0; + // RoomService.DeleteRoom was called + ROOM_END_API_DELETE = 1; + // the room sat empty past its empty or departure timeout + ROOM_END_IDLE_TIMEOUT = 2; + // the server instance is shutting down + ROOM_END_SERVER_SHUTDOWN = 3; + // the room was restarted or replaced elsewhere + ROOM_END_SUPERSEDED = 4; + // the room failed to finish opening + ROOM_END_OPEN_FAILED = 5; +} + +message RTPDrift { + google.protobuf.Timestamp start_time = 1; + google.protobuf.Timestamp end_time = 2; + double duration = 3; + + uint64 start_timestamp = 4; + uint64 end_timestamp = 5; + uint64 rtp_clock_ticks = 6; + int64 drift_samples = 7; + double drift_ms = 8; + double clock_rate = 9; +} + +message RTPStats { + google.protobuf.Timestamp start_time = 1; + google.protobuf.Timestamp end_time = 2; + double duration = 3; + + uint32 packets = 4; + double packet_rate = 5; + + uint64 bytes = 6; + uint64 header_bytes = 39; + double bitrate = 7; + + uint32 packets_lost = 8; + double packet_loss_rate = 9; + float packet_loss_percentage = 10; + + uint32 packets_duplicate = 11; + double packet_duplicate_rate = 12; + + uint64 bytes_duplicate = 13; + uint64 header_bytes_duplicate = 40; + double bitrate_duplicate = 14; + + uint32 packets_padding = 15; + double packet_padding_rate = 16; + + uint64 bytes_padding = 17; + uint64 header_bytes_padding = 41; + double bitrate_padding = 18; + + uint32 packets_out_of_order = 19; + + uint32 frames = 20; + double frame_rate = 21; + + double jitter_current = 22; + double jitter_max = 23; + + map gap_histogram = 24; + + uint32 nacks = 25; + uint32 nack_acks = 37; + uint32 nack_misses = 26; + uint32 nack_repeated = 38; + + uint32 plis = 27; + google.protobuf.Timestamp last_pli = 28; + + uint32 firs = 29; + google.protobuf.Timestamp last_fir = 30; + + uint32 rtt_current = 31; + uint32 rtt_max = 32; + + uint32 key_frames = 33; + google.protobuf.Timestamp last_key_frame = 34; + + uint32 layer_lock_plis = 35; + google.protobuf.Timestamp last_layer_lock_pli = 36; + + RTPDrift packet_drift = 44; + RTPDrift ntp_report_drift = 45; + RTPDrift rebased_report_drift = 46; + RTPDrift received_report_drift = 47; + + uint32 fec_packets = 48; + uint64 fec_bytes = 49; + uint32 fec_packets_discarded = 50; + uint32 fec_packets_recovered = 51; + // NEXT_ID: 52 +} + +message RTCPSenderReportState { + uint32 rtp_timestamp = 1; + uint64 rtp_timestamp_ext = 2; + uint64 ntp_timestamp = 3; + int64 at = 4; // time at which this happened + int64 at_adjusted = 5; + uint32 packets = 6; + uint64 octets = 7; +} + +message RTPForwarderState { + bool started = 1; + int32 reference_layer_spatial = 2; + int64 pre_start_time = 3; + uint64 ext_first_timestamp = 4; + uint64 dummy_start_timestamp_offset = 5; + RTPMungerState rtp_munger = 6; + oneof codec_munger { + VP8MungerState vp8_munger = 7; + } + repeated RTCPSenderReportState sender_report_state = 8; + DependencyDescriptorForwarderState dependency_descriptor_forwarder = 9; +} + +// State of a forwarder that writes the dependency descriptor for the subscriber, so +// its frame numbers continue on a migrated subscriber. +message DependencyDescriptorForwarderState { + // highest extended frame number sent to the subscriber + uint64 ext_last_frame_number = 1; +} + +message RTPMungerState { + uint64 ext_last_sequence_number = 1; + uint64 ext_second_last_sequence_number = 2; + uint64 ext_last_timestamp = 3; + uint64 ext_second_last_timestamp = 4; + bool last_marker = 5; + bool second_last_marker = 6; +} + +message VP8MungerState { + int32 ext_last_picture_id = 1 [(logger.name) = "extLastPictureID"]; + bool picture_id_used = 2; + uint32 last_tl0_pic_idx = 3; + bool tl0_pic_idx_used = 4; + bool tid_used = 5; + uint32 last_key_idx = 6; + bool key_idx_used = 7; +} + +message TimedVersion { + int64 unix_micro = 1; + int32 ticks = 2; +} + +enum ReconnectReason { + RR_UNKNOWN = 0; + RR_SIGNAL_DISCONNECTED = 1; + RR_PUBLISHER_FAILED = 2; + RR_SUBSCRIBER_FAILED = 3; + RR_SWITCH_CANDIDATE = 4; +} + +enum SubscriptionError { + SE_UNKNOWN = 0; + SE_CODEC_UNSUPPORTED = 1; + SE_TRACK_NOTFOUND = 2; +} + +enum AudioTrackFeature { + TF_STEREO = 0; + TF_NO_DTX = 1; + TF_AUTO_GAIN_CONTROL = 2; + TF_ECHO_CANCELLATION = 3; + TF_NOISE_SUPPRESSION = 4; + TF_ENHANCED_NOISE_CANCELLATION = 5; + TF_PRECONNECT_BUFFER = 6; // client will buffer audio once available and send it to the server via bytes stream once connected +} + +enum PacketTrailerFeature { + PTF_USER_TIMESTAMP = 0; + PTF_FRAME_ID = 1; + PTF_USER_DATA = 2; +} + +message DataStream { + // enum for operation types (specific to TextHeader) + enum OperationType { + CREATE = 0; + UPDATE = 1; + DELETE = 2; + REACTION = 3; + } + + // header properties specific to text streams + message TextHeader { + OperationType operation_type = 1; + int32 version = 2; // Optional: Version for updates/edits + string reply_to_stream_id = 3 [(logger.name) = "replyToStreamID"]; // Optional: Reply to specific message + repeated string attached_stream_ids = 4; // file attachments for text streams + bool generated = 5; // true if the text has been generated by an agent from a participant's audio transcription + } + + // header properties specific to byte or file streams + message ByteHeader { + string name = 1; + } + + // The compression type of the whole data stream + // + // This will only get populated when send to participants with a + // client protocol >= 2 which advertise a client capability of CAP_COMPRESSION_DEFLATE_RAW + enum CompressionType { + NONE = 0; + DEFLATE_RAW = 1; // DEFLATE_RAW = DEFLATE without header+checksum/trailer + } + + // main DataStream.Header that contains a oneof for specific headers + message Header { + string stream_id = 1 [(logger.name) = "streamID"]; // unique identifier for this data stream + int64 timestamp = 2; // using int64 for Unix timestamp + string topic = 3; + string mime_type = 4; + optional uint64 total_length = 5; // only populated for finite streams, if it's a stream of unknown size this stays empty + Encryption.Type encryption_type = 7 [deprecated = true]; // this is set on the DataPacket + map attributes = 8; // user defined attributes map that can carry additional info + + // oneof to choose between specific header types + oneof content_header { + TextHeader text_header = 9; + ByteHeader byte_header = 10; + } + + // Optional inline content so that a data stream can be sent as a single packet for short payloads. + optional bytes inline_content = 11; // content as binary (bytes) + CompressionType compression = 12; + } + + message Chunk { + string stream_id = 1 [(logger.name) = "streamID"]; // unique identifier for this data stream to map it to the correct header + uint64 chunk_index = 2; + bytes content = 3; // content as binary (bytes) + int32 version = 4; // a version indicating that this chunk_index has been retroactively modified and the original one needs to be replaced + optional bytes iv = 5 [deprecated = true]; // this is set on the DataPacket + } + + message Trailer { + string stream_id = 1 [(logger.name) = "streamID"]; // unique identifier for this data stream + string reason = 2; // reason why the stream was closed (could contain "error" / "interrupted" / empty for expected end) + map attributes = 3; // finalizing updates for the stream, can also include additional insights for errors or endTime for transcription + } +} + +message FilterParams { + repeated string include_events = 1; + repeated string exclude_events = 2; +} + +message WebhookConfig { + string url = 1; + string signing_key = 2; + FilterParams filter_params = 3; +} + +message SubscribedAudioCodec { + string codec = 1; + bool enabled = 2; +} diff --git a/webrtc-livekit/src/main/proto/livekit_rtc.proto b/webrtc-livekit/src/main/proto/livekit_rtc.proto new file mode 100644 index 00000000..206517b3 --- /dev/null +++ b/webrtc-livekit/src/main/proto/livekit_rtc.proto @@ -0,0 +1,683 @@ +// Copyright 2023 LiveKit, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package livekit; +option java_package = "dev.onvoid.webrtc.livekit.internal.proto"; + +import "livekit_models.proto"; +import "logger/options.proto"; + +option csharp_namespace = "LiveKit.Proto"; +option go_package = "github.com/livekit/protocol/livekit"; +option ruby_package = "LiveKit::Proto"; + +message SignalRequest { + oneof message { + // participant offer for publisher + SessionDescription offer = 1; + // participant answering subscriber offer + SessionDescription answer = 2; + TrickleRequest trickle = 3; + AddTrackRequest add_track = 4; + // mute the participant's published tracks + MuteTrackRequest mute = 5; + // Subscribe or unsubscribe from tracks + UpdateSubscription subscription = 6; + // Update settings of subscribed tracks + UpdateTrackSettings track_setting = 7; + // Immediately terminate session + LeaveRequest leave = 8; + // Update published video layers + UpdateVideoLayers update_layers = 10 [deprecated = true]; + // Update subscriber permissions + SubscriptionPermission subscription_permission = 11; + // sync client's subscribe state to server during reconnect + SyncState sync_state = 12; + // Simulate conditions, for client validations + SimulateScenario simulate = 13; + // client triggered ping to server + int64 ping = 14; // deprecated by ping_req (message Ping) + // update a participant's own metadata, name, or attributes + // requires canUpdateOwnParticipantMetadata permission + UpdateParticipantMetadata update_metadata = 15; + Ping ping_req = 16; + // Update local audio track settings + UpdateLocalAudioTrack update_audio_track = 17; + // Update local video track settings + UpdateLocalVideoTrack update_video_track = 18; + // Publish a data track + PublishDataTrackRequest publish_data_track_request = 19; + // Unpublish a data track + UnpublishDataTrackRequest unpublish_data_track_request = 20; + // Update subscription state for one or more data tracks + UpdateDataSubscription update_data_subscription = 21; + // Store a data blob. + StoreDataBlobRequest store_data_blob_request = 22; + // Retrieve a stored data blob. + GetDataBlobRequest get_data_blob_request = 23; + } +} + +message SignalResponse { + oneof message { + // sent when join is accepted + JoinResponse join = 1; + // sent when server answers publisher + SessionDescription answer = 2; + // sent when server is sending subscriber an offer + SessionDescription offer = 3; + // sent when an ICE candidate is available + TrickleRequest trickle = 4; + // sent when participants in the room has changed + ParticipantUpdate update = 5; + // sent to the participant when their track has been published + TrackPublishedResponse track_published = 6; + // Immediately terminate session + LeaveRequest leave = 8; + // server initiated mute + MuteTrackRequest mute = 9; + // indicates changes to speaker status, including when they've gone to not speaking + SpeakersChanged speakers_changed = 10; + // sent when metadata of the room has changed + RoomUpdate room_update = 11; + // when connection quality changed + ConnectionQualityUpdate connection_quality = 12; + // when streamed tracks state changed, used to notify when any of the streams were paused due to + // congestion + StreamStateUpdate stream_state_update = 13; + // when max subscribe quality changed, used by dynamic broadcasting to disable unused layers + SubscribedQualityUpdate subscribed_quality_update = 14; + // when subscription permission changed + SubscriptionPermissionUpdate subscription_permission_update = 15; + // update the token the client was using, to prevent an active client from using an expired token + string refresh_token = 16 [(logger.sensitivity) = SENSITIVITY_SECRET]; + // server initiated track unpublish + TrackUnpublishedResponse track_unpublished = 17; + // respond to ping + int64 pong = 18; // deprecated by pong_resp (message Pong) + // sent when client reconnects + ReconnectResponse reconnect = 19; + // respond to Ping + Pong pong_resp = 20; + // Subscription response, client should not expect any media from this subscription if it fails + SubscriptionResponse subscription_response = 21; + // Response relating to user inititated requests that carry a `request_id` + RequestResponse request_response = 22; + // notify to the publisher when a published track has been subscribed for the first time + TrackSubscribed track_subscribed = 23; + // notify to the participant when they have been moved to a new room + RoomMovedResponse room_moved = 24; + // notify number of required media sections to satisfy subscribed tracks + MediaSectionsRequirement media_sections_requirement = 25; + // when audio subscription changes, used to enable simulcasting of audio codecs based on subscriptions + SubscribedAudioCodecUpdate subscribed_audio_codec_update = 26; + // Sent in response to `PublishDataTrackRequest`. + PublishDataTrackResponse publish_data_track_response = 27; + // Sent in response to `UnpublishDataTrackRequest` or SFU-initiated unpublish. + UnpublishDataTrackResponse unpublish_data_track_response = 28; + // Sent to data track subscribers to provide mapping from track SIDs to handles. + DataTrackSubscriberHandles data_track_subscriber_handles = 29; + // Sent in response to `StoreDataBlobRequest`. + StoreDataBlobResponse store_data_blob_response = 30; + // Sent in response to `GetDataBlobRequest`. + GetDataBlobResponse get_data_blob_response = 31; + } +} + +enum SignalTarget { + PUBLISHER = 0; + SUBSCRIBER = 1; +} + +message SimulcastCodec { + string codec = 1; + string cid = 2; + repeated VideoLayer layers = 4; + VideoLayer.Mode video_layer_mode = 5; + + // NEXT-ID: 6 +} + +message AddTrackRequest { + // client ID of track, to match it when RTC track is received + string cid = 1; + string name = 2; + TrackType type = 3; + uint32 width = 4; + uint32 height = 5; + // true to add track and initialize to muted + bool muted = 6; + // true if DTX (Discontinuous Transmission) is disabled for audio + bool disable_dtx = 7 [deprecated = true]; // deprecated in favor of audio_features + TrackSource source = 8; + repeated VideoLayer layers = 9; + + repeated SimulcastCodec simulcast_codecs = 10; + + // server ID of track, publish new codec to exist track + string sid = 11; + + bool stereo = 12 [deprecated = true]; // deprecated in favor of audio_features + // true if RED (Redundant Encoding) is disabled for audio + bool disable_red = 13; + + Encryption.Type encryption = 14; + // which stream the track belongs to, used to group tracks together. + // if not specified, server will infer it from track source to bundle camera/microphone, screenshare/audio together + string stream = 15; + BackupCodecPolicy backup_codec_policy = 16; + + repeated AudioTrackFeature audio_features = 17; + + repeated PacketTrailerFeature packet_trailer_features = 18; +} + +message PublishDataTrackRequest { + // Client-assigned, 16-bit identifier that will be attached to packets sent by the publisher. + // This must be non-zero and unique for each data track published by the publisher. + uint32 pub_handle = 1; + + // Human-readable identifier (e.g., `geoLocation`, `servoPosition.x`, etc.), unique per publisher. + // This must be non-empty and no longer than 256 characters. + string name = 2; + + // Method used for end-to-end encryption (E2EE) on frame payloads. + Encryption.Type encryption = 3; + + // Encoding for frame payloads on this track. If unspecified, the track is untyped. + optional DataTrackFrameEncoding frame_encoding = 4; + + // ID of the schema used by frames on this track if the track is typed. + // If set, the associated schema must be stored with `StoreDataBlobRequest`. + optional DataTrackSchemaId schema = 5; +} + +message PublishDataTrackResponse { + // Information about the published track. + DataTrackInfo info = 1; +} + +message UnpublishDataTrackRequest { + // Publisher handle of the track to unpublish. + uint32 pub_handle = 1; +} + +message UnpublishDataTrackResponse { + // Information about the unpublished track. + DataTrackInfo info = 1; +} + +message DataTrackSubscriberHandles { + message PublishedDataTrack { + string publisher_identity = 1; + string publisher_sid = 2; + string track_sid = 3; + } + + // Maps handles from incoming packets to the track SIDs that the packets belong to. + map sub_handles = 1; +} + +message TrickleRequest { + string candidateInit = 1; + SignalTarget target = 2; + bool final = 3; +} + +message MuteTrackRequest { + string sid = 1; + bool muted = 2; +} + +message JoinResponse { + Room room = 1; + ParticipantInfo participant = 2; + repeated ParticipantInfo other_participants = 3; + // deprecated. use server_info.version instead. + string server_version = 4; + repeated ICEServer ice_servers = 5; + // use subscriber as the primary PeerConnection + bool subscriber_primary = 6; + // when the current server isn't available, return alternate url to retry connection + // when this is set, the other fields will be largely empty + string alternative_url = 7; + ClientConfiguration client_configuration = 8; + // deprecated. use server_info.region instead. + string server_region = 9; + int32 ping_timeout = 10; + int32 ping_interval = 11; + ServerInfo server_info = 12; + // Server-Injected-Frame byte trailer, used to identify unencrypted frames when e2ee is enabled + bytes sif_trailer = 13; + repeated Codec enabled_publish_codecs = 14; + // when set, client should attempt to establish publish peer connection when joining room to speed up publishing + bool fast_publish = 15; +} + +message ReconnectResponse { + repeated ICEServer ice_servers = 1; + ClientConfiguration client_configuration = 2; + ServerInfo server_info = 3; + + // last sequence number of reliable message received before resuming + uint32 last_message_seq = 4; +} + +message TrackPublishedResponse { + string cid = 1; + TrackInfo track = 2; +} + +message TrackUnpublishedResponse { + string track_sid = 1; +} + +message SessionDescription { + string type = 1; // "answer" | "offer" | "pranswer" | "rollback" + string sdp = 2; + uint32 id = 3; + map mid_to_track_id = 4 [(logger.name) = "midToTrackID"]; +} + +message ParticipantUpdate { + repeated ParticipantInfo participants = 1; + + // NEXT-ID: 3 +} + +message UpdateSubscription { + repeated string track_sids = 1; + bool subscribe = 2; + repeated ParticipantTracks participant_tracks = 3; +} + +message UpdateDataSubscription { + message Update { + string track_sid = 1; + bool subscribe = 2; + // Options to apply when initially subscribing or updating an existing subscription. + // When unsubscribing, this field is ignored. + DataTrackSubscriptionOptions options = 3; + } + repeated Update updates = 1; +} + +message StoreDataBlobRequest { + uint32 request_id = 1 [(logger.name) = "requestID"]; + DataBlob blob = 2; +} + +message StoreDataBlobResponse { + uint32 request_id = 1 [(logger.name) = "requestID"]; + // Unique key the data blob was stored under. + DataBlobKey key = 2; +} + +message GetDataBlobRequest { + uint32 request_id = 1 [(logger.name) = "requestID"]; + // Identity of the participant who owns the blob. + string participant_identity = 2; + // Unique key of the data blob to retrieve. + DataBlobKey key = 3; +} + +message GetDataBlobResponse { + uint32 request_id = 1 [(logger.name) = "requestID"]; + DataBlob blob = 2; +} + +message UpdateTrackSettings { + repeated string track_sids = 1; + // when true, the track is placed in a paused state, with no new data returned + bool disabled = 3; + // deprecated in favor of width & height + VideoQuality quality = 4; + // for video, width to receive + uint32 width = 5; + // for video, height to receive + uint32 height = 6; + uint32 fps = 7; + // subscription priority. 1 being the highest (0 is unset) + // when unset, server sill assign priority based on the order of subscription + // server will use priority in the following ways: + // 1. when subscribed tracks exceed per-participant subscription limit, server will + // pause the lowest priority tracks + // 2. when the network is congested, server will assign available bandwidth to + // higher priority tracks first. lowest priority tracks can be paused + uint32 priority = 8; +} + +message UpdateLocalAudioTrack { + string track_sid = 1; + repeated AudioTrackFeature features = 2; +} + +message UpdateLocalVideoTrack { + string track_sid = 1; + uint32 width = 2; + uint32 height = 3; +} + +message LeaveRequest { + // indicates action clients should take on receiving this message + enum Action { + DISCONNECT = 0; // should disconnect + RESUME = 1; // should attempt a resume with `reconnect=1` in join URL + RECONNECT = 2; // should attempt a reconnect, i. e. no `reconnect=1` + } + + // sent when server initiates the disconnect due to server-restart + // indicates clients should attempt full-reconnect sequence + // NOTE: `can_reconnect` obsoleted by `action` starting in protocol version 13 + bool can_reconnect = 1; + DisconnectReason reason = 2; + Action action = 3; + RegionSettings regions = 4; +} + +// message to indicate published video track dimensions are changing +message UpdateVideoLayers { + option deprecated = true; + string track_sid = 1; + repeated VideoLayer layers = 2; +} + +message UpdateParticipantMetadata { + string metadata = 1 [ + (logger.sensitivity) = SENSITIVITY_PII, + (logger.redact_format) = "" + ]; + string name = 2 [ + (logger.sensitivity) = SENSITIVITY_PII, + (logger.redact_format) = "" + ]; + // attributes to update. it only updates attributes that have been set + // to delete attributes, set the value to an empty string + map attributes = 3 [ + (logger.sensitivity) = SENSITIVITY_PII, + (logger.redact_format) = "" + ]; + uint32 request_id = 4 [(logger.name) = "requestID"]; +} + +message ICEServer { + repeated string urls = 1; + string username = 2 [ + (logger.sensitivity) = SENSITIVITY_PII + ]; + string credential = 3 [ + (logger.sensitivity) = SENSITIVITY_SECRET + ]; +} + +message SpeakersChanged { + repeated SpeakerInfo speakers = 1; +} + +message RoomUpdate { + Room room = 1; +} + +message ConnectionQualityInfo { + string participant_sid = 1; + ConnectionQuality quality = 2; + float score = 3; +} + +message ConnectionQualityUpdate { + repeated ConnectionQualityInfo updates = 1; +} + +enum StreamState { + ACTIVE = 0; + PAUSED = 1; +} + +message StreamStateInfo { + string participant_sid = 1; + string track_sid = 2; + StreamState state = 3; +} + +message StreamStateUpdate { + repeated StreamStateInfo stream_states = 1; +} + +message SubscribedQuality { + VideoQuality quality = 1; + bool enabled = 2; +} + +message SubscribedCodec { + string codec = 1; + repeated SubscribedQuality qualities = 2; +} + +message SubscribedQualityUpdate { + string track_sid = 1; + repeated SubscribedQuality subscribed_qualities = 2 [deprecated = true]; + repeated SubscribedCodec subscribed_codecs = 3; +} + +message SubscribedAudioCodecUpdate { + string track_sid = 1; + repeated SubscribedAudioCodec subscribed_audio_codecs = 2; +} + +message TrackPermission { + // permission could be granted either by participant sid or identity + string participant_sid = 1; + bool all_tracks = 2; + repeated string track_sids = 3; + string participant_identity = 4; +} + +message SubscriptionPermission { + bool all_participants = 1; + repeated TrackPermission track_permissions = 2; +} + +message SubscriptionPermissionUpdate { + string participant_sid = 1; + string track_sid = 2; + bool allowed = 3; +} + +message RoomMovedResponse { + // information about the new room + Room room = 1; + // new reconnect token that can be used to reconnect to the new room + string token = 2 [(logger.sensitivity) = SENSITIVITY_SECRET]; + ParticipantInfo participant = 3; + repeated ParticipantInfo other_participants = 4; +} + +message SyncState { + // last subscribe/publish answer before reconnecting + // subscribe answer if using dual peer connection + // publish answer if using single peer connection + SessionDescription answer = 1; + UpdateSubscription subscription = 2; + repeated TrackPublishedResponse publish_tracks = 3; + repeated DataChannelInfo data_channels = 4; + // last received server side offer/sent client side offer before reconnecting + // received server side offer if using dual peer connection + // sent client side offer if using single peer connection + SessionDescription offer = 5; + repeated string track_sids_disabled = 6; + repeated DataChannelReceiveState datachannel_receive_states = 7; + repeated PublishDataTrackResponse publish_data_tracks = 8; + UpdateDataSubscription data_subscription = 9; +} + +message DataChannelReceiveState { + string publisher_sid = 1; + uint32 last_seq = 2; +} + +message DataChannelInfo { + string label = 1; + uint32 id = 2; + SignalTarget target = 3; +} + +enum NegotiatedDataChannelID { + NDCI_INVALID = 0; + NDCI_RELIABLE = 100; + NDCI_LOSSY = 101; + NDCI_DATA_TRACK = 102; +} + +enum CandidateProtocol { + UDP = 0; + TCP = 1; + TLS = 2; +} + +message SimulateScenario { + oneof scenario { + // simulate N seconds of speaker activity + int32 speaker_update = 1; + // simulate local node failure + bool node_failure = 2; + // simulate migration + bool migration = 3; + // server to send leave + bool server_leave = 4; + // switch candidate protocol to tcp + CandidateProtocol switch_candidate_protocol = 5; + // maximum bandwidth for subscribers, in bps + // when zero, clears artificial bandwidth limit + int64 subscriber_bandwidth = 6; + // disconnect signal on resume + bool disconnect_signal_on_resume = 7; + // disconnect signal on resume before sending any messages from server + bool disconnect_signal_on_resume_no_messages = 8; + // full reconnect leave request + bool leave_request_full_reconnect = 9; + } +} + +message Ping { + int64 timestamp = 1; + // rtt in milliseconds calculated by client + int64 rtt = 2; +} + +message Pong { + // timestamp field of last received ping request + int64 last_ping_timestamp = 1; + int64 timestamp = 2; +} + +message RegionSettings { + repeated RegionInfo regions = 1; +} + +message RegionInfo { + string region = 1; + string url = 2; + int64 distance = 3; +} + +message SubscriptionResponse { + string track_sid = 1; + SubscriptionError err = 2; +} + +message RequestResponse { + enum Reason { + OK = 0; + NOT_FOUND = 1; + NOT_ALLOWED = 2; + LIMIT_EXCEEDED = 3; + QUEUED = 4; + UNSUPPORTED_TYPE = 5; + UNCLASSIFIED_ERROR = 6; + INVALID_HANDLE = 7; + INVALID_NAME = 8; + DUPLICATE_HANDLE = 9; + DUPLICATE_NAME = 10; + INVALID_REQUEST = 11; + } + + uint32 request_id = 1 [(logger.name) = "requestID"]; + Reason reason = 2; + string message = 3; + oneof request { + TrickleRequest trickle = 4; + AddTrackRequest add_track = 5; + MuteTrackRequest mute = 6; + UpdateParticipantMetadata update_metadata = 7; + UpdateLocalAudioTrack update_audio_track = 8; + UpdateLocalVideoTrack update_video_track = 9; + PublishDataTrackRequest publish_data_track = 10; + UnpublishDataTrackRequest unpublish_data_track = 11; + } +} + +message TrackSubscribed { + string track_sid = 1; +} + +message ConnectionSettings { + bool auto_subscribe = 1; + bool adaptive_stream = 2; + optional bool subscriber_allow_pause = 3; + bool disable_ice_lite = 4; + optional bool auto_subscribe_data_track = 5; +} + +message JoinRequest { + ClientInfo client_info = 1; + ConnectionSettings connection_settings = 2; + + string metadata = 3 [ + (logger.sensitivity) = SENSITIVITY_PII, + (logger.redact_format) = "" + ]; // if not empty, will overwrite `metadata` in token + + // will set keys provided via this + // will overwrite if the same key is in the token + // will not delete keys from token if there is a key collision and this sets that key to empty value + map participant_attributes = 4 [ + (logger.sensitivity) = SENSITIVITY_PII, + (logger.redact_format) = "" + ]; + + repeated AddTrackRequest add_track_requests = 5; + SessionDescription publisher_offer = 6; + + bool reconnect = 7; + ReconnectReason reconnect_reason = 8; + string participant_sid = 9; + SyncState sync_state = 10; +} + +message WrappedJoinRequest { + enum Compression { + NONE = 0; + GZIP = 1; + } + + Compression compression = 1; + bytes join_request = 2; // marshalled JoinRequest + potentially compressed +} + +message MediaSectionsRequirement { + uint32 num_audios = 1; + uint32 num_videos = 2; +} diff --git a/webrtc-livekit/src/main/proto/logger/options.proto b/webrtc-livekit/src/main/proto/logger/options.proto new file mode 100644 index 00000000..6313ca3e --- /dev/null +++ b/webrtc-livekit/src/main/proto/logger/options.proto @@ -0,0 +1,33 @@ +syntax = "proto3"; +package logger; +option java_package = "dev.onvoid.webrtc.livekit.internal.proto.logger"; + +option go_package = "github.com/livekit/protocol/livekit/logger"; +option csharp_namespace = "LiveKit.Proto"; +option ruby_package = "LiveKit::Proto"; + +import "google/protobuf/descriptor.proto"; + +enum Sensitivity { + // Unannotated default — safe to log. + // Identifiers, IDs, timestamps, enums, status fields. + SENSITIVITY_UNSPECIFIED = 0; + + // User-identifying or user-controlled data: display names, phone numbers, + // metadata, attributes, custom headers, auth usernames, semi-public account + // identifiers (AWS role ARNs, Azure account names). + // Recorded under a sensitivity tag by a sink that can tag; redacted by one + // that cannot. + SENSITIVITY_PII = 1; + + // Credentials and authentication material: passwords, access keys, session + // tokens, signing keys, API keys, ICE credentials. + // ALWAYS redacted: no sink may record it. + SENSITIVITY_SECRET = 2; +} + +extend google.protobuf.FieldOptions { + string redact_format = 1286; + string name = 1287; + Sensitivity sensitivity = 1288; +} diff --git a/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/RoomIntegrationTest.java b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/RoomIntegrationTest.java new file mode 100644 index 00000000..93b401fa --- /dev/null +++ b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/RoomIntegrationTest.java @@ -0,0 +1,200 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import dev.onvoid.webrtc.client.AudioMode; +import dev.onvoid.webrtc.client.CustomVideo; +import dev.onvoid.webrtc.client.MediaKind; +import dev.onvoid.webrtc.client.RemoteTrack; +import dev.onvoid.webrtc.client.WebRtc; +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; + +/** + * Runs against a LiveKit server, such as {@code livekit-server --dev}, whose + * URL the system property livekit.url names ({@code -Dlivekit.url=ws://localhost:7880}). + * Without it the tests are skipped. + */ +@EnabledIfSystemProperty(named = "livekit.url", matches = ".+") +class RoomIntegrationTest { + + private static final long TIMEOUT_SECONDS = 20; + + private WebRtc rtc; + + private ScheduledExecutorService frames; + + private String roomName; + + + @BeforeEach + void setUp() { + rtc = WebRtc.builder().audio(AudioMode.HEADLESS).build(); + frames = Executors.newSingleThreadScheduledExecutor(); + roomName = "test-" + UUID.randomUUID(); + } + + @AfterEach + void tearDown() { + frames.shutdownNow(); + rtc.close(); + } + + @Test + void participantsSeeEachOther() throws Exception { + Room alice = join("alice"); + CompletableFuture joined = new CompletableFuture<>(); + CompletableFuture left = new CompletableFuture<>(); + alice.onParticipantJoined(joined::complete); + alice.onParticipantLeft(left::complete); + + Room bob = join("bob"); + + assertEquals(roomName, alice.name()); + assertEquals(RoomState.CONNECTED, bob.state()); + assertEquals("bob", await(joined).identity()); + assertEquals("alice", bob.remoteParticipants().get(0).identity()); + + bob.closeAsync().get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + + assertEquals(RoomState.DISCONNECTED, bob.state()); + assertEquals("bob", await(left).identity()); + + alice.close(); + } + + @Test + void subscriberReceivesPublishedVideo() throws Exception { + Room alice = join("alice"); + Room bob = join("bob"); + + CompletableFuture publisherOfTrack = new CompletableFuture<>(); + CompletableFuture received = new CompletableFuture<>(); + bob.onTrackSubscribed((participant, track) -> { + publisherOfTrack.complete(participant); + received.complete(track); + }); + + CustomVideo video = rtc.media().customVideo("camera-" + UUID.randomUUID()); + frames.scheduleAtFixedRate(() -> push(video), 0, 66, TimeUnit.MILLISECONDS); + + Publication publication = await(alice.publish(video, TrackSource.CAMERA)); + + assertTrue(publication.sid().startsWith("TR_"), publication::toString); + assertEquals(TrackSource.CAMERA, publication.source()); + + RemoteTrack track = await(received); + assertEquals("alice", await(publisherOfTrack).identity()); + assertEquals(MediaKind.VIDEO, track.kind()); + assertEquals(publication.sid(), track.id()); + + CompletableFuture frameSize = new CompletableFuture<>(); + track.asVideo().onFrame(frame -> frameSize.complete( + new int[] { frame.buffer.getWidth(), frame.buffer.getHeight() })); + assertArrayEquals(new int[] { 320, 240 }, await(frameSize)); + + bob.close(); + alice.close(); + video.close(); + } + + @Test + void dataReachesOtherParticipant() throws Exception { + Room alice = join("alice"); + Room bob = join("bob"); + + CompletableFuture received = new CompletableFuture<>(); + bob.onData(received::complete); + + byte[] payload = "slide 7".getBytes(StandardCharsets.UTF_8); + await(alice.publishData(payload, "slides", true)); + + DataMessage message = await(received); + assertEquals("alice", message.senderIdentity()); + assertEquals("slides", message.topic().orElseThrow()); + assertTrue(message.isReliable()); + assertTrue(Arrays.equals(payload, message.payload())); + + bob.close(); + alice.close(); + } + + @Test + void badTokenFailsJoin() { + CompletableFuture room = LiveKit.room(rtc, url(), "not-a-token").connect(); + + try { + room.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + fail("Joined with a bad token"); + } + catch (ExecutionException e) { + assertTrue(e.getCause() instanceof LiveKitException, e::toString); + assertTrue(e.getCause().getMessage().contains("HTTP 401"), e.getCause()::getMessage); + } + catch (Exception e) { + fail(e); + } + + assertFalse(rtc.peers().stream().anyMatch(peer -> peer.name().startsWith("livekit"))); + } + + private Room join(String identity) throws Exception { + return await(LiveKit.room(rtc, url(), TestTokens.token(identity, roomName)).connect()); + } + + private static URI url() { + return URI.create(System.getProperty("livekit.url")); + } + + private static T await(CompletableFuture future) throws Exception { + return future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + private static void push(CustomVideo video) { + NativeI420Buffer buffer = NativeI420Buffer.allocate(320, 240); + VideoFrame frame = new VideoFrame(buffer, System.nanoTime()); + + try { + video.push(frame); + } + finally { + frame.release(); + } + } + +} diff --git a/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/SignalingFormatTests.java b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/SignalingFormatTests.java new file mode 100644 index 00000000..7ee477a9 --- /dev/null +++ b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/SignalingFormatTests.java @@ -0,0 +1,99 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URI; + +import org.junit.jupiter.api.Test; + +import dev.onvoid.webrtc.RTCIceCandidate; + +class SignalingFormatTests { + + private static final String CANDIDATE = + "candidate:1 1 udp 2122260223 192.168.1.2 50000 typ host generation 0"; + + @Test + void candidateRoundTrips() { + RTCIceCandidate candidate = new RTCIceCandidate("0", 1, CANDIDATE); + + RTCIceCandidate decoded = CandidateJson.decode(CandidateJson.encode(candidate)); + + assertEquals("0", decoded.sdpMid); + assertEquals(1, decoded.sdpMLineIndex); + assertEquals(CANDIDATE, decoded.sdp); + } + + @Test + void candidateOfServerParses() { + // As pion's ICECandidateInit marshals it. + String json = "{\"candidate\":\"" + CANDIDATE + "\",\"sdpMid\":\"2\",\"sdpMLineIndex\":2," + + "\"usernameFragment\":null}"; + + RTCIceCandidate decoded = CandidateJson.decode(json); + + assertEquals("2", decoded.sdpMid); + assertEquals(2, decoded.sdpMLineIndex); + assertEquals(CANDIDATE, decoded.sdp); + } + + @Test + void candidateWithoutMidParses() { + RTCIceCandidate decoded = CandidateJson.decode(" { \"candidate\" : \"a\\\"b\\u0041\" } "); + + assertNull(decoded.sdpMid); + assertEquals(0, decoded.sdpMLineIndex); + assertEquals("a\"bA", decoded.sdp); + } + + @Test + void candidateWithoutCandidateFails() { + assertThrows(IllegalArgumentException.class, () -> CandidateJson.decode("{\"sdpMid\":\"0\"}")); + assertThrows(IllegalArgumentException.class, () -> CandidateJson.decode("{\"candidate\":")); + } + + @Test + void rtcUriOfServerUrl() { + URI uri = SignalClient.rtcUri(URI.create("https://live.example.com/"), 16, true); + + assertEquals("wss", uri.getScheme()); + assertEquals("live.example.com", uri.getHost()); + assertEquals("/rtc", uri.getPath()); + assertTrue(uri.getQuery().contains("protocol=16"), uri::toString); + assertTrue(uri.getQuery().contains("auto_subscribe=1"), uri::toString); + } + + @Test + void rtcUriKeepsPortAndPath() { + URI uri = SignalClient.rtcUri(URI.create("ws://localhost:7880/lk"), 16, false); + + assertEquals("ws://localhost:7880/lk/rtc", uri.toString().substring(0, uri.toString().indexOf('?'))); + assertTrue(uri.getQuery().contains("auto_subscribe=0"), uri::toString); + } + + @Test + void rtcUriRefusesOtherSchemes() { + assertThrows(IllegalArgumentException.class, + () -> SignalClient.rtcUri(URI.create("ftp://example.com"), 16, true)); + } + +} diff --git a/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/TestTokens.java b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/TestTokens.java new file mode 100644 index 00000000..8426aa2e --- /dev/null +++ b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/TestTokens.java @@ -0,0 +1,63 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import java.nio.charset.StandardCharsets; +import java.util.Base64; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +/** + * Access tokens for tests, signed with the key and secret of a server started + * with --dev unless the system properties livekit.key and livekit.secret name + * others. Applications get their tokens from their backend instead. + */ +final class TestTokens { + + private TestTokens() { + } + + static String token(String identity, String room) { + String key = System.getProperty("livekit.key", "devkey"); + String secret = System.getProperty("livekit.secret", "secret"); + long now = System.currentTimeMillis() / 1000; + + String header = "{\"alg\":\"HS256\",\"typ\":\"JWT\"}"; + String claims = "{\"iss\":\"" + key + "\",\"sub\":\"" + identity + "\",\"name\":\"" + identity + "\"" + + ",\"nbf\":" + (now - 10) + ",\"exp\":" + (now + 600) + + ",\"video\":{\"room\":\"" + room + "\",\"roomJoin\":true,\"canPublish\":true" + + ",\"canSubscribe\":true,\"canPublishData\":true}}"; + + String unsigned = encode(header.getBytes(StandardCharsets.UTF_8)) + "." + + encode(claims.getBytes(StandardCharsets.UTF_8)); + + try { + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256")); + return unsigned + "." + encode(mac.doFinal(unsigned.getBytes(StandardCharsets.UTF_8))); + } + catch (Exception e) { + throw new IllegalStateException(e); + } + } + + private static String encode(byte[] data) { + return Base64.getUrlEncoder().withoutPadding().encodeToString(data); + } + +} From 485d4adff9406dc17413672243fe7824883eb7e7 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Fri, 20 Nov 2026 08:38:15 +0100 Subject: [PATCH 2/5] feat: publish simulcast video with dynacast in webrtc-livekit Video goes out in LiveKit's q/h/f layers, the layers no subscriber wants pause on the server's quality updates, and subscribers can request a layer. --- .../onvoid/webrtc/livekit/Publication.java | 20 +- .../onvoid/webrtc/livekit/PublishOptions.java | 239 ++++++++++++++++++ .../java/dev/onvoid/webrtc/livekit/Room.java | 24 ++ .../dev/onvoid/webrtc/livekit/RoomImpl.java | 147 +++++++++-- .../onvoid/webrtc/livekit/VideoQuality.java | 74 ++++++ .../webrtc/livekit/PublishOptionsTests.java | 78 ++++++ .../webrtc/livekit/RoomIntegrationTest.java | 83 +++++- 7 files changed, 647 insertions(+), 18 deletions(-) create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/PublishOptions.java create mode 100644 webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/VideoQuality.java create mode 100644 webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/PublishOptionsTests.java diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Publication.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Publication.java index 7ee3d9cc..f6aa0f0d 100644 --- a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Publication.java +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Publication.java @@ -16,7 +16,10 @@ package dev.onvoid.webrtc.livekit; +import java.util.List; + import dev.onvoid.webrtc.client.LocalTrack; +import dev.onvoid.webrtc.client.Sender; /** * A local track the server accepted and that is negotiated to it. @@ -29,11 +32,14 @@ public final class Publication { private final LocalTrack track; + private final Sender sender; + - Publication(String sid, TrackSource source, LocalTrack track) { + Publication(String sid, TrackSource source, LocalTrack track, Sender sender) { this.sid = sid; this.source = source; this.track = track; + this.sender = sender; } /** @@ -57,6 +63,18 @@ public LocalTrack track() { return track; } + /** + * @return the rids of the simulcast layers the track is sent in, none + * without simulcast. + */ + public List rids() { + return sender.rids(); + } + + Sender sender() { + return sender; + } + @Override public String toString() { return "Publication[" + sid + ", " + source + ", " + track.id() + "]"; diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/PublishOptions.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/PublishOptions.java new file mode 100644 index 00000000..b9a7172a --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/PublishOptions.java @@ -0,0 +1,239 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import static java.util.Objects.requireNonNull; + +import java.util.ArrayList; +import java.util.List; + +import dev.onvoid.webrtc.client.Encoding; +import dev.onvoid.webrtc.client.Simulcast; + +/** + * How a track is published. + *

{@code
+ * room.publish(camera, PublishOptions.builder()
+ *         .source(TrackSource.CAMERA)
+ *         .videoSize(1280, 720)
+ *         .simulcast(true)
+ *         .build());
+ * }
+ */ +public final class PublishOptions { + + final TrackSource source; + + final String name; + + final int width; + + final int height; + + final boolean simulcast; + + + private PublishOptions(Builder builder) { + this.source = builder.source; + this.name = builder.name; + this.width = builder.width; + this.height = builder.height; + this.simulcast = builder.simulcast; + } + + /** + * @return a builder of a track of unknown source, without simulcast. + */ + public static Builder builder() { + return new Builder(); + } + + /** + * The layers a video of this size is sent in, as LiveKit's own clients + * send them: from the smallest up, with the rids {@code q}, {@code h} and + * {@code f}, and the bitrates of LiveKit's presets. + * + * @return the layers, none if the video is too small for simulcast. + */ + List layers() { + List layers = new ArrayList<>(); + + if (!simulcast || height < 360) { + return layers; + } + + int[] scales = height >= 720 ? new int[] { 4, 2, 1 } : new int[] { 2, 1 }; + for (int i = 0; i < scales.length; i++) { + int layerWidth = ceilDiv(width, scales[i]); + int layerHeight = ceilDiv(height, scales[i]); + + layers.add(new Layer(VideoQuality.values()[i], scales[i], layerWidth, layerHeight, + bitrate(layerHeight))); + } + + return layers; + } + + /** + * @return the layers as the client API sends them, or null without + * simulcast. + */ + Simulcast simulcastLayers() { + List layers = layers(); + if (layers.isEmpty()) { + return null; + } + + Simulcast.Builder builder = Simulcast.builder(); + for (Layer layer : layers) { + builder.layer(layer.quality.rid(), Encoding.builder() + .scaleDownBy(layer.scale) + .maxBitrate(layer.bitrate) + .build()); + } + return builder.build(); + } + + /** The bitrate of LiveKit's video preset for this height. */ + private static int bitrate(int height) { + if (height >= 1080) { + return 3_000_000; + } + if (height >= 720) { + return 1_700_000; + } + if (height >= 540) { + return 800_000; + } + if (height >= 360) { + return 450_000; + } + return 160_000; + } + + private static int ceilDiv(int value, int divisor) { + return (value + divisor - 1) / divisor; + } + + + + static final class Layer { + + final VideoQuality quality; + + final int scale; + + final int width; + + final int height; + + final int bitrate; + + + Layer(VideoQuality quality, int scale, int width, int height, int bitrate) { + this.quality = quality; + this.scale = scale; + this.width = width; + this.height = height; + this.bitrate = bitrate; + } + } + + + + /** Builds {@link PublishOptions}. */ + public static final class Builder { + + private TrackSource source = TrackSource.UNKNOWN; + + private String name; + + private int width; + + private int height; + + private boolean simulcast; + + + private Builder() { + } + + /** + * @param source where the track comes from. + * + * @return this builder. + */ + public Builder source(TrackSource source) { + this.source = requireNonNull(source, "source"); + return this; + } + + /** + * @param name the name other participants see, the track's id by + * default. + * + * @return this builder. + */ + public Builder name(String name) { + this.name = requireNonNull(name, "name"); + return this; + } + + /** + * @param width the width the video is captured in. + * @param height the height the video is captured in. + * + * @return this builder. + */ + public Builder videoSize(int width, int height) { + if (width <= 0 || height <= 0) { + throw new IllegalArgumentException("The size must be positive, got " + width + "x" + height); + } + this.width = width; + this.height = height; + return this; + } + + /** + * Sends a video track in simulcast layers, so that the server can + * forward each subscriber the one its connection and view take: three + * from 720 lines up, two from 360 lines up, none below. Needs the + * {@link #videoSize video size}. + * + * @param simulcast true to send the layers. + * + * @return this builder. + */ + public Builder simulcast(boolean simulcast) { + this.simulcast = simulcast; + return this; + } + + /** + * @return the options. + * + * @throws IllegalStateException if simulcast lacks the video size. + */ + public PublishOptions build() { + if (simulcast && height == 0) { + throw new IllegalStateException("Simulcast needs the video size"); + } + return new PublishOptions(this); + } + } + +} diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java index 79fce90f..3f03baaf 100644 --- a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java @@ -68,6 +68,30 @@ public interface Room extends AutoCloseable { */ CompletableFuture publish(LocalTrack track, TrackSource source); + /** + * Publishes a local track as the options say, such as a video track in + * simulcast layers. With simulcast the server pauses the layers no + * subscriber wants (LiveKit's dynacast), and resumes them once one does. + * + * @param track the track, which stays the application's to close. + * @param options how to publish it. + * + * @return a future with the publication, which fails if the server + * refuses the track or does not answer in time. + */ + CompletableFuture publish(LocalTrack track, PublishOptions options); + + /** + * Asks the server to forward a remote video track in another simulcast + * layer. The server forwards a smaller one while the connection takes + * no more. + * + * @param trackSid the server's id of the remote track, its {@link + * RemoteTrack#id() id}. + * @param quality the layer. + */ + void requestVideoQuality(String trackSid, VideoQuality quality); + /** * Sends data to every other participant of the room. * diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java index 71507815..910cd382 100644 --- a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java @@ -23,6 +23,7 @@ import java.net.http.HttpClient; import java.time.Duration; import java.util.ArrayList; +import java.util.EnumMap; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -48,12 +49,15 @@ import dev.onvoid.webrtc.RTCSessionDescription; import dev.onvoid.webrtc.client.ChannelOptions; import dev.onvoid.webrtc.client.DataChannel; +import dev.onvoid.webrtc.client.Encoding; import dev.onvoid.webrtc.client.LocalTrack; import dev.onvoid.webrtc.client.MediaKind; import dev.onvoid.webrtc.client.Negotiation; import dev.onvoid.webrtc.client.Peer; import dev.onvoid.webrtc.client.PeerOptions; import dev.onvoid.webrtc.client.RemoteTrack; +import dev.onvoid.webrtc.client.Sender; +import dev.onvoid.webrtc.client.Simulcast; import dev.onvoid.webrtc.client.SignalMessage; import dev.onvoid.webrtc.client.Subscription; import dev.onvoid.webrtc.client.WebRtc; @@ -62,6 +66,7 @@ import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.ParticipantInfo; import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.TrackType; import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.UserPacket; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.VideoLayer; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.AddTrackRequest; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.ICEServer; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.JoinResponse; @@ -72,8 +77,12 @@ import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SignalRequest; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SignalResponse; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SignalTarget; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SubscribedCodec; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SubscribedQuality; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SubscribedQualityUpdate; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.TrackPublishedResponse; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.TrickleRequest; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.UpdateTrackSettings; /** * The room: the state LiveKit's signaling describes, and the publisher and @@ -129,6 +138,12 @@ final class RoomImpl implements Room, SignalClient.Handler { private final Map pendingPublishes = new HashMap<>(); + /** The published tracks, by sid. */ + private final Map publications = new HashMap<>(); + + /** Hear which layers dynacast left on, once applied. For tests. */ + final List>> layerListeners = new CopyOnWriteArrayList<>(); + private Peer publisher; private Peer subscriber; @@ -223,17 +238,46 @@ public List remoteParticipants() { @Override public CompletableFuture publish(LocalTrack track, TrackSource source) { - requireNonNull(track, "track"); requireNonNull(source, "source"); + return publish(track, PublishOptions.builder().source(source).build()); + } + + @Override + public CompletableFuture publish(LocalTrack track, PublishOptions options) { + requireNonNull(track, "track"); + requireNonNull(options, "options"); + + if (options.simulcast && track.kind() != MediaKind.VIDEO) { + throw new IllegalArgumentException("Simulcast takes a video track, got " + track.kind()); + } + CompletableFuture result = new CompletableFuture<>(); result.orTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS); - post(() -> startPublish(track, source, result), result); + post(() -> startPublish(track, options, result), result); return result; } + @Override + public void requestVideoQuality(String trackSid, VideoQuality quality) { + requireNonNull(trackSid, "trackSid"); + requireNonNull(quality, "quality"); + + executor.execute(() -> { + if (state != RoomState.CONNECTED) { + return; + } + + signal.send(SignalRequest.newBuilder() + .setTrackSetting(UpdateTrackSettings.newBuilder() + .addTrackSids(trackSid) + .setQuality(quality.proto())) + .build()); + }); + } + @Override public CompletableFuture publishData(byte[] payload, String topic, boolean reliable) { requireNonNull(payload, "payload"); @@ -318,6 +362,9 @@ public void onSignal(SignalResponse response) { case TRACK_PUBLISHED: onTrackPublished(response.getTrackPublished()); break; + case SUBSCRIBED_QUALITY_UPDATE: + onSubscribedQualities(response.getSubscribedQualityUpdate()); + break; case LEAVE: LeaveRequest leave = response.getLeave(); disconnect("The server ended the session (" + leave.getReason() + ", " + leave.getAction() + ")", @@ -560,7 +607,7 @@ private void onOffer(SessionDescription offer) { // Publishing. - private void startPublish(LocalTrack track, TrackSource source, CompletableFuture result) { + private void startPublish(LocalTrack track, PublishOptions options, CompletableFuture result) { if (state != RoomState.CONNECTED) { result.completeExceptionally(new LiveKitException("The room is " + state)); return; @@ -573,16 +620,36 @@ private void startPublish(LocalTrack track, TrackSource source, CompletableFutur return; } - pendingPublishes.put(cid, new PendingPublish(track, source, result)); + pendingPublishes.put(cid, new PendingPublish(track, options, result)); result.whenCompleteAsync((p, e) -> pendingPublishes.remove(cid), executor); - signal.send(SignalRequest.newBuilder() - .setAddTrack(AddTrackRequest.newBuilder() - .setCid(cid) - .setName(track.id()) - .setType(track.kind() == MediaKind.VIDEO ? TrackType.VIDEO : TrackType.AUDIO) - .setSource(source.proto())) - .build()); + AddTrackRequest.Builder request = AddTrackRequest.newBuilder() + .setCid(cid) + .setName(options.name != null ? options.name : track.id()) + .setType(track.kind() == MediaKind.VIDEO ? TrackType.VIDEO : TrackType.AUDIO) + .setSource(options.source.proto()); + + if (track.kind() == MediaKind.VIDEO && options.height > 0) { + request.setWidth(options.width).setHeight(options.height); + + List layers = options.layers(); + if (layers.isEmpty()) { + request.addLayers(VideoLayer.newBuilder() + .setQuality(VideoQuality.HIGH.proto()) + .setWidth(options.width) + .setHeight(options.height)); + } + for (PublishOptions.Layer layer : layers) { + request.addLayers(VideoLayer.newBuilder() + .setQuality(layer.quality.proto()) + .setRid(layer.quality.rid()) + .setWidth(layer.width) + .setHeight(layer.height) + .setBitrate(layer.bitrate)); + } + } + + signal.send(SignalRequest.newBuilder().setAddTrack(request).build()); } private void onTrackPublished(TrackPublishedResponse published) { @@ -591,26 +658,73 @@ private void onTrackPublished(TrackPublishedResponse published) { return; } + Sender sender; try { - publisher.addTrack(pending.track); + Simulcast simulcast = pending.options.simulcastLayers(); + sender = simulcast != null + ? publisher.addTrack(pending.track, simulcast) + : publisher.addTrack(pending.track); } catch (RuntimeException e) { pending.result.completeExceptionally(e); return; } - Publication publication = new Publication(published.getTrack().getSid(), pending.source, pending.track); + Publication publication = new Publication(published.getTrack().getSid(), pending.options.source, + pending.track, sender); negotiatePublisher().whenComplete((v, error) -> { if (error != null) { pending.result.completeExceptionally(error); } else { + publications.put(publication.sid(), publication); pending.result.complete(publication); } }); } + /** + * Dynacast: the server names the layers some subscriber wants. Sending + * the others would only cost bandwidth and CPU, so they pause. + */ + private void onSubscribedQualities(SubscribedQualityUpdate update) { + Publication publication = publications.get(update.getTrackSid()); + if (publication == null || publication.rids().isEmpty()) { + return; + } + + // One codec is published; older servers send the qualities without it. + List qualities = update.getSubscribedQualitiesList(); + for (SubscribedCodec codec : update.getSubscribedCodecsList()) { + qualities = codec.getQualitiesList(); + } + + Map enabled = new EnumMap<>(VideoQuality.class); + List> changes = new ArrayList<>(); + + for (SubscribedQuality quality : qualities) { + VideoQuality layer = VideoQuality.of(quality.getQuality()); + if (layer == null || !publication.rids().contains(layer.rid())) { + continue; + } + + enabled.put(layer, quality.getEnabled()); + changes.add(publication.sender().configure(layer.rid(), + Encoding.builder().active(quality.getEnabled()).build())); + } + + CompletableFuture.allOf(changes.toArray(new CompletableFuture[0])).whenCompleteAsync((v, error) -> { + if (error != null) { + LOG.log(Level.WARNING, "Pausing or resuming layers of " + publication.sid() + " failed", error); + return; + } + for (BiConsumer> listener : layerListeners) { + call(() -> listener.accept(publication, enabled)); + } + }, executor); + } + // Data. private void sendData(byte[] payload, String topic, boolean reliable, CompletableFuture result) { @@ -846,6 +960,7 @@ private void disconnect(String reason, boolean sendLeave) { waiters.forEach(waiter -> waiter.completeExceptionally(error)); orphanTracks.clear(); + publications.clear(); CompletableFuture publisherClosed = publisher != null ? publisher.closeAsync() @@ -906,14 +1021,14 @@ private static final class PendingPublish { final LocalTrack track; - final TrackSource source; + final PublishOptions options; final CompletableFuture result; - PendingPublish(LocalTrack track, TrackSource source, CompletableFuture result) { + PendingPublish(LocalTrack track, PublishOptions options, CompletableFuture result) { this.track = track; - this.source = source; + this.options = options; this.result = result; } } diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/VideoQuality.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/VideoQuality.java new file mode 100644 index 00000000..6106d941 --- /dev/null +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/VideoQuality.java @@ -0,0 +1,74 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels; + +/** + * A simulcast layer of a video track, by LiveKit's names: the server forwards + * each subscriber one of them. + */ +public enum VideoQuality { + + /** The smallest layer, rid {@code q}. */ + LOW(LivekitModels.VideoQuality.LOW, "q"), + + /** The middle layer, rid {@code h}. */ + MEDIUM(LivekitModels.VideoQuality.MEDIUM, "h"), + + /** The largest layer, rid {@code f}. */ + HIGH(LivekitModels.VideoQuality.HIGH, "f"); + + private final LivekitModels.VideoQuality proto; + + private final String rid; + + VideoQuality(LivekitModels.VideoQuality proto, String rid) { + this.proto = proto; + this.rid = rid; + } + + LivekitModels.VideoQuality proto() { + return proto; + } + + /** + * @return the rid LiveKit's clients send this layer with. + */ + String rid() { + return rid; + } + + static VideoQuality of(LivekitModels.VideoQuality proto) { + for (VideoQuality quality : values()) { + if (quality.proto == proto) { + return quality; + } + } + return null; + } + + static VideoQuality ofRid(String rid) { + for (VideoQuality quality : values()) { + if (quality.rid.equals(rid)) { + return quality; + } + } + return null; + } + +} diff --git a/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/PublishOptionsTests.java b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/PublishOptionsTests.java new file mode 100644 index 00000000..382b94bd --- /dev/null +++ b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/PublishOptionsTests.java @@ -0,0 +1,78 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.stream.Collectors; + +import org.junit.jupiter.api.Test; + +class PublishOptionsTests { + + @Test + void threeLayersFrom720() { + List layers = layers(1280, 720); + + assertEquals("LOW 320x180 160000, MEDIUM 640x360 450000, HIGH 1280x720 1700000", describe(layers)); + assertEquals(List.of("q", "h", "f"), options(1280, 720).simulcastLayers().rids()); + } + + @Test + void twoLayersFrom360() { + assertEquals("LOW 320x180 160000, MEDIUM 640x360 450000", describe(layers(640, 360))); + assertEquals("LOW 427x270 160000, MEDIUM 853x540 800000", describe(layers(853, 540))); + } + + @Test + void noLayersBelow360() { + assertTrue(layers(480, 270).isEmpty()); + assertNull(options(480, 270).simulcastLayers()); + } + + @Test + void noLayersWithoutSimulcast() { + PublishOptions options = PublishOptions.builder().videoSize(1920, 1080).build(); + + assertTrue(options.layers().isEmpty()); + } + + @Test + void simulcastNeedsSize() { + assertThrows(IllegalStateException.class, () -> PublishOptions.builder().simulcast(true).build()); + assertThrows(IllegalArgumentException.class, () -> PublishOptions.builder().videoSize(0, 720)); + } + + private static PublishOptions options(int width, int height) { + return PublishOptions.builder().videoSize(width, height).simulcast(true).build(); + } + + private static List layers(int width, int height) { + return options(width, height).layers(); + } + + private static String describe(List layers) { + return layers.stream() + .map(layer -> layer.quality + " " + layer.width + "x" + layer.height + " " + layer.bitrate) + .collect(Collectors.joining(", ")); + } + +} diff --git a/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/RoomIntegrationTest.java b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/RoomIntegrationTest.java index 93b401fa..c7377391 100644 --- a/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/RoomIntegrationTest.java +++ b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/RoomIntegrationTest.java @@ -25,12 +25,17 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.util.Arrays; +import java.util.List; +import java.util.Map; import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ExecutionException; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.function.BiConsumer; +import java.util.function.IntPredicate; +import java.util.function.Predicate; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; @@ -41,6 +46,7 @@ import dev.onvoid.webrtc.client.CustomVideo; import dev.onvoid.webrtc.client.MediaKind; import dev.onvoid.webrtc.client.RemoteTrack; +import dev.onvoid.webrtc.client.Subscription; import dev.onvoid.webrtc.client.WebRtc; import dev.onvoid.webrtc.media.video.NativeI420Buffer; import dev.onvoid.webrtc.media.video.VideoFrame; @@ -133,6 +139,51 @@ void subscriberReceivesPublishedVideo() throws Exception { video.close(); } + @Test + void simulcastFollowsSubscribers() throws Exception { + Room alice = join("alice"); + Room bob = join("bob"); + + CompletableFuture received = new CompletableFuture<>(); + bob.onTrackSubscribed((participant, track) -> received.complete(track)); + + CustomVideo video = rtc.media().customVideo("camera-" + UUID.randomUUID()); + frames.scheduleAtFixedRate(() -> push(video, 1280, 720), 0, 66, TimeUnit.MILLISECONDS); + + Publication publication = await(alice.publish(video, PublishOptions.builder() + .source(TrackSource.CAMERA) + .videoSize(1280, 720) + .simulcast(true) + .build())); + assertEquals(List.of("q", "h", "f"), publication.rids()); + + RemoteTrack track = await(received); + + // Bob wants the smallest layer: the server forwards it, and, as + // nobody else watches, has Alice pause the larger ones. + CompletableFuture> paused = layersOnce(alice, layers -> !layers.get(VideoQuality.HIGH)); + CompletableFuture small = frameWidth(track, width -> width == 320); + bob.requestVideoQuality(track.id(), VideoQuality.LOW); + + assertEquals(320, await(small)); + Map layers = await(paused); + assertTrue(layers.get(VideoQuality.LOW), layers::toString); + assertFalse(layers.get(VideoQuality.HIGH), layers::toString); + + // And back up: the larger layers resume, and Bob gets more than the + // smallest once the bandwidth estimate allows it. + CompletableFuture> resumed = layersOnce(alice, l -> l.get(VideoQuality.HIGH)); + CompletableFuture large = frameWidth(track, width -> width >= 640); + bob.requestVideoQuality(track.id(), VideoQuality.HIGH); + + assertTrue(await(resumed).get(VideoQuality.HIGH)); + assertTrue(await(large) >= 640); + + bob.close(); + alice.close(); + video.close(); + } + @Test void dataReachesOtherParticipant() throws Exception { Room alice = join("alice"); @@ -185,8 +236,38 @@ private static T await(CompletableFuture future) throws Exception { return future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); } + private static CompletableFuture> layersOnce(Room room, + Predicate> condition) { + CompletableFuture> result = new CompletableFuture<>(); + List>> listeners = ((RoomImpl) room).layerListeners; + BiConsumer> listener = (publication, layers) -> { + if (layers.containsKey(VideoQuality.HIGH) && condition.test(layers)) { + result.complete(layers); + } + }; + listeners.add(listener); + result.whenComplete((r, e) -> listeners.remove(listener)); + return result; + } + + private static CompletableFuture frameWidth(RemoteTrack track, IntPredicate condition) { + CompletableFuture result = new CompletableFuture<>(); + Subscription subscription = track.asVideo().onFrame(frame -> { + int width = frame.buffer.getWidth(); + if (condition.test(width)) { + result.complete(width); + } + }); + result.whenComplete((r, e) -> subscription.close()); + return result; + } + private static void push(CustomVideo video) { - NativeI420Buffer buffer = NativeI420Buffer.allocate(320, 240); + push(video, 320, 240); + } + + private static void push(CustomVideo video, int width, int height) { + NativeI420Buffer buffer = NativeI420Buffer.allocate(width, height); VideoFrame frame = new VideoFrame(buffer, System.nanoTime()); try { From e4406b0f25ca21ec06153ae4c52a30e219b84577 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Fri, 20 Nov 2026 08:54:26 +0100 Subject: [PATCH 3/5] feat: resume or rejoin LiveKit rooms after a lost connection A lost signaling connection, a failed peer, missed pings or the server's leave resume the session with an ICE restart; a session the server cannot resume is joined anew, with the tracks published again. --- .../onvoid/webrtc/livekit/Publication.java | 40 +- .../java/dev/onvoid/webrtc/livekit/Room.java | 19 + .../dev/onvoid/webrtc/livekit/RoomImpl.java | 781 +++++++++++++++--- .../dev/onvoid/webrtc/livekit/RoomState.java | 7 + .../onvoid/webrtc/livekit/SignalClient.java | 23 +- .../livekit/ReconnectIntegrationTest.java | 227 +++++ 6 files changed, 972 insertions(+), 125 deletions(-) create mode 100644 webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/ReconnectIntegrationTest.java diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Publication.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Publication.java index f6aa0f0d..e6d51fec 100644 --- a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Publication.java +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Publication.java @@ -20,33 +20,41 @@ import dev.onvoid.webrtc.client.LocalTrack; import dev.onvoid.webrtc.client.Sender; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.TrackInfo; /** - * A local track the server accepted and that is negotiated to it. + * A local track the server accepted and that is negotiated to it. After a + * full reconnect the room publishes the track again, and the publication + * takes the server's new id. */ public final class Publication { - private final String sid; - private final TrackSource source; private final LocalTrack track; - private final Sender sender; + private final PublishOptions options; + + private volatile TrackInfo info; + + private volatile Sender sender; - Publication(String sid, TrackSource source, LocalTrack track, Sender sender) { - this.sid = sid; - this.source = source; + Publication(TrackInfo info, PublishOptions options, LocalTrack track, Sender sender) { + this.info = info; + this.source = options.source; + this.options = options; this.track = track; this.sender = sender; } /** * @return the server's id of the track, as other participants see it. + * It changes when the room publishes the track again after a full + * reconnect. */ public String sid() { - return sid; + return info.getSid(); } /** @@ -75,9 +83,23 @@ Sender sender() { return sender; } + TrackInfo info() { + return info; + } + + PublishOptions options() { + return options; + } + + /** Takes what publishing the track again in a new session gave. */ + void republished(TrackInfo info, Sender sender) { + this.info = info; + this.sender = sender; + } + @Override public String toString() { - return "Publication[" + sid + ", " + source + ", " + track.id() + "]"; + return "Publication[" + sid() + ", " + source + ", " + track.id() + "]"; } } diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java index 3f03baaf..450382aa 100644 --- a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java @@ -135,6 +135,25 @@ public interface Room extends AutoCloseable { */ Subscription onData(Consumer handler); + /** + * @param handler gets the reason each time the connection is lost and + * the room starts to get it back. + * + * @return removes the handler. + */ + Subscription onReconnecting(Consumer handler); + + /** + * @param handler runs each time the connection is back. If the session + * could not be resumed, the room joined anew: the local + * participant then has a new sid, the participants were + * reported as left and joined again, and the publications + * have new sids. + * + * @return removes the handler. + */ + Subscription onReconnected(Runnable handler); + /** * @param handler gets the reason once the room is left, for whatever * reason, the application's close included. diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java index 910cd382..47e9c9ac 100644 --- a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java @@ -55,10 +55,11 @@ import dev.onvoid.webrtc.client.Negotiation; import dev.onvoid.webrtc.client.Peer; import dev.onvoid.webrtc.client.PeerOptions; +import dev.onvoid.webrtc.client.PeerState; import dev.onvoid.webrtc.client.RemoteTrack; import dev.onvoid.webrtc.client.Sender; -import dev.onvoid.webrtc.client.Simulcast; import dev.onvoid.webrtc.client.SignalMessage; +import dev.onvoid.webrtc.client.Simulcast; import dev.onvoid.webrtc.client.Subscription; import dev.onvoid.webrtc.client.WebRtc; import dev.onvoid.webrtc.livekit.internal.proto.LivekitModels.DataPacket; @@ -77,19 +78,28 @@ import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SignalRequest; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SignalResponse; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SignalTarget; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SimulateScenario; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SubscribedCodec; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SubscribedQuality; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SubscribedQualityUpdate; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SyncState; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.TrackPublishedResponse; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.TrickleRequest; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.UpdateSubscription; import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.UpdateTrackSettings; /** * The room: the state LiveKit's signaling describes, and the publisher and * subscriber peers it negotiates. Everything but the handler lists is * confined to the room's single thread, which also runs the handlers. + *

+ * A lost connection is recovered as LiveKit's own clients do. A resume opens + * the signaling again for the same session and restarts ICE on both peers, + * which keeps the peers, their tracks and data channels. Should the server + * refuse it, or should it fail, the room joins anew, with new peers, and + * publishes its tracks again. */ -final class RoomImpl implements Room, SignalClient.Handler { +final class RoomImpl implements Room { private static final System.Logger LOG = System.getLogger(Room.class.getName()); @@ -97,17 +107,28 @@ final class RoomImpl implements Room, SignalClient.Handler { private static final String LOSSY_LABEL = "_lossy"; + /** Reconnect attempts before the room gives up, as LiveKit's clients make. */ + private static final int MAX_ATTEMPTS = 10; + + private static final long BACKOFF_STEP_MILLIS = 300; + + private static final long MAX_BACKOFF_MILLIS = 10_000; + private final WebRtc rtc; + private final URI url; + + private final boolean autoSubscribe; + private final Duration timeout; + private final HttpClient http = HttpClient.newHttpClient(); + private final ScheduledExecutorService thread; /** Runs on the room's thread, and drops what comes after the room is gone. */ private final Executor executor; - private final SignalClient signal; - private final CompletableFuture joined = new CompletableFuture<>(); private final CompletableFuture closed = new CompletableFuture<>(); @@ -120,12 +141,22 @@ final class RoomImpl implements Room, SignalClient.Handler { private final List> dataHandlers = new CopyOnWriteArrayList<>(); + private final List> reconnectingHandlers = new CopyOnWriteArrayList<>(); + + private final List reconnectedHandlers = new CopyOnWriteArrayList<>(); + private final List> disconnectHandlers = new CopyOnWriteArrayList<>(); + /** Hear which layers dynacast left on, once applied. For tests. */ + final List>> layerListeners = new CopyOnWriteArrayList<>(); + // Room thread only, unless volatile. private volatile RoomState state = RoomState.CONNECTING; + /** The token for the next connection; the server refreshes it. */ + private volatile String token; + private volatile String name = ""; private volatile Participant local; @@ -133,16 +164,23 @@ final class RoomImpl implements Room, SignalClient.Handler { /** By sid, in join order. Written on the room thread, read anywhere. */ private volatile Map participants = Map.of(); + /** Completes whenever the room is connected; operations wait on it. */ + private CompletableFuture connectedGate = new CompletableFuture<>(); + + private SignalClient signal; + + /** The first message the open signaling waits for: a join or reconnect response. */ + private CompletableFuture firstResponse; + + private SignalResponse.MessageCase firstResponseCase; + /** Tracks that arrived before the participant that publishes them, by participant sid. */ private final Map> orphanTracks = new HashMap<>(); private final Map pendingPublishes = new HashMap<>(); /** The published tracks, by sid. */ - private final Map publications = new HashMap<>(); - - /** Hear which layers dynacast left on, once applied. For tests. */ - final List>> layerListeners = new CopyOnWriteArrayList<>(); + private final Map publications = new LinkedHashMap<>(); private Peer publisher; @@ -158,6 +196,8 @@ final class RoomImpl implements Room, SignalClient.Handler { private boolean publisherNegotiated; + private boolean subscriberAnswered; + /** Wait for the offer that is out. */ private List> negotiationWaiters = new ArrayList<>(); @@ -168,13 +208,31 @@ final class RoomImpl implements Room, SignalClient.Handler { private int dataSequence = 1; + private int pingIntervalSeconds; + + private int pingTimeoutSeconds; + private ScheduledFuture pingTask; private long lastPongMillis; + // Recovery. + + private boolean fullReconnect; + + private int attempts; + + /** Counts the attempts, so that the steps of an abandoned one stop. */ + private int generation; + + private CompletableFuture currentAttempt; + - private RoomImpl(WebRtc rtc, Duration timeout) { + private RoomImpl(WebRtc rtc, URI url, String token, boolean autoSubscribe, Duration timeout) { this.rtc = rtc; + this.url = url; + this.token = token; + this.autoSubscribe = autoSubscribe; this.timeout = timeout; this.thread = Executors.newSingleThreadScheduledExecutor(task -> { Thread t = new Thread(task, "livekit-room"); @@ -189,29 +247,38 @@ private RoomImpl(WebRtc rtc, Duration timeout) { // The room is closed and every future of it completed. } }; - this.signal = new SignalClient(executor, this); } static CompletableFuture connect(WebRtc rtc, URI url, String token, boolean autoSubscribe, Duration timeout) { - URI uri = SignalClient.rtcUri(url, LiveKit.PROTOCOL, autoSubscribe); - RoomImpl room = new RoomImpl(rtc, timeout); - - room.executor.execute(() -> { - ScheduledFuture deadline = room.thread.schedule( - () -> room.failJoin(new LiveKitException("No answer to the join within " + timeout)), - timeout.toMillis(), TimeUnit.MILLISECONDS); - room.joined.whenComplete((r, e) -> deadline.cancel(false)); - - room.signal.open(HttpClient.newHttpClient(), uri, token, timeout) - .whenCompleteAsync((v, e) -> { - if (e != null) { - room.failJoin(e instanceof CompletionException && e.getCause() != null - ? e.getCause() - : e); - } - }, room.executor); - }); + // Fails here on a URL that is not one. + SignalClient.rtcUri(url, LiveKit.PROTOCOL, autoSubscribe); + + RoomImpl room = new RoomImpl(rtc, url, token, autoSubscribe, timeout); + + // Not async: the session is set up before the room thread takes the + // messages that follow the join, such as the server's first offer. + room.executor.execute(() -> room.openSignal(false).whenComplete((response, error) -> { + if (error != null) { + room.failJoin(unwrap(error)); + return; + } + if (room.joined.isDone()) { + return; + } + + try { + room.setUpSession(response.getJoin(), false); + } + catch (RuntimeException e) { + room.failJoin(e); + return; + } + + room.state = RoomState.CONNECTED; + room.connectedGate.complete(null); + room.joined.complete(room); + })); return room.joined.copy(); } @@ -255,7 +322,7 @@ public CompletableFuture publish(LocalTrack track, PublishOptions o CompletableFuture result = new CompletableFuture<>(); result.orTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS); - post(() -> startPublish(track, options, result), result); + post(() -> whenConnected(result, () -> startPublish(track, options, null, result)), result); return result; } @@ -286,7 +353,7 @@ public CompletableFuture publishData(byte[] payload, String topic, boolean CompletableFuture result = new CompletableFuture<>(); result.orTimeout(timeout.toMillis(), TimeUnit.MILLISECONDS); - post(() -> sendData(data, topic, reliable, result), result); + post(() -> whenConnected(result, () -> sendData(data, topic, reliable, result)), result); return result; } @@ -311,6 +378,16 @@ public Subscription onData(Consumer handler) { return subscribe(dataHandlers, handler); } + @Override + public Subscription onReconnecting(Consumer handler) { + return subscribe(reconnectingHandlers, handler); + } + + @Override + public Subscription onReconnected(Runnable handler) { + return subscribe(reconnectedHandlers, handler); + } + @Override public Subscription onDisconnected(Consumer handler) { return subscribe(disconnectHandlers, handler); @@ -335,17 +412,87 @@ public void close() { } } + // For tests. + + /** Cuts the signaling connection off, as a network failure would. */ + void cutSignal() { + executor.execute(() -> { + if (signal != null) { + signal.abort(); + } + }); + } + + /** Has the server act out a scenario, as a server started with --dev does. */ + void simulate(SimulateScenario scenario) { + executor.execute(() -> signal.send(SignalRequest.newBuilder().setSimulate(scenario).build())); + } + // Signaling, on the room thread. - @Override - public void onSignal(SignalResponse response) { + /** + * Opens a signaling connection, which replaces the one before, and waits + * for its first response: a join response for a new session, a reconnect + * response for a resumed one. The future completes on the room thread, + * while it handles that response, so that stages added without async + * run before the messages behind it. + */ + private CompletableFuture openSignal(boolean resume) { + SignalClient[] self = new SignalClient[1]; + SignalClient client = new SignalClient(executor, new SignalClient.Handler() { + + @Override + public void onSignal(SignalResponse response) { + if (signal == self[0]) { + RoomImpl.this.onSignal(response); + } + } + + @Override + public void onSignalClosed(String reason) { + if (signal == self[0]) { + RoomImpl.this.onSignalClosed(reason); + } + } + }); + self[0] = client; + + if (signal != null) { + signal.close(); + } + signal = client; + + CompletableFuture first = new CompletableFuture<>(); + firstResponse = first; + firstResponseCase = resume ? SignalResponse.MessageCase.RECONNECT : SignalResponse.MessageCase.JOIN; + + URI uri = SignalClient.rtcUri(url, LiveKit.PROTOCOL, autoSubscribe, resume ? local.sid() : null); + + client.open(http, uri, token, timeout).whenCompleteAsync((v, error) -> { + if (error != null) { + first.completeExceptionally(unwrap(error)); + } + }, executor); + + ScheduledFuture deadline = thread.schedule(() -> first.completeExceptionally( + new LiveKitException("No " + (resume ? "reconnect" : "join") + " response within " + timeout)), + timeout.toMillis(), TimeUnit.MILLISECONDS); + first.whenComplete((r, e) -> deadline.cancel(false)); + + return first; + } + + private void onSignal(SignalResponse response) { if (state == RoomState.DISCONNECTED) { return; } switch (response.getMessageCase()) { case JOIN: - onJoin(response.getJoin()); + case RECONNECT: + if (firstResponse != null && firstResponseCase == response.getMessageCase()) { + firstResponse.complete(response); + } break; case ANSWER: onAnswer(response.getAnswer()); @@ -366,13 +513,14 @@ public void onSignal(SignalResponse response) { onSubscribedQualities(response.getSubscribedQualityUpdate()); break; case LEAVE: - LeaveRequest leave = response.getLeave(); - disconnect("The server ended the session (" + leave.getReason() + ", " + leave.getAction() + ")", - false); + onLeave(response.getLeave()); break; case PONG_RESP: lastPongMillis = System.currentTimeMillis(); break; + case REFRESH_TOKEN: + token = response.getRefreshToken(); + break; case REQUEST_RESPONSE: RequestResponse answer = response.getRequestResponse(); if (answer.getReason() != RequestResponse.Reason.OK) { @@ -385,21 +533,58 @@ public void onSignal(SignalResponse response) { } } - @Override - public void onSignalClosed(String reason) { + private void onSignalClosed(String reason) { if (!joined.isDone()) { failJoin(new LiveKitException(reason)); + return; } - else { - disconnect(reason, false); + + if (firstResponse != null && !firstResponse.isDone()) { + firstResponse.completeExceptionally(new LiveKitException(reason)); + } + + if (state == RoomState.CONNECTED) { + recover(false, reason); + } + else if (state == RoomState.RECONNECTING && currentAttempt != null) { + currentAttempt.completeExceptionally(new LiveKitException(reason)); } } - private void onJoin(JoinResponse join) { - if (joined.isDone()) { + private void onLeave(LeaveRequest leave) { + String reason = "The server ended the session (" + leave.getReason() + ", " + leave.getAction() + ")"; + + if (!joined.isDone()) { + failJoin(new LiveKitException(reason)); return; } + LeaveRequest.Action action = leave.getAction(); + if (action == LeaveRequest.Action.DISCONNECT && leave.getCanReconnect()) { + // What older servers send for a reconnect. + action = LeaveRequest.Action.RECONNECT; + } + + switch (action) { + case RESUME: + recover(false, reason); + break; + case RECONNECT: + recover(true, reason); + break; + default: + disconnect(reason, false); + } + } + + /** + * Takes the response of a join: the room, the participants and the ICE + * servers, from which it makes the peers of the session. + * + * @param rejoined true if this is a full reconnect, whose participants + * are reported as joined again. + */ + private void setUpSession(JoinResponse join, boolean rejoined) { name = join.getRoom().getName(); local = new Participant(join.getParticipant()); @@ -409,35 +594,44 @@ private void onJoin(JoinResponse join) { } participants = others; - try { - publisher = createPeer(SignalTarget.PUBLISHER, join.getIceServersList()); - subscriber = createPeer(SignalTarget.SUBSCRIBER, join.getIceServersList()); + publisher = createPeer(SignalTarget.PUBLISHER, join.getIceServersList()); + subscriber = createPeer(SignalTarget.SUBSCRIBER, join.getIceServersList()); - // The channels go into the first offer of the publisher. - reliable = publisher.createChannel(RELIABLE_LABEL, ChannelOptions.builder().ordered(true).build()); - lossy = publisher.createChannel(LOSSY_LABEL, - ChannelOptions.builder().ordered(false).maxRetransmits(0).build()); - } - catch (RuntimeException e) { - failJoin(e); - return; - } + // The channels go into the first offer of the publisher. + reliable = publisher.createChannel(RELIABLE_LABEL, ChannelOptions.builder().ordered(true).build()); + lossy = publisher.createChannel(LOSSY_LABEL, + ChannelOptions.builder().ordered(false).maxRetransmits(0).build()); receiveData(reliable); receiveData(lossy); - subscriber.onTrack(track -> executor.execute(() -> onRemoteTrack(track))); - subscriber.onChannel(channel -> executor.execute(() -> receiveData(channel))); + Peer sessionSubscriber = subscriber; + sessionSubscriber.onTrack(track -> executor.execute(() -> { + if (sessionSubscriber == subscriber) { + onRemoteTrack(track); + } + })); + sessionSubscriber.onChannel(channel -> executor.execute(() -> { + if (sessionSubscriber == subscriber) { + receiveData(channel); + } + })); - startPing(join.getPingInterval(), join.getPingTimeout()); + pingIntervalSeconds = join.getPingInterval(); + pingTimeoutSeconds = join.getPingTimeout(); + startPing(); - state = RoomState.CONNECTED; + if (rejoined) { + for (Participant participant : others.values()) { + for (Consumer handler : joinedHandlers) { + call(() -> handler.accept(participant)); + } + } + } if (!join.getSubscriberPrimary() || join.getFastPublish()) { negotiatePublisher(); } - - joined.complete(this); } private Peer createPeer(SignalTarget target, List servers) { @@ -453,18 +647,24 @@ private Peer createPeer(SignalTarget target, List servers) { }); for (ICEServer server : servers) { - for (String url : server.getUrlsList()) { + for (String serverUrl : server.getUrlsList()) { if (server.getUsername().isEmpty()) { - options.iceServer(url); + options.iceServer(serverUrl); } else { - options.iceServer(url, server.getUsername(), server.getCredential()); + options.iceServer(serverUrl, server.getUsername(), server.getCredential()); } } } Peer peer = rtc.createPeer(options.build()); peer.onError(e -> LOG.log(Level.WARNING, "The " + target + " peer failed", e)); + peer.onStateChange(peerState -> executor.execute(() -> { + boolean current = peer == publisher || peer == subscriber; + if (current && peerState == PeerState.FAILED && state == RoomState.CONNECTED) { + recover(false, "The " + target.name().toLowerCase() + " connection failed"); + } + })); return peer; } @@ -522,8 +722,12 @@ private CompletableFuture negotiatePublisher() { private void sendOffer() { negotiating = true; + Peer peer = publisher; - publisher.negotiator().createOffer().whenCompleteAsync((offer, error) -> { + peer.negotiator().createOffer().whenCompleteAsync((offer, error) -> { + if (peer != publisher) { + return; + } if (error != null) { negotiationDone(error); return; @@ -546,8 +750,13 @@ private void onAnswer(SessionDescription answer) { return; } - publisher.negotiator().applyAnswer(new RTCSessionDescription(RTCSdpType.ANSWER, answer.getSdp())) - .whenCompleteAsync((v, error) -> negotiationDone(error), executor); + Peer peer = publisher; + peer.negotiator().applyAnswer(new RTCSessionDescription(RTCSdpType.ANSWER, answer.getSdp())) + .whenCompleteAsync((v, error) -> { + if (peer == publisher) { + negotiationDone(error); + } + }, executor); } private void negotiationDone(Throwable error) { @@ -579,6 +788,28 @@ private void negotiationDone(Throwable error) { } } + /** + * Offers again with ICE restarted, for a resumed session: the answer to an + * offer that was out may have been lost with the old connection, so its + * waiters wait for the new one. + */ + private void restartPublisherIce() { + List> waiters = new ArrayList<>(negotiationWaiters); + waiters.addAll(nextNegotiationWaiters); + negotiationWaiters = new ArrayList<>(); + nextNegotiationWaiters = new ArrayList<>(); + negotiating = false; + renegotiate = false; + + if (!publisherNegotiated && waiters.isEmpty()) { + return; + } + + publisher.restartIce(); + negotiationWaiters.addAll(waiters); + sendOffer(); + } + // The server offers, the subscriber answers. private void onOffer(SessionDescription offer) { @@ -586,16 +817,18 @@ private void onOffer(SessionDescription offer) { return; } - subscriber.negotiator().answer(new RTCSessionDescription(RTCSdpType.OFFER, offer.getSdp())) + Peer peer = subscriber; + peer.negotiator().answer(new RTCSessionDescription(RTCSdpType.OFFER, offer.getSdp())) .whenCompleteAsync((answer, error) -> { - if (error != null) { - LOG.log(Level.WARNING, "Answering the server's offer failed", error); + if (peer != subscriber || state == RoomState.DISCONNECTED) { return; } - if (state == RoomState.DISCONNECTED) { + if (error != null) { + LOG.log(Level.WARNING, "Answering the server's offer failed", error); return; } + subscriberAnswered = true; signal.send(SignalRequest.newBuilder() .setAnswer(SessionDescription.newBuilder() .setType("answer") @@ -607,12 +840,14 @@ private void onOffer(SessionDescription offer) { // Publishing. - private void startPublish(LocalTrack track, PublishOptions options, CompletableFuture result) { - if (state != RoomState.CONNECTED) { - result.completeExceptionally(new LiveKitException("The room is " + state)); - return; - } - + /** + * Asks the server to take a track. + * + * @param existing the publication to publish again in a new session, or + * null for a new one. + */ + private void startPublish(LocalTrack track, PublishOptions options, Publication existing, + CompletableFuture result) { // The server knows the track by this id until it names its own. String cid = track.id(); if (pendingPublishes.containsKey(cid)) { @@ -620,8 +855,13 @@ private void startPublish(LocalTrack track, PublishOptions options, CompletableF return; } - pendingPublishes.put(cid, new PendingPublish(track, options, result)); - result.whenCompleteAsync((p, e) -> pendingPublishes.remove(cid), executor); + pendingPublishes.put(cid, new PendingPublish(track, options, existing, result)); + result.whenCompleteAsync((p, e) -> { + PendingPublish pending = pendingPublishes.get(cid); + if (pending != null && pending.result == result) { + pendingPublishes.remove(cid); + } + }, executor); AddTrackRequest.Builder request = AddTrackRequest.newBuilder() .setCid(cid) @@ -670,17 +910,27 @@ private void onTrackPublished(TrackPublishedResponse published) { return; } - Publication publication = new Publication(published.getTrack().getSid(), pending.options.source, - pending.track, sender); + Publication publication; + if (pending.existing != null) { + publication = pending.existing; + publication.republished(published.getTrack(), sender); + } + else { + publication = new Publication(published.getTrack(), pending.options, pending.track, sender); + } + Peer peer = publisher; negotiatePublisher().whenComplete((v, error) -> { if (error != null) { pending.result.completeExceptionally(error); } - else { + else if (peer == publisher) { publications.put(publication.sid(), publication); pending.result.complete(publication); } + else { + pending.result.completeExceptionally(new LiveKitException("The session ended")); + } }); } @@ -728,11 +978,6 @@ private void onSubscribedQualities(SubscribedQualityUpdate update) { // Data. private void sendData(byte[] payload, String topic, boolean reliable, CompletableFuture result) { - if (state != RoomState.CONNECTED) { - result.completeExceptionally(new LiveKitException("The room is " + state)); - return; - } - UserPacket.Builder user = UserPacket.newBuilder().setPayload(ByteString.copyFrom(payload)); if (topic != null) { user.setTopic(topic); @@ -843,9 +1088,7 @@ else if (participant == null) { } } for (Participant participant : leftNow) { - for (Consumer handler : leftHandlers) { - call(() -> handler.accept(participant)); - } + fireLeft(participant); } } @@ -893,28 +1136,291 @@ private void trackSubscribed(Participant participant, RemoteTrack track) { } } + private void fireLeft(Participant participant) { + for (Consumer handler : leftHandlers) { + call(() -> handler.accept(participant)); + } + } + // Keepalive. - private void startPing(int intervalSeconds, int timeoutSeconds) { - if (intervalSeconds <= 0) { + private void startPing() { + stopPing(); + + if (pingIntervalSeconds <= 0) { return; } - long timeoutMillis = TimeUnit.SECONDS.toMillis(timeoutSeconds); + long timeoutMillis = TimeUnit.SECONDS.toMillis(pingTimeoutSeconds); lastPongMillis = System.currentTimeMillis(); pingTask = thread.scheduleAtFixedRate(() -> { long now = System.currentTimeMillis(); if (timeoutMillis > 0 && now - lastPongMillis > timeoutMillis) { - disconnect("The server did not answer pings for " + timeoutSeconds + " s", false); + recover(false, "The server did not answer pings for " + pingTimeoutSeconds + " s"); return; } signal.send(SignalRequest.newBuilder() .setPingReq(Ping.newBuilder().setTimestamp(now)) .build()); - }, intervalSeconds, intervalSeconds, TimeUnit.SECONDS); + }, pingIntervalSeconds, pingIntervalSeconds, TimeUnit.SECONDS); + } + + private void stopPing() { + if (pingTask != null) { + pingTask.cancel(false); + pingTask = null; + } + } + + // Recovery, on the room thread. + + /** + * Starts getting the connection back, unless the room already is. + * + * @param full true to join anew, as the server asks when it cannot + * resume the session. + */ + private void recover(boolean full, String reason) { + if (state == RoomState.DISCONNECTED || state == RoomState.CONNECTING) { + return; + } + + if (state == RoomState.RECONNECTING) { + if (full && !fullReconnect) { + fullReconnect = true; + if (currentAttempt != null) { + currentAttempt.completeExceptionally(new LiveKitException(reason)); + } + } + return; + } + + LOG.log(Level.INFO, "Reconnecting the room " + name + ": " + reason); + + state = RoomState.RECONNECTING; + connectedGate = new CompletableFuture<>(); + fullReconnect = full; + attempts = 0; + + stopPing(); + signal.close(); + + for (Consumer handler : reconnectingHandlers) { + call(() -> handler.accept(reason)); + } + + nextAttempt(reason); + } + + private void nextAttempt(String lastFailure) { + if (state != RoomState.RECONNECTING) { + return; + } + if (attempts >= MAX_ATTEMPTS) { + disconnect("Reconnecting failed " + attempts + " times, last: " + lastFailure, false); + return; + } + + long delay = Math.min(BACKOFF_STEP_MILLIS * attempts * attempts, MAX_BACKOFF_MILLIS); + attempts++; + + thread.schedule(this::runAttempt, delay, TimeUnit.MILLISECONDS); + } + + private void runAttempt() { + if (state != RoomState.RECONNECTING) { + return; + } + + int attemptGeneration = ++generation; + boolean full = fullReconnect; + CompletableFuture attempt = new CompletableFuture<>(); + currentAttempt = attempt; + + (full ? restartSession(attemptGeneration) : resumeSession(attemptGeneration)).whenComplete((v, error) -> { + if (error != null) { + attempt.completeExceptionally(unwrap(error)); + } + else { + attempt.complete(null); + } + }); + + attempt.orTimeout(timeout.toMillis() * 2, TimeUnit.MILLISECONDS).whenCompleteAsync((v, error) -> { + if (attempt != currentAttempt || state != RoomState.RECONNECTING) { + return; + } + currentAttempt = null; + + if (error == null) { + reconnected(full); + return; + } + + String failure = String.valueOf(unwrap(error).getMessage()); + LOG.log(Level.INFO, (full ? "Rejoining" : "Resuming") + " the room " + name + " failed: " + failure); + + // A session that cannot be resumed is joined anew. + fullReconnect = true; + nextAttempt(failure); + }, executor); + } + + /** + * Resumes the session: the signaling opens for the same participant, the + * publisher restarts ICE, the server restarts it on the subscriber with an + * offer, and once both are connected the server hears what this side has. + */ + private CompletableFuture resumeSession(int attemptGeneration) { + return openSignal(true) + .thenCompose(response -> { + checkGeneration(attemptGeneration); + startPing(); + restartPublisherIce(); + return whenPeersConnected(); + }) + .thenComposeAsync(v -> { + checkGeneration(attemptGeneration); + return sendSyncState(); + }, executor); + } + + /** + * Joins anew: the old peers go, the participants are reported as left, + * and the published tracks are published again on new peers. + */ + private CompletableFuture restartSession(int attemptGeneration) { + endSession(new LiveKitException("The session is joined anew")); + + return openSignal(false) + .thenCompose(response -> { + checkGeneration(attemptGeneration); + setUpSession(response.getJoin(), true); + return republish(); + }); + } + + /** Drops the peers and participants of the session, for a new one. */ + private void endSession(LiveKitException error) { + List left = new ArrayList<>(participants.values()); + participants = Map.of(); + orphanTracks.clear(); + left.forEach(this::fireLeft); + + failPending(error); + + if (publisher != null) { + publisher.closeAsync(); + } + if (subscriber != null) { + subscriber.closeAsync(); + } + publisher = null; + subscriber = null; + reliable = null; + lossy = null; + negotiating = false; + renegotiate = false; + publisherNegotiated = false; + subscriberAnswered = false; + } + + private CompletableFuture republish() { + List previous = new ArrayList<>(publications.values()); + publications.clear(); + + List> republished = new ArrayList<>(); + for (Publication publication : previous) { + CompletableFuture result = new CompletableFuture<>(); + startPublish(publication.track(), publication.options(), publication, result); + republished.add(result); + } + + return CompletableFuture.allOf(republished.toArray(new CompletableFuture[0])); + } + + private CompletableFuture whenPeersConnected() { + List> connected = new ArrayList<>(); + + if (publisherNegotiated || negotiating) { + connected.add(whenConnected(publisher)); + } + if (subscriberAnswered) { + connected.add(whenConnected(subscriber)); + } + + return CompletableFuture.allOf(connected.toArray(new CompletableFuture[0])); + } + + private static CompletableFuture whenConnected(Peer peer) { + CompletableFuture connected = new CompletableFuture<>(); + Subscription subscription = peer.onStateChange(peerState -> { + if (peerState == PeerState.CONNECTED) { + connected.complete(null); + } + }); + connected.whenComplete((v, e) -> subscription.close()); + + if (peer.state() == PeerState.CONNECTED) { + connected.complete(null); + } + + return connected; + } + + /** + * Tells the server what the subscriber negotiated, what it subscribes to + * and what this side publishes, as a resumed session needs when it moved + * to another server. + */ + private CompletableFuture sendSyncState() { + if (!subscriberAnswered) { + return CompletableFuture.completedFuture(null); + } + + return subscriber.withNative(connection -> new RTCSessionDescription[] { + connection.getRemoteDescription(), connection.getLocalDescription() }) + .thenAcceptAsync(descriptions -> { + if (descriptions[0] == null || descriptions[1] == null) { + return; + } + + SyncState.Builder sync = SyncState.newBuilder() + .setOffer(SessionDescription.newBuilder().setType("offer").setSdp(descriptions[0].sdp)) + .setAnswer(SessionDescription.newBuilder().setType("answer").setSdp(descriptions[1].sdp)) + // With auto subscribe, the tracks listed would be those unsubscribed. + .setSubscription(UpdateSubscription.newBuilder().setSubscribe(!autoSubscribe)); + + for (Publication publication : publications.values()) { + sync.addPublishTracks(TrackPublishedResponse.newBuilder() + .setCid(publication.track().id()) + .setTrack(publication.info())); + } + + signal.send(SignalRequest.newBuilder().setSyncState(sync).build()); + }, executor); + } + + private void reconnected(boolean full) { + LOG.log(Level.INFO, (full ? "Rejoined" : "Resumed") + " the room " + name); + + state = RoomState.CONNECTED; + attempts = 0; + fullReconnect = false; + connectedGate.complete(null); + + for (Runnable handler : reconnectedHandlers) { + call(handler); + } + } + + private void checkGeneration(int attemptGeneration) { + if (attemptGeneration != generation || state != RoomState.RECONNECTING) { + throw new LiveKitException("The attempt was abandoned"); + } } // Leaving. @@ -932,33 +1438,32 @@ private void disconnect(String reason, boolean sendLeave) { if (state == RoomState.DISCONNECTED) { return; } + boolean wasConnected = state == RoomState.CONNECTED; state = RoomState.DISCONNECTED; - if (pingTask != null) { - pingTask.cancel(false); - } - if (sendLeave) { - signal.send(SignalRequest.newBuilder() - .setLeave(LeaveRequest.newBuilder() - .setReason(DisconnectReason.CLIENT_INITIATED) - .setAction(LeaveRequest.Action.DISCONNECT)) - .build()); + stopPing(); + if (signal != null) { + if (sendLeave && wasConnected) { + signal.send(SignalRequest.newBuilder() + .setLeave(LeaveRequest.newBuilder() + .setReason(DisconnectReason.CLIENT_INITIATED) + .setAction(LeaveRequest.Action.DISCONNECT)) + .build()); + } + signal.close(); } - signal.close(); LiveKitException error = new LiveKitException("The room was left: " + reason); joined.completeExceptionally(error); + connectedGate.completeExceptionally(error); + if (firstResponse != null) { + firstResponse.completeExceptionally(error); + } + if (currentAttempt != null) { + currentAttempt.completeExceptionally(error); + } - List publishes = new ArrayList<>(pendingPublishes.values()); - pendingPublishes.clear(); - publishes.forEach(pending -> pending.result.completeExceptionally(error)); - - List> waiters = new ArrayList<>(negotiationWaiters); - waiters.addAll(nextNegotiationWaiters); - negotiationWaiters.clear(); - nextNegotiationWaiters.clear(); - waiters.forEach(waiter -> waiter.completeExceptionally(error)); - + failPending(error); orphanTracks.clear(); publications.clear(); @@ -989,8 +1494,43 @@ private void disconnect(String reason, boolean sendLeave) { })); } + /** Fails the publishes and negotiations of the session. */ + private void failPending(LiveKitException error) { + List publishes = new ArrayList<>(pendingPublishes.values()); + pendingPublishes.clear(); + publishes.forEach(pending -> pending.result.completeExceptionally(error)); + + List> waiters = new ArrayList<>(negotiationWaiters); + waiters.addAll(nextNegotiationWaiters); + negotiationWaiters = new ArrayList<>(); + nextNegotiationWaiters = new ArrayList<>(); + waiters.forEach(waiter -> waiter.completeExceptionally(error)); + } + // Helpers. + /** + * Runs an operation on the room thread once the room is connected, at + * once if it is, and fails the future if the room is left first. + */ + private void whenConnected(CompletableFuture result, Runnable operation) { + connectedGate.whenCompleteAsync((v, error) -> { + if (result.isDone()) { + return; + } + if (error != null) { + result.completeExceptionally(unwrap(error)); + return; + } + if (state != RoomState.CONNECTED) { + // Lost again before the operation got its turn: wait for the next. + whenConnected(result, operation); + return; + } + operation.run(); + }, executor); + } + private void post(Runnable task, CompletableFuture future) { try { thread.execute(task); @@ -1000,6 +1540,13 @@ private void post(Runnable task, CompletableFuture future) { } } + private static Throwable unwrap(Throwable error) { + while (error instanceof CompletionException && error.getCause() != null) { + error = error.getCause(); + } + return error; + } + private static Subscription subscribe(List handlers, T handler) { requireNonNull(handler, "handler"); handlers.add(handler); @@ -1023,12 +1570,16 @@ private static final class PendingPublish { final PublishOptions options; + final Publication existing; + final CompletableFuture result; - PendingPublish(LocalTrack track, PublishOptions options, CompletableFuture result) { + PendingPublish(LocalTrack track, PublishOptions options, Publication existing, + CompletableFuture result) { this.track = track; this.options = options; + this.existing = existing; this.result = result; } } diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomState.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomState.java index f732bf9e..88befed9 100644 --- a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomState.java +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomState.java @@ -27,6 +27,13 @@ public enum RoomState { /** The room is joined. */ CONNECTED, + /** + * The connection to the server was lost, and the room is getting it back: + * by resuming the session, or by joining anew and publishing the tracks + * again. Publishing waits until it is back. + */ + RECONNECTING, + /** The room is left, by the application, the server or a failure. Final. */ DISCONNECTED diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/SignalClient.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/SignalClient.java index b5426147..3a959f86 100644 --- a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/SignalClient.java +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/SignalClient.java @@ -83,6 +83,14 @@ interface Handler { * out, such as wss://example.com, with https and http taken as wss and ws. */ static URI rtcUri(URI url, int protocol, boolean autoSubscribe) { + return rtcUri(url, protocol, autoSubscribe, null); + } + + /** + * @param resumeSid the participant sid of the session to resume, or null + * to join anew. + */ + static URI rtcUri(URI url, int protocol, boolean autoSubscribe, String resumeSid) { String scheme = url.getScheme() == null ? "" : url.getScheme().toLowerCase(Locale.ROOT); switch (scheme) { case "https": scheme = "wss"; break; @@ -102,7 +110,8 @@ static URI rtcUri(URI url, int protocol, boolean autoSubscribe) { + "&sdk=" + encode("java") + "&version=" + encode(LiveKit.VERSION) + "&os=" + encode(System.getProperty("os.name", "")) - + "&auto_subscribe=" + (autoSubscribe ? 1 : 0); + + "&auto_subscribe=" + (autoSubscribe ? 1 : 0) + + (resumeSid == null ? "" : "&reconnect=1&sid=" + encode(resumeSid)); String authority = url.getRawAuthority(); return URI.create(scheme + "://" + authority + path + "/rtc?" + query); @@ -173,6 +182,18 @@ void close() { TimeUnit.MILLISECONDS).execute(webSocket::abort)); } + /** + * Cuts the connection off as a network failure would, and reports it. + * For tests. + */ + void abort() { + WebSocket webSocket = socket; + if (webSocket != null) { + webSocket.abort(); + } + post(() -> handler.onSignalClosed("The signaling connection was cut off")); + } + @Override public void onOpen(WebSocket webSocket) { webSocket.request(1); diff --git a/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/ReconnectIntegrationTest.java b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/ReconnectIntegrationTest.java new file mode 100644 index 00000000..92bedb82 --- /dev/null +++ b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/ReconnectIntegrationTest.java @@ -0,0 +1,227 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.livekit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; + +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.EnabledIfSystemProperty; + +import dev.onvoid.webrtc.client.AudioMode; +import dev.onvoid.webrtc.client.CustomVideo; +import dev.onvoid.webrtc.client.RemoteTrack; +import dev.onvoid.webrtc.client.Subscription; +import dev.onvoid.webrtc.client.WebRtc; +import dev.onvoid.webrtc.livekit.internal.proto.LivekitRtc.SimulateScenario; +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; + +/** + * Loses and recovers the connection to a LiveKit server started with --dev, + * whose URL the system property livekit.url names. Skipped without it. + */ +@EnabledIfSystemProperty(named = "livekit.url", matches = ".+") +class ReconnectIntegrationTest { + + private static final long TIMEOUT_SECONDS = 30; + + private WebRtc rtc; + + private ScheduledExecutorService frames; + + private String roomName; + + + @BeforeEach + void setUp() { + rtc = WebRtc.builder().audio(AudioMode.HEADLESS).build(); + frames = Executors.newSingleThreadScheduledExecutor(); + roomName = "test-" + UUID.randomUUID(); + } + + @AfterEach + void tearDown() { + frames.shutdownNow(); + rtc.close(); + } + + @Test + void resumesAfterSignalLoss() throws Exception { + Room alice = join("alice"); + Room bob = join("bob"); + + CompletableFuture received = new CompletableFuture<>(); + bob.onTrackSubscribed((participant, track) -> received.complete(track)); + CompletableFuture left = new CompletableFuture<>(); + bob.onParticipantLeft(left::complete); + + CustomVideo video = publishVideo(alice); + RemoteTrack track = await(received); + + CompletableFuture reconnecting = new CompletableFuture<>(); + CompletableFuture reconnected = new CompletableFuture<>(); + alice.onReconnecting(reconnecting::complete); + alice.onReconnected(() -> reconnected.complete(null)); + String sid = alice.localParticipant().sid(); + + CompletableFuture data = new CompletableFuture<>(); + bob.onData(data::complete); + + ((RoomImpl) alice).cutSignal(); + // Sent while the room may be reconnecting: it waits, or the data + // channel, which a resume keeps, carries it. + CompletableFuture sent = alice.publishData(bytes("during"), "test", true); + + await(reconnecting); + await(reconnected); + await(sent); + + // The same session: the same participant, its peers and channels. + assertEquals(RoomState.CONNECTED, alice.state()); + assertEquals(sid, alice.localParticipant().sid()); + assertEquals("during", text(await(data))); + + // Media and data still flow, both ways. + assertFrame(track); + CompletableFuture back = new CompletableFuture<>(); + alice.onData(back::complete); + await(bob.publishData(bytes("back"), "test", true)); + assertEquals("back", text(await(back))); + + assertFalse(left.isDone(), "Bob saw Alice leave"); + + bob.close(); + alice.close(); + video.close(); + } + + @Test + void rejoinsAfterNodeFailure() throws Exception { + Room alice = join("alice"); + Room bob = join("bob"); + + List tracks = new CopyOnWriteArrayList<>(); + CompletableFuture first = new CompletableFuture<>(); + CompletableFuture second = new CompletableFuture<>(); + bob.onTrackSubscribed((participant, track) -> { + tracks.add(track); + (tracks.size() == 1 ? first : second).complete(track); + }); + CompletableFuture left = new CompletableFuture<>(); + bob.onParticipantLeft(left::complete); + CompletableFuture rejoined = new CompletableFuture<>(); + bob.onParticipantJoined(rejoined::complete); + + CustomVideo video = rtc.media().customVideo("camera-" + UUID.randomUUID()); + frames.scheduleAtFixedRate(() -> push(video), 0, 66, TimeUnit.MILLISECONDS); + Publication publication = await(alice.publish(video, TrackSource.CAMERA)); + String oldTrackSid = publication.sid(); + assertEquals(oldTrackSid, await(first).id()); + + CompletableFuture reconnected = new CompletableFuture<>(); + alice.onReconnected(() -> reconnected.complete(null)); + String oldSid = alice.localParticipant().sid(); + + // The server forgets the session, as when its node fails; LiveKit's Go + // SDK tests full reconnects this way. + ((RoomImpl) alice).simulate(SimulateScenario.newBuilder().setNodeFailure(true).build()); + + await(reconnected); + + // A new session: a new participant sid, and the track published again. + assertEquals(RoomState.CONNECTED, alice.state()); + assertNotEquals(oldSid, alice.localParticipant().sid()); + assertNotEquals(oldTrackSid, publication.sid()); + + assertEquals(oldSid, await(left).sid()); + assertEquals(alice.localParticipant().sid(), await(rejoined).sid()); + RemoteTrack again = await(second); + assertEquals(publication.sid(), again.id()); + assertFrame(again); + + CompletableFuture data = new CompletableFuture<>(); + bob.onData(data::complete); + await(alice.publishData(bytes("after"), "test", true)); + assertEquals("after", text(await(data))); + + bob.close(); + alice.close(); + video.close(); + } + + private CustomVideo publishVideo(Room room) throws Exception { + CustomVideo video = rtc.media().customVideo("camera-" + UUID.randomUUID()); + frames.scheduleAtFixedRate(() -> push(video), 0, 66, TimeUnit.MILLISECONDS); + await(room.publish(video, TrackSource.CAMERA)); + return video; + } + + private static void assertFrame(RemoteTrack track) throws Exception { + CompletableFuture frame = new CompletableFuture<>(); + Subscription subscription = track.asVideo().onFrame(f -> frame.complete(f.buffer.getWidth())); + try { + assertEquals(320, await(frame)); + } + finally { + subscription.close(); + } + } + + private Room join(String identity) throws Exception { + URI url = URI.create(System.getProperty("livekit.url")); + return await(LiveKit.room(rtc, url, TestTokens.token(identity, roomName)).connect()); + } + + private static T await(CompletableFuture future) throws Exception { + return future.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + } + + private static byte[] bytes(String text) { + return text.getBytes(StandardCharsets.UTF_8); + } + + private static String text(DataMessage message) { + return new String(message.payload(), StandardCharsets.UTF_8); + } + + private static void push(CustomVideo video) { + NativeI420Buffer buffer = NativeI420Buffer.allocate(320, 240); + VideoFrame frame = new VideoFrame(buffer, System.nanoTime()); + + try { + video.push(frame); + } + finally { + frame.release(); + } + } + +} From 4ab845ce7a569ba6b7c092ebf257bd7c1609a088 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Fri, 20 Nov 2026 17:54:55 +0100 Subject: [PATCH 4/5] feat: hand earlier tracks to late LiveKit track handlers A handler added after connect() missed the tracks that arrived before it. Also tests publishing audio. --- .../java/dev/onvoid/webrtc/livekit/Room.java | 4 +- .../dev/onvoid/webrtc/livekit/RoomImpl.java | 48 +++++++++++++++- .../webrtc/livekit/RoomIntegrationTest.java | 57 +++++++++++++++++++ 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java index 450382aa..22d551d1 100644 --- a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/Room.java @@ -122,7 +122,9 @@ public interface Room extends AutoCloseable { /** * @param handler gets each remote track this client receives, with the - * participant that publishes it. + * participant that publishes it. Tracks that arrived + * before the handler, and have not ended, are handed to it + * too. * * @return removes the handler. */ diff --git a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java index 47e9c9ac..e02330ff 100644 --- a/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java +++ b/webrtc-livekit/src/main/java/dev/onvoid/webrtc/livekit/RoomImpl.java @@ -177,6 +177,9 @@ final class RoomImpl implements Room { /** Tracks that arrived before the participant that publishes them, by participant sid. */ private final Map> orphanTracks = new HashMap<>(); + /** The remote tracks handed to the track handlers, for handlers added later. */ + private final List subscribedTracks = new ArrayList<>(); + private final Map pendingPublishes = new HashMap<>(); /** The published tracks, by sid. */ @@ -370,7 +373,28 @@ public Subscription onParticipantLeft(Consumer handler) { @Override public Subscription onTrackSubscribed(BiConsumer handler) { - return subscribe(trackHandlers, handler); + requireNonNull(handler, "handler"); + + // On the room thread, so that a track is handed over either here or + // when it arrives, never both. + boolean[] removed = new boolean[1]; + executor.execute(() -> { + if (removed[0]) { + return; + } + trackHandlers.add(handler); + + for (SubscribedTrack subscribed : List.copyOf(subscribedTracks)) { + if (!subscribed.track.isEnded()) { + call(() -> handler.accept(subscribed.participant, subscribed.track)); + } + } + }); + + return () -> executor.execute(() -> { + removed[0] = true; + trackHandlers.remove(handler); + }); } @Override @@ -1131,12 +1155,17 @@ private void onRemoteTrack(RemoteTrack track) { } private void trackSubscribed(Participant participant, RemoteTrack track) { + subscribedTracks.removeIf(subscribed -> subscribed.track.isEnded()); + subscribedTracks.add(new SubscribedTrack(participant, track)); + for (BiConsumer handler : trackHandlers) { call(() -> handler.accept(participant, track)); } } private void fireLeft(Participant participant) { + subscribedTracks.removeIf(subscribed -> subscribed.participant == participant); + for (Consumer handler : leftHandlers) { call(() -> handler.accept(participant)); } @@ -1465,6 +1494,7 @@ private void disconnect(String reason, boolean sendLeave) { failPending(error); orphanTracks.clear(); + subscribedTracks.clear(); publications.clear(); CompletableFuture publisherClosed = publisher != null @@ -1584,4 +1614,20 @@ private static final class PendingPublish { } } + + + /** A remote track and the participant that publishes it. */ + private static final class SubscribedTrack { + + final Participant participant; + + final RemoteTrack track; + + + SubscribedTrack(Participant participant, RemoteTrack track) { + this.participant = participant; + this.track = track; + } + } + } diff --git a/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/RoomIntegrationTest.java b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/RoomIntegrationTest.java index c7377391..0b484496 100644 --- a/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/RoomIntegrationTest.java +++ b/webrtc-livekit/src/test/java/dev/onvoid/webrtc/livekit/RoomIntegrationTest.java @@ -19,6 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -43,8 +44,10 @@ import org.junit.jupiter.api.condition.EnabledIfSystemProperty; import dev.onvoid.webrtc.client.AudioMode; +import dev.onvoid.webrtc.client.CustomAudio; import dev.onvoid.webrtc.client.CustomVideo; import dev.onvoid.webrtc.client.MediaKind; +import dev.onvoid.webrtc.client.PcmFormat; import dev.onvoid.webrtc.client.RemoteTrack; import dev.onvoid.webrtc.client.Subscription; import dev.onvoid.webrtc.client.WebRtc; @@ -139,6 +142,60 @@ void subscriberReceivesPublishedVideo() throws Exception { video.close(); } + @Test + void subscriberReceivesPublishedAudio() throws Exception { + Room alice = join("alice"); + Room bob = join("bob"); + + CompletableFuture received = new CompletableFuture<>(); + bob.onTrackSubscribed((participant, track) -> received.complete(track)); + + // A second of silence, pushed every second. + PcmFormat format = PcmFormat.of(48000, 1); + CustomAudio audio = rtc.media().customAudio("microphone-" + UUID.randomUUID()); + byte[] second = new byte[format.sampleRate() * 2]; + frames.scheduleAtFixedRate(() -> audio.push(second, format), 0, 1, TimeUnit.SECONDS); + + Publication publication = await(alice.publish(audio, TrackSource.MICROPHONE)); + assertEquals(TrackSource.MICROPHONE, publication.source()); + + RemoteTrack track = await(received); + assertEquals(MediaKind.AUDIO, track.kind()); + assertEquals(publication.sid(), track.id()); + + CompletableFuture chunk = new CompletableFuture<>(); + track.asAudio().onAudio(c -> chunk.complete(c.frames())); + assertTrue(await(chunk) > 0); + + bob.close(); + alice.close(); + audio.close(); + } + + @Test + void lateHandlerGetsEarlierTracks() throws Exception { + Room alice = join("alice"); + + CustomVideo video = rtc.media().customVideo("camera-" + UUID.randomUUID()); + frames.scheduleAtFixedRate(() -> push(video), 0, 66, TimeUnit.MILLISECONDS); + Publication publication = await(alice.publish(video, TrackSource.CAMERA)); + + // Bob joins after the track was published, and receives it before + // he adds a handler. + Room bob = join("bob"); + CompletableFuture first = new CompletableFuture<>(); + bob.onTrackSubscribed((participant, track) -> first.complete(track)); + assertEquals(publication.sid(), await(first).id()); + + CompletableFuture late = new CompletableFuture<>(); + bob.onTrackSubscribed((participant, track) -> late.complete(track)); + assertSame(await(first), await(late)); + + bob.close(); + alice.close(); + video.close(); + } + @Test void simulcastFollowsSubscribers() throws Exception { Room alice = join("alice"); From b6bdbd81ded7ebef01ab89b43db023d17f2631d6 Mon Sep 17 00:00:00 2001 From: Alex Andres Date: Fri, 20 Nov 2026 17:54:55 +0100 Subject: [PATCH 5/5] docs: add LiveKit guide and example --- README.md | 1 + docs/.vitepress/sidebar.ts | 1 + docs/guide/client-api.md | 1 + docs/guide/client/livekit.md | 228 +++++++++++++++++ docs/guide/examples.md | 18 ++ docs/guide/index.md | 1 + webrtc-examples/pom.xml | 5 + .../webrtc/examples/LiveKitExample.java | 240 ++++++++++++++++++ .../src/main/java/module-info.java | 1 + 9 files changed, 496 insertions(+) create mode 100644 docs/guide/client/livekit.md create mode 100644 webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/LiveKitExample.java diff --git a/README.md b/README.md index bfcaaeaf..fff9472c 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,7 @@ The library provides a comprehensive set of Java classes that map to the WebRTC **Optional modules** - **[Media playback](https://jrtc.dev/guide/media/media-files)** (`webrtc-java-media`) - Stream video and audio files or RTSP sources into a call, decoded with FFmpeg +- **[LiveKit rooms](https://jrtc.dev/guide/client/livekit)** (`webrtc-java-livekit`) - Join rooms of a LiveKit server, publish and receive tracks in simulcast, exchange data and reconnect after a lost connection - **[Call recording](https://jrtc.dev/guide/media/media-recording)** (`webrtc-java-media`) - Save what a call sends or receives into MKV, WebM or MP4 files, without re-encoding, as one file, rotating segments, or a replay buffer of the last seconds **Platforms** diff --git a/docs/.vitepress/sidebar.ts b/docs/.vitepress/sidebar.ts index 0f9434c8..5b5aeeb9 100644 --- a/docs/.vitepress/sidebar.ts +++ b/docs/.vitepress/sidebar.ts @@ -28,6 +28,7 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] { { text: 'Transforms and Encryption', link: '/client/transforms' }, { text: 'Data Channels', link: '/client/data-channels' }, { text: 'WHIP and WHEP', link: '/client/whip-whep' }, + { text: 'LiveKit Rooms', link: '/client/livekit' }, { text: 'Statistics and Call Checks', link: '/client/statistics' }, ], }, diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index 087f5229..b3ad1316 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -51,6 +51,7 @@ The client API is described on these pages: | [Transforms and Encryption](/guide/client/transforms) | Encoded transforms and end-to-end encryption of a peer's media. | | [Data Channels](/guide/client/data-channels) | Messages between peers, flow control and channel options. | | [WHIP and WHEP](/guide/client/whip-whep) | Publishing to and playing from media servers. | +| [LiveKit Rooms](/guide/client/livekit) | Joining rooms of a LiveKit server, with the `webrtc-java-livekit` module. | | [Statistics and Call Checks](/guide/client/statistics) | Connection statistics, diagnosis, and checking devices and network before a call. | ## The Runtime diff --git a/docs/guide/client/livekit.md b/docs/guide/client/livekit.md new file mode 100644 index 00000000..d653b58d --- /dev/null +++ b/docs/guide/client/livekit.md @@ -0,0 +1,228 @@ +# LiveKit Rooms + +The `webrtc-java-livekit` module joins rooms of a [LiveKit](https://livekit.io) server. Each participant publishes its tracks once, and the server forwards them to everyone else in the room: + +```java +Room room = LiveKit.room(rtc, URI.create("wss://live.example.com"), token) + .connect() + .get(15, TimeUnit.SECONDS); + +room.onTrackSubscribed((participant, track) -> + System.out.println("Receiving " + track.kind() + " of " + participant.identity())); + +room.publish(rtc.media().camera(), TrackSource.CAMERA); +room.publish(rtc.media().microphone(), TrackSource.MICROPHONE); +``` + +The module builds on the [Client API](/guide/client-api): tracks are the runtime's local and remote tracks, and the room holds two of its peers. + +::: warning +The module is new. Its API may still change between releases. +::: + +## Adding the Module + +Depend on the module beside `webrtc-java`. It brings `protobuf-java` along, which carries LiveKit's signaling messages: + +```xml + + dev.onvoid.webrtc + webrtc-java-livekit + {{ VERSION }} + +``` + +Its module name is `webrtc.java.livekit`. The module has no natives of its own. + +## Joining a Room + +`LiveKit.room(rtc, url, token)` takes the server's URL and an access token for one room. `connect()` opens the signaling connection and joins. Its future completes once the room is joined. + +The token is a JWT that your backend signs with the server's API key and secret. It names the participant's identity, the room, and what the participant may do there. Never ship the API secret in a client. LiveKit's [server SDKs](https://docs.livekit.io/home/server/generating-tokens/) create tokens in a few lines. + +The future fails with a `LiveKitException` if the server refuses the token or does not answer in time. The message names the HTTP status and what the server said. The timeout is ten seconds by default: + +```java +Room room = LiveKit.room(rtc, url, token) + .timeout(Duration.ofSeconds(5)) + .connect() + .get(15, TimeUnit.SECONDS); +``` + +## Participants + +`localParticipant()` is this client, and `remoteParticipants()` lists everyone else. A `Participant` has the server's `sid()`, the `identity()` from the token, a display `name()`, and the `metadata()` and `attributes()` your backend gave it: + +```java +room.onParticipantJoined(participant -> System.out.println(participant.identity() + " joined")); +room.onParticipantLeft(participant -> System.out.println(participant.identity() + " left")); +``` + +The participant objects stay the same while their fields change. A participant that leaves is reported once. + +## Publishing Tracks + +`publish(track, source)` publishes any local track: a camera, a microphone, a shared screen or a pushed track. The server first accepts the track, then the room negotiates it. The future holds a `Publication`: + +```java +Publication publication = room.publish(rtc.media().microphone(), TrackSource.MICROPHONE) + .get(15, TimeUnit.SECONDS); + +System.out.println("Published as " + publication.sid()); +``` + +`publication.sid()` is the id the other participants see the track under. The track stays yours: close it after you left the room. + +### Simulcast + +With simulcast, a video track is sent in several sizes at once. The server forwards each subscriber the one its connection and view take. `PublishOptions` turns it on, and needs the size the video is captured in: + +```java +Publication publication = room.publish(camera, PublishOptions.builder() + .source(TrackSource.CAMERA) + .videoSize(1280, 720) + .simulcast(true) + .build()) + .get(15, TimeUnit.SECONDS); +``` + +The room picks the layers the way LiveKit's own clients do: + +| Captured height | Layers | Example for 1280x720 | +|---|---|---| +| 720 lines or more | `q` at a quarter, `h` at half, `f` at full size | 320x180, 640x360, 1280x720 | +| 360 to 719 lines | `q` at half, `h` at full size | - | +| Below 360 lines | None: the track is sent in one size | - | + +The server pauses the layers that no subscriber watches, and resumes them once one does (LiveKit's dynacast). `publication.rids()` lists the layers. The bitrates are those of LiveKit's presets, listed under [Details](#simulcast-bitrates). + +## Receiving Tracks + +By default, the room receives every track published in it. `onTrackSubscribed` hands each one over with the participant that publishes it. Tracks that arrived before the handler was added are handed to it too: + +```java +room.onTrackSubscribed((participant, track) -> { + if (track.isVideo()) { + track.asVideo().onFrame(frame -> System.out.println( + participant.identity() + ": " + frame.buffer.getWidth() + "x" + frame.buffer.getHeight())); + } +}); +``` + +The track is a regular remote track, as described in [Receiving Media](/guide/client/receiving-media). Its `id()` is the publication's sid. + +A video sent in simulcast arrives in the largest layer the connection takes. A small view, such as a thumbnail, needs no more than the smallest layer: + +```java +room.requestVideoQuality(track.id(), VideoQuality.LOW); +``` + +`LOW`, `MEDIUM` and `HIGH` are the layers `q`, `h` and `f`. The server still forwards a smaller layer while the connection takes no more. + +## Sending Data + +`publishData(payload, topic, reliable)` sends bytes to every other participant. The topic lets receivers tell messages apart: + +```java +room.publishData("slide 12".getBytes(StandardCharsets.UTF_8), "slide", true); + +room.onData(message -> { + if (message.topic().orElse("").equals("slide")) { + showSlide(new String(message.payload(), StandardCharsets.UTF_8)); + } +}); +``` + +- **Reliable** (`true`): ordered and retransmitted, for events that must arrive, such as the slide a presentation is on. +- **Lossy** (`false`): sent once, unordered, for data that is stale soon, such as a pointer position. + +A `DataMessage` also names the `senderIdentity()` and whether it came reliably. The future of `publishData` completes once the data is handed to the channel. + +## Reconnecting + +If the connection is lost, the room gets it back by itself. The application hears of it through two handlers: + +```java +room.onReconnecting(reason -> System.out.println("Reconnecting: " + reason)); +room.onReconnected(() -> System.out.println("Reconnected")); +``` + +1. The room first resumes the session. The participant, its peers and its tracks stay as they are. +2. If the server cannot resume it, the room joins anew and publishes its tracks again. +3. If ten attempts fail, the room is left, and `onDisconnected` gets the reason. + +While the room reconnects, `publish` and `publishData` wait until it is connected again. After a new join, the room behaves as if everyone had left and joined again: + +- The local participant has a new sid. +- `onParticipantLeft` and then `onParticipantJoined` report every other participant. +- Each `Publication` keeps its object, but has a new sid. +- The remote tracks arrive anew through `onTrackSubscribed`. + +`state()` tells where the room is: + +| State | Meaning | +|---|---| +| `CONNECTING` | Joining the room. | +| `CONNECTED` | Joined, media and data flow. | +| `RECONNECTING` | The connection was lost, and the room is getting it back. | +| `DISCONNECTED` | The room was left, and cannot be joined again. | + +## Threads + +The room runs its handlers on a thread of its own, one at a time, in the order the server sent what they report. Its futures complete on that thread too. + +A handler that blocks holds up the whole room. Long work belongs on another thread. In particular, never wait for a room's future inside a handler of the same room: the future can only complete once the handler returned. + +## Leaving a Room + +`room.close()` leaves the room and waits until its peers are closed. `closeAsync()` returns a future instead. `onDisconnected` gets the reason, whether the application left, the server ended the session, or reconnecting failed. + +The tracks stay open: they are the application's, and may still be sent elsewhere. + +## Details + +### Simulcast Bitrates + +Each layer gets the bitrate of LiveKit's preset for its height: + +| Layer height | Bitrate | +|---|---| +| 1080 lines or more | 3 Mbit/s | +| 720 to 1079 lines | 1.7 Mbit/s | +| 540 to 719 lines | 800 kbit/s | +| 360 to 539 lines | 450 kbit/s | +| Below 360 lines | 160 kbit/s | + +### What Starts a Reconnect + +- The signaling connection closes. +- The server stops answering pings, within the timeout it named at the join. +- One of the two peers fails. +- The server asks for it, with a leave request that allows a resume or a new join. + +The first attempt starts at once. The attempts after it wait 300 ms times the square of their number, at most ten seconds. Each attempt may take twice the room's timeout. A resume that fails turns the next attempt into a new join. + +If the server sends a new token, the room uses it for later attempts. + +### Limits + +- With `autoSubscribe(false)`, the room receives no tracks. The module has no call yet to subscribe to single tracks. +- A track that is being published while the room joins anew fails, instead of being published in the new session. Published tracks are published again. +- The ICE servers the server names when a session resumes are not applied. The peers keep those of the join. +- The module speaks LiveKit's signaling protocol 16. It is tested against `livekit-server` 1.13. +- End-to-end encryption, LiveKit's RPC and text and byte streams are not supported. + +## Example + +The [`LiveKitExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/LiveKitExample.java) joins a room, publishes a test pattern in simulcast and sends an event every five seconds. Start a server with `livekit-server --dev`, then run the example twice: + +```bash +mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.LiveKitExample" -D"exec.args=ws://localhost:7880 lecture alice 30" +mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.LiveKitExample" -D"exec.args=ws://localhost:7880 lecture bob 30" +``` + +## Related + +- [Sending Media](/guide/client/sending-media) - The local tracks a room publishes +- [Receiving Media](/guide/client/receiving-media) - Frames, views and audio of remote tracks +- [WHIP and WHEP](/guide/client/whip-whep) - Publishing to and playing from media servers without rooms diff --git a/docs/guide/examples.md b/docs/guide/examples.md index ecf81bae..1e53e2d0 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -88,6 +88,24 @@ This example is useful for applications that need to receive media streams from mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.WhepExample" -D"exec.args=http://localhost:8889/mystream/whep 10" ``` +## LiveKit + +The [`LiveKitExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/LiveKitExample.java) joins a room of a LiveKit server with the [LiveKit Rooms](/guide/client/livekit) module. It needs no camera, microphone or speakers. + +**Key features demonstrated:** +- Joining a room with an access token +- Publishing a pushed test pattern in three simulcast layers +- Receiving the other participants' video in the smallest layer +- Sending an event to the room every five seconds, and receiving those of the others +- Following participants, and the reconnects after a lost connection + +Start a server with `livekit-server --dev`, then run the example twice, in two shells. Against another server, set `LIVEKIT_TOKEN` to a token for the room: + +```bash +mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.LiveKitExample" -D"exec.args=ws://localhost:7880 lecture alice 30" +mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.LiveKitExample" -D"exec.args=ws://localhost:7880 lecture bob 30" +``` + ## Codec List The [`CodecListExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/CodecListExample.java) demonstrates how to list all supported codecs with the WebRTC peer-connection-factory. diff --git a/docs/guide/index.md b/docs/guide/index.md index 003430ce..def212b7 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -18,6 +18,7 @@ Every guide of the webrtc-java library, in the order of the sidebar. New to the - [Transforms and Encryption](/guide/client/transforms) - Encoded transforms and end-to-end encryption of a peer's media - [Data Channels](/guide/client/data-channels) - Messages between peers, flow control and channel options - [WHIP and WHEP](/guide/client/whip-whep) - Publishing to and playing from media servers +- [LiveKit Rooms](/guide/client/livekit) - Joining rooms of a LiveKit server, with simulcast, data and reconnects - [Statistics and Call Checks](/guide/client/statistics) - Connection statistics, diagnosis and checks before a call ## Media Basics diff --git a/webrtc-examples/pom.xml b/webrtc-examples/pom.xml index feb70aee..263e6c49 100644 --- a/webrtc-examples/pom.xml +++ b/webrtc-examples/pom.xml @@ -136,6 +136,11 @@ webrtc-java-javafx ${project.version} + + ${project.groupId} + webrtc-java-livekit + ${project.version} + org.openjfx javafx-controls diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/LiveKitExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/LiveKitExample.java new file mode 100644 index 00000000..640f6068 --- /dev/null +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/LiveKitExample.java @@ -0,0 +1,240 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.examples; + +import java.net.URI; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.Base64; +import java.util.UUID; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import dev.onvoid.webrtc.client.AudioMode; +import dev.onvoid.webrtc.client.CustomVideo; +import dev.onvoid.webrtc.client.WebRtc; +import dev.onvoid.webrtc.livekit.LiveKit; +import dev.onvoid.webrtc.livekit.Publication; +import dev.onvoid.webrtc.livekit.PublishOptions; +import dev.onvoid.webrtc.livekit.Room; +import dev.onvoid.webrtc.livekit.TrackSource; +import dev.onvoid.webrtc.livekit.VideoQuality; +import dev.onvoid.webrtc.media.video.NativeI420Buffer; +import dev.onvoid.webrtc.media.video.VideoFrame; + +/** + * Example of a LiveKit client, with the webrtc-java-livekit module. + *

+ * This class demonstrates: + *

    + *
  • Joining a LiveKit room with an access token
  • + *
  • Publishing a video track in simulcast layers
  • + *
  • Receiving the other participants' video, in the smallest layer
  • + *
  • Sending and receiving data messages by topic
  • + *
  • Following participants, and the reconnects after a lost connection
  • + *
+ *

+ * The video is a test pattern the example pushes, so no camera is needed, and the runtime is + * headless, so no microphone or speakers either. + *

+ * Start a LiveKit server with {@code livekit-server --dev}, then run the example twice, in two + * shells, to see the two participants receive each other: + *

+ * mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.LiveKitExample" -D"exec.args=ws://localhost:7880 lecture alice 30"
+ * mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.LiveKitExample" -D"exec.args=ws://localhost:7880 lecture bob 30"
+ * 
+ * Against another server, set the environment variable {@code LIVEKIT_TOKEN} to a token for the + * room, or {@code LIVEKIT_API_KEY} and {@code LIVEKIT_API_SECRET} to the server's key and secret. + * + * @author Alex Andres + */ +public class LiveKitExample { + + private static final int WIDTH = 1280; + + private static final int HEIGHT = 720; + + + public static void main(String[] args) throws Exception { + URI url = URI.create(args.length > 0 ? args[0] : "ws://localhost:7880"); + String roomName = args.length > 1 ? args[1] : "lecture"; + String identity = args.length > 2 ? args[2] : "java-" + UUID.randomUUID().toString().substring(0, 8); + int seconds = args.length > 3 ? Integer.parseInt(args[3]) : 30; + + // Normally the application's backend issues the token. + String token = System.getenv("LIVEKIT_TOKEN"); + if (token == null) { + token = devToken(identity, roomName); + } + + // Pushes the video and sends the slide events. + ScheduledExecutorService timer = Executors.newSingleThreadScheduledExecutor(); + + try (WebRtc rtc = WebRtc.builder().audio(AudioMode.HEADLESS).build()) { + System.out.println("Joining the room " + roomName + " on " + url + " as " + identity); + + // The main thread may wait for the room; a handler must not. + Room room = LiveKit.room(rtc, url, token) + .connect() + .get(15, TimeUnit.SECONDS); + CustomVideo video = rtc.media().customVideo("pattern"); + + try { + run(room, video, timer, seconds); + } + finally { + timer.shutdownNow(); + timer.awaitTermination(1, TimeUnit.SECONDS); + + // Leaves the room and closes its peers; the track stays ours. + room.close(); + video.close(); + } + } + finally { + // Also if joining failed. + timer.shutdownNow(); + } + // The runtime closed the factory. + } + + /** + * Follows the room, publishes the video and sends a slide event every five seconds. + */ + private static void run(Room room, CustomVideo video, ScheduledExecutorService timer, int seconds) + throws Exception { + System.out.println("Joined as " + room.localParticipant().sid() + ", with " + + room.remoteParticipants().size() + " other participants"); + + // Handlers run on the room's thread, one at a time. + room.onParticipantJoined(participant -> System.out.println(participant.identity() + " joined")); + room.onParticipantLeft(participant -> System.out.println(participant.identity() + " left")); + + room.onTrackSubscribed((participant, track) -> { + System.out.println("Receiving " + track.kind() + " " + track.id() + " of " + participant.identity()); + + if (track.isVideo()) { + // A thumbnail needs no more than the smallest layer. + room.requestVideoQuality(track.id(), VideoQuality.LOW); + + AtomicInteger frames = new AtomicInteger(); + track.asVideo().onFrame(frame -> { + if (frames.incrementAndGet() % 100 == 1) { + System.out.printf("Video frame %d of %s: %dx%d%n", frames.get(), + participant.identity(), frame.buffer.getWidth(), frame.buffer.getHeight()); + } + }); + } + }); + + room.onData(message -> System.out.println(message.senderIdentity() + " sent " + + message.topic().orElse("data") + ": " + + new String(message.payload(), StandardCharsets.UTF_8))); + + room.onReconnecting(reason -> System.out.println("Reconnecting: " + reason)); + room.onReconnected(() -> System.out.println("Reconnected as " + room.localParticipant().sid())); + room.onDisconnected(reason -> System.out.println("Disconnected: " + reason)); + + // A pushed test pattern, sent in three layers: 320x180, 640x360 and 1280x720. + AtomicInteger frameNumber = new AtomicInteger(); + timer.scheduleAtFixedRate(() -> push(video, frameNumber.getAndIncrement()), + 0, 66, TimeUnit.MILLISECONDS); + + Publication publication = room.publish(video, PublishOptions.builder() + .source(TrackSource.CAMERA) + .videoSize(WIDTH, HEIGHT) + .simulcast(true) + .build()) + .get(15, TimeUnit.SECONDS); + + System.out.println("Publishing " + publication.sid() + " in the layers " + publication.rids()); + + // Events everyone in the room receives, such as the slide a lecture is on. + AtomicInteger slide = new AtomicInteger(); + timer.scheduleAtFixedRate(() -> { + String event = "slide " + slide.incrementAndGet(); + room.publishData(event.getBytes(StandardCharsets.UTF_8), "slide", true); + }, 5, 5, TimeUnit.SECONDS); + + Thread.sleep(seconds * 1000L); + } + + /** + * Draws a bar that moves across a grey frame and pushes it. + */ + private static void push(CustomVideo video, int number) { + NativeI420Buffer buffer = NativeI420Buffer.allocate(WIDTH, HEIGHT); + + byte[] row = new byte[WIDTH]; + Arrays.fill(row, (byte) 80); + int bar = (number * 8) % WIDTH; + Arrays.fill(row, bar, Math.min(bar + 64, WIDTH), (byte) 235); + + ByteBuffer y = buffer.getDataY(); + for (int line = 0; line < HEIGHT; line++) { + y.put(line * buffer.getStrideY(), row); + } + fill(buffer.getDataU(), (byte) 128); + fill(buffer.getDataV(), (byte) 128); + + VideoFrame frame = new VideoFrame(buffer, System.nanoTime()); + try { + video.push(frame); + } + finally { + frame.release(); + } + } + + private static void fill(ByteBuffer plane, byte value) { + for (int i = 0; i < plane.capacity(); i++) { + plane.put(i, value); + } + } + + /** + * Signs a token for one room, ten minutes long, as a backend would. The key and secret are + * those of a server started with --dev, unless LIVEKIT_API_KEY and LIVEKIT_API_SECRET name + * others. Never ship the secret in a client. + */ + private static String devToken(String identity, String room) throws Exception { + String key = System.getenv().getOrDefault("LIVEKIT_API_KEY", "devkey"); + String secret = System.getenv().getOrDefault("LIVEKIT_API_SECRET", "secret"); + long now = System.currentTimeMillis() / 1000; + + String header = "{\"alg\":\"HS256\",\"typ\":\"JWT\"}"; + String claims = "{\"iss\":\"" + key + "\",\"sub\":\"" + identity + "\",\"name\":\"" + identity + "\"" + + ",\"nbf\":" + (now - 10) + ",\"exp\":" + (now + 600) + + ",\"video\":{\"room\":\"" + room + "\",\"roomJoin\":true,\"canPublish\":true" + + ",\"canSubscribe\":true,\"canPublishData\":true}}"; + + Base64.Encoder base64 = Base64.getUrlEncoder().withoutPadding(); + String unsigned = base64.encodeToString(header.getBytes(StandardCharsets.UTF_8)) + "." + + base64.encodeToString(claims.getBytes(StandardCharsets.UTF_8)); + + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(secret.getBytes(StandardCharsets.UTF_8), "HmacSHA256")); + + return unsigned + "." + base64.encodeToString(mac.doFinal(unsigned.getBytes(StandardCharsets.UTF_8))); + } +} diff --git a/webrtc-examples/src/main/java/module-info.java b/webrtc-examples/src/main/java/module-info.java index a00a3e6c..d92b117a 100644 --- a/webrtc-examples/src/main/java/module-info.java +++ b/webrtc-examples/src/main/java/module-info.java @@ -9,6 +9,7 @@ requires org.eclipse.jetty.websocket.server; requires webrtc.java; requires webrtc.java.javafx; + requires webrtc.java.livekit; requires webrtc.java.swing; requires webrtc.java.media;