diff --git a/docs/.vitepress/nav.ts b/docs/.vitepress/nav.ts index fbe4320c0..9567eaa3f 100644 --- a/docs/.vitepress/nav.ts +++ b/docs/.vitepress/nav.ts @@ -16,8 +16,8 @@ export const navbar: DefaultTheme.NavItem[] = [ link: '/tools', }, { - text: 'Client API Design', - link: '/guide/client-api-design', + text: 'Client API', + link: '/guide/client-api', }, { text: IS_NEXT ? PROJECT_VARS.VERSION_SNAPSHOT : PROJECT_VARS.VERSION, diff --git a/docs/.vitepress/sidebar.ts b/docs/.vitepress/sidebar.ts index 5d2c47950..0f9434c82 100644 --- a/docs/.vitepress/sidebar.ts +++ b/docs/.vitepress/sidebar.ts @@ -11,10 +11,24 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] { text: 'Introduction', collapsed: false, items: [ - { text: 'What is webrtc-java?', link: '/introduction' }, + { text: 'Introduction', link: '/introduction' }, { text: 'Getting Started', link: '/get-started' }, + { text: 'Core API', link: '/core-api' }, + ], + }, + { + text: 'Client API', + collapsed: false, + items: [ { text: 'Client API', link: '/client-api' }, - { text: 'Client API Design', link: '/client-api-design' }, + { text: 'Peers and Signaling', link: '/client/peers' }, + { text: 'Threads', link: '/client/threads' }, + { text: 'Sending Media', link: '/client/sending-media' }, + { text: 'Receiving Media', link: '/client/receiving-media' }, + { text: 'Transforms and Encryption', link: '/client/transforms' }, + { text: 'Data Channels', link: '/client/data-channels' }, + { text: 'WHIP and WHEP', link: '/client/whip-whep' }, + { text: 'Statistics and Call Checks', link: '/client/statistics' }, ], }, { @@ -85,6 +99,14 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] { { text: 'GraalVM Native Image', link: '/advanced/native-image' }, ], }, + { + text: 'Background', + collapsed: false, + items: [ + { text: 'Client API Design', link: '/client-api-design' }, + { text: 'Build Notes', link: '/build' }, + ], + }, ] } @@ -97,7 +119,7 @@ function sidebarTools(): DefaultTheme.SidebarItem[] { { text: 'Audio Converter', link: '/audio/audio-converter' }, { text: 'Audio Recorder', link: '/audio/audio-recorder' }, { text: 'Audio Player', link: '/audio/audio-player' }, - { text: 'Voice Activity Detector', link: '/audio/voice-activity-detector' }, + { text: 'Voice Activity Detection', link: '/audio/voice-activity-detector' }, ], }, { @@ -113,7 +135,7 @@ function sidebarTools(): DefaultTheme.SidebarItem[] { collapsed: false, items: [ { text: 'Screen Capturer', link: '/desktop/screen-capturer' }, - { text: 'Window Capturer ', link: '/desktop/window-capturer' }, + { text: 'Window Capturer', link: '/desktop/window-capturer' }, { text: 'Power Management', link: '/desktop/power-management' }, ], }, diff --git a/docs/.vitepress/theme/ApiNote.vue b/docs/.vitepress/theme/ApiNote.vue new file mode 100644 index 000000000..7a40f0b5a --- /dev/null +++ b/docs/.vitepress/theme/ApiNote.vue @@ -0,0 +1,24 @@ + + + diff --git a/docs/.vitepress/theme/DocsHome.vue b/docs/.vitepress/theme/DocsHome.vue index bffd1c9a9..da895f8dc 100644 --- a/docs/.vitepress/theme/DocsHome.vue +++ b/docs/.vitepress/theme/DocsHome.vue @@ -51,7 +51,7 @@ async function copyDependency() { const paths = [ { label: 'START BUILDING', title: 'Your first connection', description: 'Install the library, connect two peers, and send your first audio and video.', link: '/guide/get-started', action: 'Get started', icon: 'code' }, { label: 'EXPLORE MEDIA', title: 'Bring your media', description: 'Capture a camera, share a screen, play an RTSP stream, or record a call.', link: '/guide/media/media-devices', action: 'Explore media', icon: 'video' }, - { label: 'CONNECT TO MEDIA SERVERS', title: 'WHIP / WHEP', description: 'Publish live audio and video with WHIP, or receive streams with WHEP over HTTP.', link: '/guide/client-api#whip-and-whep', action: 'Explore WHIP / WHEP', icon: 'arrows' }, + { label: 'CONNECT TO MEDIA SERVERS', title: 'WHIP / WHEP', description: 'Publish live audio and video with WHIP, or receive streams with WHEP over HTTP.', link: '/guide/client/whip-whep', action: 'Explore WHIP / WHEP', icon: 'arrows' }, ] const topics = [ { title: 'Audio & video capture', detail: 'Connect microphones, cameras, and custom sources.', link: '/guide/media/media-devices', icon: 'video' }, diff --git a/docs/.vitepress/theme/index.ts b/docs/.vitepress/theme/index.ts index 35b96fe48..72af76d39 100644 --- a/docs/.vitepress/theme/index.ts +++ b/docs/.vitepress/theme/index.ts @@ -1,10 +1,12 @@ // https://vitepress.dev/guide/custom-theme import { h } from 'vue' -import type { Theme } from 'vitepress' +import { inBrowser, type Theme } from 'vitepress' import DefaultTheme from 'vitepress/theme' import NextBanner from './NextBanner.vue' import DocsHome from './DocsHome.vue' import ThreadingDiagram from './ThreadingDiagram.vue' +import ApiNote from './ApiNote.vue' +import { movedSection } from './moved' import './style.css' export default { @@ -19,5 +21,33 @@ export default { enhanceApp({ app, router, siteData }) { app.component('DocsHome', DocsHome) app.component('ThreadingDiagram', ThreadingDiagram) + app.component('ApiNote', ApiNote) + + // A link to a section that moved to a page of its own goes on to that page, + // in place of the old address, so that going back does not come here again. + if (inBrowser) { + const base = siteData.value.base + const target = (url: URL) => { + const path = '/' + url.pathname.slice(base.length) + const moved = movedSection(path, url.hash) + return moved && base + moved.slice(1) + } + + // The page that was served is the old one, so the new one is loaded in its place. + const first = target(new URL(location.href)) + if (first) { + location.replace(first) + } + + const previous = router.onBeforeRouteChange + router.onBeforeRouteChange = (to) => { + const moved = target(new URL(to, location.href)) + if (moved) { + router.go(moved, { replace: true }) + return false + } + return previous?.(to) + } + } } } satisfies Theme diff --git a/docs/.vitepress/theme/moved.ts b/docs/.vitepress/theme/moved.ts new file mode 100644 index 000000000..8cad395c9 --- /dev/null +++ b/docs/.vitepress/theme/moved.ts @@ -0,0 +1,56 @@ +// Sections that moved to pages of their own. A link to one of them, from a page +// of an earlier release or from another site, is sent on to where it is now. +export const MOVED_SECTIONS: Record> = { + '/guide/client-api': { + 'peers-and-signaling': '/guide/client/peers', + 'signaling': '/guide/client/peers#signaling', + 'polite-and-impolite-peers': '/guide/client/peers#polite-and-impolite-peers', + 'peer-state': '/guide/client/peers#peer-state', + 'automatic-ice-recovery': '/guide/client/peers#automatic-ice-recovery', + 'threads': '/guide/client/threads', + 'limiting-threads': '/guide/client/threads#limiting-threads', + 'handlers-on-the-ui-thread': '/guide/client/threads#handlers-on-the-ui-thread', + 'don-t-wait-in-a-handler': '/guide/client/threads#don-t-wait-in-a-handler', + 'sending-media': '/guide/client/sending-media', + 'sending-a-media-file': '/guide/client/sending-media#sending-a-media-file', + 'pushing-video-and-audio': '/guide/client/sending-media#pushing-video-and-audio', + 'controlling-senders-and-tracks': '/guide/client/sending-media#controlling-senders-and-tracks', + 'switching-cameras': '/guide/client/sending-media#switching-cameras', + 'devices': '/guide/client/sending-media#devices', + 'receiving-media': '/guide/client/receiving-media', + 'video-frames': '/guide/client/receiving-media#video-frames', + 'audio': '/guide/client/receiving-media#audio', + 'receiving-without-sending': '/guide/client/receiving-media#receiving-without-sending', + 'recording': '/guide/client/receiving-media#recording', + 'encoded-transforms': '/guide/client/transforms', + 'data-channels': '/guide/client/data-channels', + 'whip-and-whep': '/guide/client/whip-whep', + 'manual-negotiation': '/guide/client/peers#manual-negotiation', + 'statistics': '/guide/client/statistics', + 'diagnosis-and-quality-events': '/guide/client/statistics#diagnosis-and-quality-events', + 'checking-before-a-call': '/guide/client/statistics#checking-before-a-call', + }, + '/guide/get-started': { + 'using-the-core-api': '/guide/core-api', + 'initialize-the-webrtc-library': '/guide/core-api#initialize-the-webrtc-library', + 'create-a-peer-connection': '/guide/core-api#create-a-peer-connection', + 'signaling': '/guide/core-api#signaling', + 'letting-the-peer-connection-create-the-description': '/guide/core-api#letting-the-peer-connection-create-the-description', + 'adding-candidates-and-knowing-the-outcome': '/guide/core-api#adding-candidates-and-knowing-the-outcome', + 'media-streams': '/guide/core-api#media-streams', + 'accessing-media-devices': '/guide/core-api#accessing-media-devices', + 'creating-media-tracks': '/guide/core-api#creating-media-tracks', + 'adding-tracks-to-peer-connection': '/guide/core-api#adding-tracks-to-peer-connection', + 'receiving-tracks': '/guide/core-api#receiving-tracks', + 'cleanup': '/guide/core-api#cleanup', + 'finding-objects-that-were-never-disposed': '/guide/core-api#finding-objects-that-were-never-disposed', + }, +} + +// Where the section a path and hash name moved to, relative to the site's base, if it did. +export function movedSection(path: string, hash: string): string | undefined { + const page = path.replace(/\.html$/, '').replace(/\/$/, '') + const anchor = decodeURIComponent(hash.replace(/^#/, '')) + + return anchor ? MOVED_SECTIONS[page]?.[anchor] : undefined +} diff --git a/docs/guide/advanced/audio-codecs.md b/docs/guide/advanced/audio-codecs.md index 23aaf1837..e18cf9688 100644 --- a/docs/guide/advanced/audio-codecs.md +++ b/docs/guide/advanced/audio-codecs.md @@ -1,5 +1,7 @@ # Audio Codecs + + A `PeerConnectionFactory` decides which audio codecs it can send and receive through an audio encoder factory and an audio decoder factory. Unless told otherwise it uses all codecs built into WebRTC. This guide explains how to limit them, for example to offer only Opus, or to put a codec first that a remote endpoint expects. The API lives in the `dev.onvoid.webrtc.media.audio.codec` package. diff --git a/docs/guide/advanced/encoded-transforms.md b/docs/guide/advanced/encoded-transforms.md index 5323e23f9..6c4297c5f 100644 --- a/docs/guide/advanced/encoded-transforms.md +++ b/docs/guide/advanced/encoded-transforms.md @@ -43,7 +43,7 @@ The transform belongs to the native sender or receiver, not to the Java object: ### With the Client API -The [Client API](/guide/client-api#encoded-transforms) takes the same transforms. Those of the peer options apply to every sender and remote track of the peer, and are set before the first frame passes either way: +The [Client API](/guide/client/transforms) takes the same transforms. Those of the peer options apply to every sender and remote track of the peer, and are set before the first frame passes either way: ```java PeerOptions.builder() diff --git a/docs/guide/advanced/end-to-end-encryption.md b/docs/guide/advanced/end-to-end-encryption.md index 616f45099..4c7d2e098 100644 --- a/docs/guide/advanced/end-to-end-encryption.md +++ b/docs/guide/advanced/end-to-end-encryption.md @@ -132,7 +132,7 @@ The frame format covers VP8, VP9, AV1 and every audio codec. WebRTC's packetizer H.264 and H.265 frames are split at their start codes before they are sent, which a ciphertext would contain anywhere, so they are dropped with `UNSUPPORTED_CODEC`. A call has to negotiate them away, which matters on macOS in particular, where WebRTC prefers H.264 through VideoToolbox: -- A peer of the client API with `e2ee(keys)` does so by itself: it offers and answers only VP8, VP9 and AV1 video, in WebRTC's order. `videoCodecs(...)` of the [WHIP and WHEP](/guide/client-api#whip-and-whep) builders still picks among them, and fails the session with a `ConfigurationException` if it names none of them. +- A peer of the client API with `e2ee(keys)` does so by itself: it offers and answers only VP8, VP9 and AV1 video, in WebRTC's order. `videoCodecs(...)` of the [WHIP and WHEP](/guide/client/whip-whep) builders still picks among them, and fails the session with a `ConfigurationException` if it names none of them. - Without the client API, call `setCodecPreferences` on each video transceiver with the codecs for which `KeyProvider.supportsCodec(codec.getMimeType())` is true, keeping `rtx`, `red` and `ulpfec`. ## Recording diff --git a/docs/guide/advanced/field-trials.md b/docs/guide/advanced/field-trials.md index ff9cd8253..b1e9209bd 100644 --- a/docs/guide/advanced/field-trials.md +++ b/docs/guide/advanced/field-trials.md @@ -1,5 +1,7 @@ # Field Trials + + WebRTC uses field trials to gate experimental features and tune internal behavior (e.g. congestion control, codec selection, message interleaving) without changing the public API. This guide explains how to enable them when creating a `PeerConnectionFactory`. Field trials are applied once, at `PeerConnectionFactory` creation, and take effect for everything created by that factory (peer connections, tracks, sources). Different `PeerConnectionFactory` instances in the same process may use different field trials. diff --git a/docs/guide/advanced/native-image.md b/docs/guide/advanced/native-image.md index 291a1e158..240fd84b4 100644 --- a/docs/guide/advanced/native-image.md +++ b/docs/guide/advanced/native-image.md @@ -103,7 +103,7 @@ in `src/main/resources/META-INF/native-image///reflect-config.j ## Example -The [`NativeImageExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/NativeImageExample.java) connects two peers in one process and checks that video, audio, data channel messages, statistics and log messages arrive. It exits with status 0 only if they all did, and builds with `--exact-reachability-metadata`, which needs GraalVM 24 or later: +The [`NativeImageExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/NativeImageExample.java) connects two peers in one process and checks that video, audio, data channel messages, statistics and log messages arrive. It exits with status 0 only if they all did, and builds with `--exact-reachability-metadata`, which needs GraalVM 24 or later: ```shell mvn -pl webrtc-examples -Pnative package diff --git a/docs/guide/advanced/video-codecs.md b/docs/guide/advanced/video-codecs.md index 86025a8cb..7e27a422f 100644 --- a/docs/guide/advanced/video-codecs.md +++ b/docs/guide/advanced/video-codecs.md @@ -1,5 +1,7 @@ # Video Codecs + + A `PeerConnectionFactory` decides which video codecs it can send and receive through a video encoder factory and a video decoder factory. Unless told otherwise it uses the codecs built into the library. This guide explains how to replace or extend them: to add a codec WebRTC does not have, to use a hardware encoder reached through another library, or to limit the codecs a factory offers. The API lives in the `dev.onvoid.webrtc.media.video.codec` package. diff --git a/docs/guide/audio/audio-devices.md b/docs/guide/audio/audio-devices.md index 70501c4e5..862075851 100644 --- a/docs/guide/audio/audio-devices.md +++ b/docs/guide/audio/audio-devices.md @@ -1,5 +1,7 @@ # Audio Devices + + This guide focuses on setting up a peer connection with audio device selection capabilities, which allows you to choose specific microphones and speakers for your WebRTC connection. ## Audio Device Selection diff --git a/docs/guide/audio/audio-processing.md b/docs/guide/audio/audio-processing.md index 36f0edd08..4a266bd99 100644 --- a/docs/guide/audio/audio-processing.md +++ b/docs/guide/audio/audio-processing.md @@ -1,5 +1,7 @@ # Audio Processing + + This guide explains how to use the audio processing capabilities provided by the `dev.onvoid.webrtc.media.audio.AudioProcessing` class. The library provides a collection of voice processing components designed for real-time communications software. ## Overview @@ -313,7 +315,7 @@ factory.addVoiceActivityListener(new VoiceActivityConfig(0.5, Duration.ofMillis( factory.removeVoiceActivityListener(listener); ``` -It works with the `AudioProcessing` the factory was built with as well as with the one it builds itself. All listeners share one detector, and each decides with the threshold and hangover of its own `VoiceActivityConfig`. Audio arrives while the audio device module records, which WebRTC starts once a peer connection sends a microphone track. A listener must not add or remove a listener. Disposing of the factory removes the listeners it still has. The [Client API](/guide/client-api#audio) offers the same through `LocalAudio.onVoiceActivity`. +It works with the `AudioProcessing` the factory was built with as well as with the one it builds itself. All listeners share one detector, and each decides with the threshold and hangover of its own `VoiceActivityConfig`. Audio arrives while the audio device module records, which WebRTC starts once a peer connection sends a microphone track. A listener must not add or remove a listener. Disposing of the factory removes the listeners it still has. The [Client API](/guide/client/receiving-media#audio) offers the same through `LocalAudio.onVoiceActivity`. ## Statistics diff --git a/docs/guide/audio/custom-audio-source.md b/docs/guide/audio/custom-audio-source.md index 01ad7c764..74b43dd25 100644 --- a/docs/guide/audio/custom-audio-source.md +++ b/docs/guide/audio/custom-audio-source.md @@ -1,5 +1,7 @@ # Custom Audio Source + + This guide explains how to use the `CustomAudioSource` class to push audio data from external sources directly to the WebRTC audio pipeline. ## Overview diff --git a/docs/guide/audio/dtmf-sender.md b/docs/guide/audio/dtmf-sender.md index 9bcc91711..bc80c3f06 100644 --- a/docs/guide/audio/dtmf-sender.md +++ b/docs/guide/audio/dtmf-sender.md @@ -1,5 +1,7 @@ # DTMF Sender + + This guide explains how to use the DTMF (Dual-Tone Multi-Frequency) sender functionality with the webrtc-java library. DTMF senders allow you to send DTMF tones over WebRTC audio connections, which is useful for interactive voice response (IVR) systems and telephony applications. ## Overview diff --git a/docs/guide/audio/headless-audio.md b/docs/guide/audio/headless-audio.md index 1444988fe..88cae83de 100644 --- a/docs/guide/audio/headless-audio.md +++ b/docs/guide/audio/headless-audio.md @@ -1,5 +1,7 @@ # Headless Audio + + The `HeadlessAudioDeviceModule` is a convenience implementation of the `AudioDeviceModule` that uses WebRTC's dummy audio layer. It avoids touching real OS audio devices while still driving the WebRTC render pipeline, which is what delivers received audio to an `AudioTrackSink`. It has no capture path; audio you want to send goes through a `CustomAudioSource`. This is ideal for: @@ -130,7 +132,7 @@ factory.dispose(); - Using a standard `AudioDeviceModule` with `AudioLayer.kDummyAudio` disables actual audio I/O; the audio pipeline is not started for playout and sinks will typically not receive audio frame callbacks. Use this only when you intentionally do not want any audio delivery (e.g., video‑only or fully custom audio). Related guides: -- [Audio Device Selection](/guide/audio/audio-devices) +- [Audio Devices](/guide/audio/audio-devices) - [Custom Audio Source](/guide/audio/custom-audio-source) ## Limitations and notes diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index 6b75fab70..142256768 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -38,6 +38,21 @@ What the client API takes care of: The client API is plain Java on top of the classes the other guides describe, and it does not replace them. Its peers, tracks and channels are the same native objects, and [Using the Low-Level API](#using-the-low-level-api) shows how to reach them for anything it does not cover. [Client API Design](/guide/client-api-design) explains the decisions behind it. +## Pages of This Guide + +The client API is described on these pages: + +| Page | What it covers | +|---|---| +| [Peers and Signaling](/guide/client/peers) | Creating peers, carrying their messages, connection state, ICE recovery and manual negotiation. | +| [Threads](/guide/client/threads) | Where handlers run, limiting threads, the UI thread, and why a handler must not wait. | +| [Sending Media](/guide/client/sending-media) | Cameras, microphones, screens, files, pushed audio and video, senders, DTMF and devices. | +| [Receiving Media](/guide/client/receiving-media) | Remote tracks, video frames, video views for Swing and JavaFX, audio and recording. | +| [Transforms and Encryption](/guide/client/transforms) | Encoded transforms and end-to-end encryption of a peer's media. | +| [Data Channels](/guide/client/data-channels) | Messages between peers, flow control and channel options. | +| [WHIP and WHEP](/guide/client/whip-whep) | Publishing to and playing from media servers. | +| [Statistics and Call Checks](/guide/client/statistics) | Connection statistics, diagnosis, and checking devices and network before a call. | + ## The Runtime A `WebRtc` runtime owns a `PeerConnectionFactory`, its audio device module and the threads of the client API. `WebRtc.create()` captures the microphone and plays to the speakers. The builder sets everything else: @@ -59,16 +74,16 @@ WebRtc rtc = WebRtc.builder() | `audioProcessing(AudioProcessingConfig)` | none | The [audio processing](/guide/audio/audio-processing) of captured audio. | | `hardwareVideoCodecs(boolean)` | `false` | Encodes and decodes video on the GPU where it can, as the [Video Codecs](/guide/advanced/video-codecs) guide describes. | | `fieldTrials(Map)` | none | [Field trials](/guide/advanced/field-trials). | -| `eventExecutor(Executor)` | threads of the runtime | Where handlers run, see [Threads](#threads). | -| `eventThreads(int)` | unlimited | The most threads the runtime runs handlers on, see [Limiting Threads](#limiting-threads). | +| `eventExecutor(Executor)` | threads of the runtime | Where handlers run, see [Threads](/guide/client/threads). | +| `eventThreads(int)` | unlimited | The most threads the runtime runs handlers on, see [Limiting Threads](/guide/client/threads#limiting-threads). | | `frameThreads(int)` | unlimited | The most threads that run `onFrame` and `onAudio` handlers. | -| `audioBuffer(Duration)` | 5 seconds | How much audio an `onAudio` subscription keeps for a handler that falls behind, see [Audio](#audio). | +| `audioBuffer(Duration)` | 5 seconds | How much audio an `onAudio` subscription keeps for a handler that falls behind, see [Audio](/guide/client/receiving-media#audio). | | `errorHandler(Consumer)` | the thread's uncaught exception handler | Receives what handlers throw, and failures nobody else hears of. | | `operationTimeout(Duration)` | 30 seconds | How long an operation such as applying a description may take before it fails with a `TimeoutException`. | | `closeTimeout(Duration)` | 10 seconds | How long `close()` waits. | | `configureFactory(Consumer)` | none | Anything else of the factory, such as codec factories of your own. | -Every timeout of the client API, these two as well as the gathering timeout and the `Whip` and `Whep` timeouts below, must be positive; none of them can be switched off. A timeout counts in milliseconds, rounded up, so one under a millisecond waits a millisecond. One too long for milliseconds to count waits as long as they can count, about 292 million years. +The timeouts are described under [Details](#timeouts). ### Audio Modes @@ -82,1024 +97,6 @@ The runtime's `AudioMode` decides whether it may open the microphone and where i Asking a `CUSTOM` or `HEADLESS` runtime for the microphone fails at once with a `ConfigurationException` that names the mode, rather than when the first track is added. The microphone is opened only while one of its tracks is sent. -## Peers and Signaling - -`rtc.createPeer(options)` creates a connection to one remote peer. It is the one call of the client API that waits, briefly, for the connection to be created, so that a configuration WebRTC refuses fails right there with a `ConfigurationException`. - -```java -Peer peer = rtc.createPeer(PeerOptions.builder() - .name("alice") - .iceServer("stun:stun.l.google.com:19302") - .iceServer("turn:turn.example.com:3478", "user", "secret") - .polite(true) - .signaling(message -> socket.send(message.toJson())) - .build()); -``` - -| Method | What it sets | -|---|---| -| `name(String)` | The name in thread names and error messages. | -| `iceServer(String...)` | A STUN or TURN server without credentials. | -| `iceServer(String, String, String)` | A TURN server with a user name and password. | -| `relayOnly(boolean)` | Connects through TURN servers only, which hides this side's addresses. | -| `signaling(Signaling)` | What carries messages to the remote peer. Required unless negotiation is manual. | -| `polite(boolean)` | The role in perfect negotiation, see below. | -| `negotiation(Negotiation)` | `AUTOMATIC` by default, or `MANUAL`, see [Manual Negotiation](#manual-negotiation). | -| `receive(MediaKind...)` | Receives audio or video without sending any, as a viewer does. | -| `gatheringTimeout(Duration)` | How long `Gathering.COMPLETE` waits for ICE gathering, 3 seconds by default. See [Manual Negotiation](#manual-negotiation). | -| `channelSendLimit(long)` | How many bytes each data channel may hold waiting to be sent, 16 MiB by default. See [Data Channels](#data-channels). | -| `channelReceiveLimit(long)` | How many bytes of received messages each data channel may hold that its handlers are not done with, 16 MiB by default. See [Data Channels](#data-channels). | -| `configure(Consumer)` | Anything else the [peer connection configuration](/guide/networking/peer-connection-config) has. | - -### Signaling - -The library has no transport of its own. A `Signaling` sends a peer's messages to the remote peer, and messages from the remote peer are handed to `peer.signal(...)`, in the order they were sent: - -```java -// Outgoing: called on the peer's event thread, in order. -Signaling signaling = message -> socket.send(message.toJson()); - -// Incoming: from any thread. -socket.onText(text -> peer.signal(SignalMessage.fromJson(text))); -``` - -A transport class can implement both directions by overriding `Signaling.attach(Consumer)`, which the peer calls once when it is created, with the consumer to push received messages into. It returns a `Subscription` that removes that consumer, and only that one: the peer closes it when it closes, after its last `send`, on its event thread. The transport itself stays open, so a WebSocket can outlive the peer and take the next one, or serve several peers at once. A transport that keeps the consumer must drop it there, or the closed peer and the runtime with it stay reachable: - -```java -@Override -public synchronized Subscription attach(Consumer inbound) { - this.inbound = inbound; - - return () -> { - synchronized (this) { - // A peer closed after the next one attached leaves that one in place. - if (this.inbound == inbound) { - this.inbound = null; - } - } - }; -} -``` - -Each end of `Signaling.loopback()` takes one peer at a time; once that peer has closed, the next one can attach, and receives what was sent to the end in between. - -A `SignalMessage` is a session description (`OFFER`, `ANSWER`) or an ICE candidate (`CANDIDATE`). `toJson()` writes the shapes the browser API uses, so a JavaScript peer can pass them to `setRemoteDescription` and `addIceCandidate` as they are: - -```json -{"type":"offer","sdp":"v=0\r\no=- 4611731400430051336 2 IN IP4 127.0.0.1\r\n..."} -{"candidate":"candidate:842163049 1 udp 1677729535 203.0.113.7 49203 typ srflx raddr 0.0.0.0 rport 0","sdpMid":"0","sdpMLineIndex":0} -``` - -`SignalMessage.fromJson(String)` reads these, what a browser's `JSON.stringify(pc.localDescription)` and `JSON.stringify(candidate)` write, and the `{"description": ...}` and `{"candidate": {...}}` envelopes of the perfect negotiation examples. Anything else throws an `IllegalArgumentException` that says what is wrong and where, as does text longer than a MiB or nested deeper than 64 levels, so a message from the remote side cannot exhaust the stack. The JSON must be strict (RFC 8259): no raw line breaks or other control characters inside strings, no member name twice in one object, no numbers like `01`, `1.` or `1e999`. An `sdpMLineIndex` must be an integer from 0 to 65535; `1.9` is rejected rather than truncated to 1. Routing, such as room names or peer ids, belongs in an envelope of your own around this JSON. - -A browser peer that only answers needs no more than this: - -```js -socket.onmessage = async ({ data }) => { - const message = JSON.parse(data); - - if (message.type) { - await pc.setRemoteDescription(message); - - if (message.type === "offer") { - await pc.setLocalDescription(); - socket.send(JSON.stringify(pc.localDescription)); - } - } - else if (message.candidate) { - await pc.addIceCandidate(message); - } -}; - -pc.onicecandidate = ({ candidate }) => candidate && socket.send(JSON.stringify(candidate)); -``` - -A browser peer that adds tracks of its own sends offers too, and then has to implement perfect negotiation as well, as the W3C example linked above does. - -For tests and demos, `Signaling.loopback()` links two peers in the same process: - -```java -SignalingPair link = Signaling.loopback(); - -Peer caller = rtc.createPeer(PeerOptions.builder().signaling(link.first()).build()); -Peer callee = rtc.createPeer(PeerOptions.builder().signaling(link.second()).polite(true).build()); -``` - -### Polite and Impolite Peers - -With automatic negotiation, a peer sends an offer whenever its tracks or channels change. When both peers send one at the same moment, the **polite** peer drops its own offer and answers the other one, and the impolite peer ignores the offer it got. Exactly one of two peers must be polite. Peers are impolite by default, so set `polite(true)` on one side, for example on the peer that joined second. - -When two impolite peers collide, neither gives way. An impolite peer that waited 5 seconds for an answer reports a `NegotiationException` that says to make one of them polite. - -### Peer State - -`peer.state()` returns the last state the connection reported: - -| State | Meaning | -|---|---| -| `NEW` | Created, nothing negotiated yet. | -| `CONNECTING` | ICE and DTLS are establishing the connection. | -| `CONNECTED` | Media and data can flow. | -| `DISCONNECTED` | Connectivity was lost. It may come back by itself. | -| `FAILED` | The connection failed. `peer.restartIce()` or the configured recovery policy tries to recover it. | -| `CLOSED` | The peer was closed. This state is final. | - -```java -peer.onStateChange(state -> statusLabel.setText(state.toString())); - -peer.whenConnected().thenAccept(p -> System.out.println("Connected")); -peer.whenClosed().thenRun(() -> System.out.println("Closed")); -``` - -`onStateChange` calls a new handler with the current state first, so a handler added late still learns where the peer stands. `whenConnected()` fails if the connection fails or closes first. - -### Automatic ICE Recovery - -Automatic peers can opt into a `RecoveryPolicy`. Without one, the application still -calls `peer.restartIce()` itself. The policy waits through short disconnections, -restarts ICE on the same peer, and bounds retries with exponential backoff: - -```java -RecoveryPolicy recovery = RecoveryPolicy.builder() - .grace(Duration.ofSeconds(2)) - .maxAttempts(3) - .backoff(Duration.ofSeconds(1), Duration.ofSeconds(30)) - .attemptTimeout(Duration.ofSeconds(15)) - .onEvent(event -> System.out.println(event.state() + " attempt " + event.attempt())) - .build(); - -Peer peer = rtc.createPeer(PeerOptions.builder() - .signaling(signaling) - .recovery(recovery) - .build()); -``` - -`DISCONNECTED` starts the grace period; `FAILED` skips it. Repeated failure -notifications during an attempt do not start parallel restarts. Events are -`WAITING`, `ATTEMPTING`, `RETRYING`, `RECOVERED` and `EXHAUSTED`, delivered on the -peer's event executor. `attempt()` counts attempts in the current outage, starting -at one; a recovery during the grace period reports zero. `cause()` gives the last -failure on retry or exhaustion. Successful signaling alone does not count as -recovery: the peer must report `CONNECTED`. That resets the budget, including -after exhaustion. Exhaustion leaves the peer open and creates no further automatic -attempts until it connects again. `whenConnected()` keeps its existing first-connection -semantics; observe `RECOVERED` for later recovery. - -Before an attempt, `iceServers(attempt -> completionStage)` can obtain fresh TURN -credentials asynchronously. It runs on the event executor and must return promptly; -the resulting list replaces **all** ICE servers, so include STUN servers that should -remain available. The library copies the supplied servers; failed, null or invalid -results fail the attempt. For example, with an application's asynchronous credential -service returning `CompletionStage>`: - -```java -RecoveryPolicy recovery = RecoveryPolicy.builder() - .iceServers(attempt -> credentials.fetchIceServers()) - .onEvent(event -> { - if (event.state() == RecoveryEvent.State.EXHAUSTED) { - showReconnectAction(event.cause()); - } - }) - .build(); -``` - -The attempt deadline covers credentials, signaling and waiting for connectivity. -An HTTP restart already in flight finishes or reaches its own operation timeout -before another starts. Late credentials and completions after recovery or `close()` -are ignored; the credential service owns cancellation of its requests. Closing the -peer or runtime cancels scheduled retries, including one whose timer has fired but -whose control task has not run yet. Event notifications already queued may still arrive. - -ICE recovery requires working signaling. Reconnecting a WebSocket or refreshing -HTTP authorization is the application's separate responsibility. A policy does not -create a replacement peer or session. If the application chooses that fallback, it -must close the old session, attach handlers to the new peer, republish local tracks, -and handle new remote tracks and data channels explicitly; their old handles are -not transferred. - -`Whip.to(endpoint).recovery(recovery)` and `Whep.from(endpoint).recovery(recovery)` -use the same policy with the session's ICE `PATCH` exchange. This replaces their -built-in restart rule for that session. Keep using `session.restartIce()` for a -manual HTTP restart; `session.peer().restartIce()` does not perform the HTTP -exchange. For a manually negotiated peer outside WHIP/WHEP, the application owns -recovery and its protocol-specific offer/answer exchange. - -## Threads - -Every method of the client API may be called from any thread, including from inside a handler. Methods that change a peer return at once and take effect in the order they were called. - -| Code | Runs on | May it block? | -|---|---|---| -| Your calls into the API | your thread | They don't block, with these exceptions. `WebRtc.createPeer`, the `LocalMedia` methods that make a track and `LocalTrack.setEnabled` and `LocalVideo.setContentHint` wait briefly for WebRTC. `camera()` and `screen()` also open their capture on your thread, which can take longer, so a UI thread should leave them to another thread. The first `media.devices()`, `screens()` and `windows()` ask the system on your thread. `WebRtc.close()` waits up to the close timeout. | -| Peer, track and channel handlers, future callbacks, `Signaling.send` | the peer's event thread | Yes, but that delays only the events of this peer. They must not wait for a future of this API, see below. | -| `onFrame` and `onAudio` handlers | a thread of each subscription | Better not. A slow video handler gets the newest frame next, and the frames in between are dropped; a slow audio handler loses the oldest audio beyond its buffer. | -| [Encoded transforms](#encoded-transforms) | a thread of each sender and receiver | Better not. A slow transform delays only its own frames, and one that falls several seconds behind has frames dropped. | -| `LocalTrack.onEnded` and `Devices.onChange` handlers, callbacks of the futures of local tracks and devices | the event thread of the runtime's local media | Yes, but that delays the other events of local media. They must not wait for a future of this API either. A track is released before its `onEnded` handlers run, so they never hold up a release. | -| WebRTC's own callbacks | WebRTC's threads | Not your code. The library only copies what they report and queues it. | - -The handlers of one peer never run at the same time, and they run in the order of the events. The handlers of different peers may run at the same time. With the default executor, the event thread is named after its peer, such as `webrtc-client-events-alice`, which shows up in thread dumps. - -An executor of your own that rejects work, such as a bounded pool that is full, loses the events it rejects, and the rejection goes to the error handler. The futures those events would have completed complete all the same, on the thread that found them rejected. Once the runtime is closed, every future of this API that is not done fails with `PeerClosedException`, including those of calls made after the close. - -### Limiting Threads - -By default the runtime starts a thread for each peer whose events wait and for each subscription whose frames wait, so a thousand peers with slow handlers take a thousand threads. `eventThreads(n)` and `frameThreads(n)` cap them: - -```java -WebRtc rtc = WebRtc.builder() - .eventThreads(8) // handlers of all peers, future callbacks, Signaling.send - .frameThreads(4) // onFrame and onAudio handlers, local previews included - .build(); -``` - -With a limit, the peers take turns, and so do the subscriptions: one that has work runs up to 64 handlers, for at most 10 ms, then goes to the back of the line. A slow handler thus delays the others but never starves them, and a peer's events still run one at a time and in order. The limit rejects nothing: events wait for their turn and are never dropped, so no future stays open, and once the runtime closes, the futures still open fail after the events queued before them. Media that waits is bounded by its subscription instead, by the `FrameDelivery` of a video handler and the audio buffer of an audio handler, which drop the oldest frames and audio when they are full. - -A limit has one cost: a handler that waits for what another peer's handler does, say on a latch, can wait forever once all threads are taken by such waits. Handlers that wait for something outside the API should hand the work to a thread of their own. - -The control threads of the library, which call into WebRTC, are not limited. Their tasks wait for one another, such as a camera switch for every peer to take the new track. - -The HTTP requests of `Whip` and `Whep` sessions take no thread while they wait, so a slow server holds up no peer: they go through Java's `HttpClient`, one of each runtime. - -`rtc.loadStats()` tells how much work waits and runs, at this moment and cheaply enough to be read every second: - -```java -LoadStats load = rtc.loadStats(); - -LoadStats.Work events = load.events(); -metrics.gauge("webrtc.events.queued", events.queued()); // events waiting for a thread -metrics.gauge("webrtc.events.oldest_ms", events.oldestWaiting().toMillis()); // how long the oldest has waited -metrics.gauge("webrtc.events.active", events.active()); // handlers running now -metrics.gauge("webrtc.video.dropped", load.video().dropped()); // frames dropped since start -metrics.gauge("webrtc.audio.dropped", load.audio().dropped()); // audio chunks dropped since start -metrics.gauge("webrtc.event_threads", load.eventThreads()); -``` - -| Part | What waits | What `dropped()` counts | -|---|---|---| -| `control()` | tasks of the library that call into WebRTC | tasks the control pool rejected while running, which it never does | -| `events()` | events of all peers, for handlers, future callbacks and `Signaling.send` | events an `eventExecutor` of your own rejected | -| `video()` | frames for `onFrame` handlers | frames a handler had no time for | -| `audio()` | chunks for `onAudio` handlers | chunks beyond the audio buffer | - -A queue that keeps growing, or an oldest wait that keeps getting longer, tells that handlers do not keep up. A higher limit helps handlers that wait for I/O; handlers that compute only spread the delay over more threads. - -### Handlers on the UI Thread - -Pass the UI toolkit's executor, and every handler may touch UI components directly: - -```java -WebRtc rtc = WebRtc.builder() - .eventExecutor(SwingUtilities::invokeLater) // or Platform::runLater for JavaFX - .build(); - -peer.onStateChange(state -> statusLabel.setText(state.toString())); -hangUpButton.addActionListener(event -> peer.close()); // never blocks the EDT -``` - -The order of each peer's events is kept with any executor, including a thread pool. Frame handlers stay on their own threads even then, since frames come far too often for a UI thread. - -The `webrtc-java-swing` module has a `VideoView` in `dev.onvoid.webrtc.client.swing` that shows video in a Swing window. The core `webrtc-java` module does not depend on `java.desktop`, so headless applications and trimmed runtime images need neither AWT nor Swing: - -```xml - - dev.onvoid.webrtc - webrtc-java-swing - {{ VERSION }} - -``` - -The view converts each frame on the frame's own thread and leaves only the painting to the EDT. It keeps the picture's aspect ratio and rotation, and can mirror a local camera preview: - -```java -import dev.onvoid.webrtc.client.swing.VideoView; - -VideoView remoteView = new VideoView(); -VideoView preview = new VideoView(); -preview.setMirrored(true); - -peer.onTrack(track -> { - if (track.isVideo()) { - track.asVideo().onFrame(remoteView); - } -}); -camera.onFrame(preview); - -frame.add(remoteView, BorderLayout.CENTER); -``` - -`snapshot()` returns a copy of the picture shown, for saving a still or for tests. - -Both views support resolutions such as 8K without a fixed pixel limit. They reject -nonpositive dimensions and frames whose four-byte pixel data cannot fit in a Java -buffer, using overflow-safe arithmetic. Conversion buffers allocate on demand, -grow geometrically when necessary, and retain their capacity across smaller frames -and `clear()`. Swing reuses one direct buffer; JavaFX reuses up to two. -JavaFX reserves a buffer only while copying it into the displayed image; a waiting -frame may be replaced by a newer frame to avoid building a queue. Buffer -memory is released when the view becomes unreachable. Memory usage follows the -largest frames received and includes displayed images, conversion buffers, and -replacement allocations awaiting garbage collection. Snapshots, input frames and -toolkit rendering resources use additional memory. - -For JavaFX, the `webrtc-java-javafx` module has a `VideoView` node in `dev.onvoid.webrtc.client.javafx` that works the same way. It needs Java 11 and the JavaFX your application already uses: - -```xml - - dev.onvoid.webrtc - webrtc-java-javafx - {{ VERSION }} - -``` - -```java -import dev.onvoid.webrtc.client.javafx.VideoView; - -WebRtc rtc = WebRtc.builder() - .eventExecutor(Platform::runLater) // handlers on the JavaFX Application Thread - .build(); - -VideoView remoteView = new VideoView(); -peer.onTrack(track -> { - if (track.isVideo()) { - track.asVideo().onFrame(remoteView); - } -}); - -stage.setScene(new Scene(new StackPane(remoteView), 960, 540)); -``` - -The view converts frames on their own thread and paints at most once per JavaFX pulse. `setMirrored(true)`, or its `mirroredProperty()`, flips a camera preview, and `snapshot()`, on the JavaFX Application Thread, returns a copy of the picture. The `JavaFxCameraExample` in the [examples](/guide/examples#javafx-camera) shows a preview and a received track side by side. - -### Don't Wait in a Handler - -A future of the client API completes on the event thread of its peer. A handler that waits for one would wait for its own thread, so `join()` and `get()` there throw a `BlockingCallException` instead of hanging. Chain the next step instead: - -```java -peer.onChannel(channel -> { - // channel.whenOpen().join() would throw here. - channel.whenOpen().thenCompose(open -> open.send("hello")); -}); -``` - -A future that is already done may be read anywhere, and any thread that is not an event thread, such as `main`, may wait. - -Each call returns a future of its own, also of `closeAsync()`, `whenClosed()`, `whenConnected()` and `whenOpen()`. Cancelling one only stops waiting: the close goes on, and the futures of other callers, and of the runtime's own close, are not affected. - -Futures derived from them with `thenApply` and the like refuse to be waited on as well. - -::: warning -A UI thread used as event executor does not refuse: waiting on it for a future of this API hangs the UI, since the future completes on it. -::: - -## Sending Media - -`rtc.media()` creates the tracks a runtime sends, and `peer.addTrack(track)` sends one. With automatic negotiation, the peer renegotiates by itself. - -```java -LocalMedia media = rtc.media(); - -LocalCamera camera = media.camera(CameraOptions.builder() - .device(media.devices().cameras().get(0)) - .resolution(1280, 720) - .frameRate(30) - .build()); -LocalAudio microphone = media.microphone(); - -Sender videoSender = peer.addTrack(camera); -Sender audioSender = peer.addTrack(microphone); -``` - -| Method | Track | -|---|---| -| `camera()`, `camera(CameraOptions)` | A camera, as a `LocalCamera` that can switch cameras. The supported capability nearest to the requested resolution and frame rate is used. | -| `microphone()`, `microphone(AudioOptions)` | The microphone, with echo cancellation, noise suppression and gain control unless `AudioOptions` say otherwise. `DEVICES` mode only. | -| `screen(ScreenOptions)` | A screen or a window, as the [Desktop Video](/guide/video/desktop-capture) guide describes. | -| `customVideo(String)` | Frames you push. | -| `customAudio(String)` | Audio you push, in every mode. | -| `videoTrack(String, VideoTrackSource)`, `audioTrack(String, AudioTrackSource)` | A source of the core API, such as one the [media module](/guide/media/media-files) feeds, or a [`SystemAudioSource`](/guide/audio/system-audio-source) for the audio the system plays. The source stays yours. | - -```java -Devices devices = media.devices(); -LocalVideo screen = media.screen(ScreenOptions.screen(devices.screens().get(0)).frameRate(15).build()); -``` - -A content hint tells the senders of a video track what to keep when the bandwidth or the CPU runs short. It can be changed at any time, also while the track is sent, without renegotiating: - -```java -screen.setContentHint(VideoContentHint.TEXT); // source code: keep the resolution -peer.addTrack(screen); - -screen.setContentHint(VideoContentHint.MOTION); // a video starts in the shared window: keep the frame rate -``` - -| Hint | Suits | Gives up first | -|---|---|---| -| `DEFAULT` | Anything | What the source suggests: the frame rate of screens, the resolution of other video | -| `MOTION` | Cameras, videos, animations | Resolution | -| `DETAIL` | Drawings, slides, photos | Frame rate | -| `TEXT` | Source code, terminals, tables | Frame rate; WebRTC treats it as `DETAIL` for now | - -The hint belongs to the track, so it applies on every peer the track is sent on, and a camera keeps it when it switches. Screens keep their resolution by default already, so the hint pays off where the content changes, or for a camera that shows a whiteboard. No hint keeps text readable at any bitrate. - -Tracks added without stream ids share one stream per peer, so that a browser plays their audio and video in sync. Pass ids to `addTrack(track, streamIds...)` to group them yourself. - -One track may be sent on any number of peers. Opening a camera that is already open shares its capture when the options are the same, and fails with a `MediaException` when they differ. - -### Sending a Media File - -The media module plays files and RTSP streams into sources of the core API. `videoTrack` and `audioTrack` put them on peers like any other track: - -```java -WebRtc rtc = WebRtc.builder().audio(AudioMode.HEADLESS).build(); // first: it loads the native library -MediaFileSource file = new MediaFileSource(Paths.get("movie.mp4")); - -peer.addTrack(rtc.media().videoTrack("movie", file.getVideoSource())); -peer.addTrack(rtc.media().audioTrack("movie", file.getAudioSource())); -file.play(); -``` - -Create the runtime before the file: the runtime loads the native library that the file's sources need. The sources belong to the `MediaFileSource`, not to the tracks, so close the runtime, or the tracks, before the file: `track.closeAsync()` returns a future that completes once the track let go of its source. The file's audio source is pushed audio, so the runtime needs the `CUSTOM` or `HEADLESS` audio mode. - -### Pushing Video and Audio - -```java -CustomVideo video = media.customVideo("render"); - -VideoFrame frame = new VideoFrame(buffer, System.nanoTime()); -video.push(frame); -frame.release(); -``` - -`push(frame)` treats the frame as captured now. A source with timing of its own, such as a file, passes a timestamp with `push(frame, timestampMicros)`; see [Custom Video Source](/guide/video/custom-video-source) for why that matters. - -```java -CustomAudio voice = media.customAudio("voice"); - -// 16-bit little-endian PCM, any length, from any thread. -voice.push(pcm, PcmFormat.of(48000, 1)); -``` - -`CustomAudio` takes PCM in pieces of any length and from any thread. It cuts the audio into the 10 ms chunks WebRTC works with and sends one every 10 ms from a thread of its own, so pushing a whole second at once is as fine as pushing every 10 ms. `queuedMillis()` tells how much is still waiting, and `clear()` drops it. That thread does nothing else, so a slow push into WebRTC delays only the audio, not the timeouts of the runtime. Should it stall, as in a long garbage collection pause, it makes up for at most the last 50 ms instead of sending the whole pause at once; the queued audio then stays behind by the rest, until `clear()` drops it. - -A producer faster than real time would otherwise fill memory and push the delay up without end, so the queue holds at most 10 seconds, and a `push` beyond that fails with an `AudioQueueFullException`, keeping none of its audio. `queueLimit(max, overflow)` sets another limit and what happens beyond it: - -```java -// Live audio: stay at most 200 ms behind, dropping the oldest audio. -voice.queueLimit(Duration.ofMillis(200), AudioOverflow.DROP_OLDEST); - -// A file, which must arrive whole: refuse the push, to try it again later. -voice.queueLimit(Duration.ofSeconds(2), AudioOverflow.REJECT); -``` - -With `REJECT` a refused push can be pushed again once `queuedMillis()` has shrunk; with `DROP_OLDEST` every push is taken, and the audio sent lags at most the limit behind. - -### Controlling Senders and Tracks - -```java -camera.setEnabled(false); // sends black frames, the camera stays open - -videoSender.configure(Encoding.builder() - .maxBitrate(1_500_000) - .maxFrameRate(30) - .build()); - -videoSender.replaceTrack(screen); // no renegotiation -audioSender.remove(); // renegotiates; the track stays open - -camera.close(); // leaves every peer and stops the camera -``` - -Closing a track stops sending it on every peer, without renegotiating, and then releases the capture. The senders stay, and can be given another track with `replaceTrack`. - -An audio sender also sends DTMF tones, for telephone systems and voice menus. The future completes once the tones have been sent, and `onTone` reports each tone as it starts, then an empty string at the end: - -```java -audioSender.onTone(tone -> System.out.println(tone.isEmpty() ? "Done" : "Tone " + tone)); -audioSender.sendTones("1234#"); // 100 ms each -audioSender.sendTones("*99,1", Duration.ofMillis(200), Duration.ofMillis(100)); -``` - -The characters 0 to 9, A to D, `#` and `*` are tones, and a comma is a pause of 2 seconds. Tones still queued from an earlier call are replaced, and that call's future is cancelled. A video sender, or one whose remote peer accepts no telephone events, fails the future with an `IllegalStateException`. So do tones WebRTC stops sending midway, for example because the sender was paused with `configure`, a few seconds after their planned end. The [DTMF Sender](/guide/audio/dtmf-sender) guide has the details. - -### Switching Cameras - -A `LocalCamera` switches to another camera without renegotiating. Every peer that sends it keeps sending it, and preview handlers keep getting frames: - -```java -LocalCamera camera = media.camera(); -peer.addTrack(camera); - -camera.switchCamera(media.devices().cameras().get(1)) - .thenRun(() -> System.out.println("Now sending " + camera.device().getName())); -``` - -The new camera keeps the resolution and frame rate of the old one, as far as it supports them. The switch opens the new camera before it closes the old one, so that the peers see no gap. Some devices cannot capture with two cameras at once, such as the two sensors of a laptop camera; then the old camera closes first, and the switch takes about half a second. If the new camera cannot be opened, the future fails with a `MediaException`, and the track keeps sending the camera it had. If a peer does not take the new camera within the operation timeout, the future fails with a `WebRtcException`; the track goes back to the old camera, or keeps the new one when the old one had to close first. A track that closed its camera first and can open neither camera ends. - -### Devices - -`media.devices()` returns the cameras, microphones and speakers. The first call reads them from the system; after that, the lists are snapshots kept current in the background, so reading them is cheap on any thread. Where the system reports no device changes, as on a Linux machine without PulseAudio, they are read on each call instead. `screens()` and `windows()` ask the system on each call. A query that fails, including an unavailable audio backend, throws a `MediaException`, so an empty list means there are no such devices. Its cause is an `AudioBackendUnavailableException` when the audio backend is missing or unreachable, or a `DeviceException` for other native device failures. - -```java -Devices devices = media.devices(); - -devices.onChange(changed -> System.out.println("Cameras: " + changed.cameras())); -devices.useMicrophone(devices.microphones().get(1)); -devices.useSpeaker(devices.speakers().get(0)); -``` - -`useMicrophone` and `useSpeaker` switch devices also for tracks that are sent or played now. A device that was disconnected since the list was read fails the returned future, and the device in use stays as it was. On Linux, where WebRTC's audio device module cannot tell devices apart, the default device is used whichever is given. - -A track does not end by itself when its device is disconnected or a captured window closes. `LocalTrack.onEnded` follows only from closing the track, or from a camera switch that can open neither camera. `onChange` tells of a device that went away; closing the track, or switching a camera to another device, is up to you. Nothing tells of a captured window that closes. - -## Receiving Media - -`peer.onTrack` is called for every track the remote peer sends. Tracks that already arrived are handed to a handler added later. - -```java -peer.onTrack(track -> { - System.out.println("Receiving " + track.kind() + " in " + track.streamIds()); - - if (track.isVideo()) { - track.asVideo().onFrame(frame -> view.render(frame)); - } - else { - track.asAudio().onAudio(chunk -> recognizer.feed(chunk.samples())); - } - - track.onMute(muted -> System.out.println("Muted: " + muted)); - track.onEnded(() -> System.out.println("Track ended")); -}); -``` - -A track ends when the remote peer stops sending it or the peer closes. - -### Video Frames - -Each `onFrame` subscription has a thread of its own, so a slow renderer holds up neither WebRTC's decoder nor the peer's other events. With `frameThreads(n)` the subscriptions share `n` threads and take turns, see [Limiting Threads](#limiting-threads). By default the handler always gets the newest frame, and the frames it had no time for are dropped. `FrameDelivery.queue(n)` keeps up to `n` frames instead: - -```java -video.onFrame(FrameDelivery.queue(10), frame -> encoder.encode(frame)); -``` - -A frame is valid while the handler runs. Call `frame.retain()` to keep it longer, and `frame.release()` when done. - -### Audio - -`onAudio` hands decoded audio to a handler in order, usually 10 ms of 16-bit PCM per `AudioChunk`. Audio reaches these handlers only while the runtime plays out, which a `HEADLESS` runtime always does and a `DEVICES` or `CUSTOM` runtime does to its speakers. `setVolume(double)` changes the playout gain of a track, from 0 to 10, where 1 plays it as received. - -A handler that falls behind gets the audio of the last 5 seconds; the oldest audio beyond that is dropped and counted in `rtc.loadStats().audio().dropped()`. `WebRtc.Builder.audioBuffer(Duration)` changes the buffer of every subscription, and `onAudio(Duration, handler)` that of one. A speech recognizer that must stay live keeps a short buffer; a recorder that can catch up after a stall keeps a long one, at about 190 KiB per second of 48 kHz stereo: - -```java -track.asAudio().onAudio(Duration.ofMillis(200), chunk -> recognizer.feed(chunk.samples())); -``` - -A local audio track tells when this side speaks. `onVoiceActivity` calls its handler with `true` when speech begins and `false` when it ends, on the event executor. The audio is analyzed in native code, and only these changes reach Java: - -```java -LocalAudio microphone = rtc.media().microphone(); -Subscription speaking = microphone.onVoiceActivity( - new VoiceActivityConfig(0.6, Duration.ofMillis(400)), - active -> System.out.println(active ? "speaking" : "quiet")); -``` - -`VoiceActivityConfig` sets the threshold, the voice probability from 0 to 1 at which audio counts as speech (0.5 by default), and the hangover, how long speech goes on once the audio fell below it (300 ms by default), which bridges the pauses between words. Pushed audio is analyzed as pushed. A microphone's audio is analyzed after audio processing, with echo cancellation, noise suppression and gain control applied as configured, so the remote peer's voice from the speakers does not count as this side speaking. It flows while WebRTC records, which it does while a peer sends a microphone track. Every microphone track of a runtime hears the same, as WebRTC records one microphone for all of them. - -### Receiving Without Sending - -A viewer that sends nothing asks for what it wants to receive, so that the offer includes it: - -```java -Peer viewer = rtc.createPeer(PeerOptions.builder() - .receive(MediaKind.VIDEO, MediaKind.AUDIO) - .signaling(signaling) - .build()); -``` - -### Recording - -The media module's `MediaRecorder` records senders and remote tracks into a file as they are, without re-encoding: - -```java -try (MediaRecorder recorder = new MediaRecorder(Paths.get("call.mkv"))) { - recorder.addTrack(videoSender); - recorder.addTrack(remoteVideo); - recorder.start(); - // ... - recorder.stop(); -} -``` - -Closing the peer, or removing a sender, during a recording ends that track in the file, and `stop()` still finishes it. The [Media Recording](/guide/media/media-recording) guide covers containers and the recorder's events. - -### Encoded Transforms - -An [encoded transform](/guide/advanced/encoded-transforms) sees every encoded frame between the encoder and the network, or between the network and the decoder, which is where end-to-end encryption goes. The transforms of the peer options apply to every sender and every remote track of the peer: - -```java -Peer peer = rtc.createPeer(PeerOptions.builder() - .signaling(signaling) - .sendTransform(frame -> encrypt(frame)) - .receiveTransform(frame -> decrypt(frame)) - .build()); -``` - -The peer sets the send transform on each sender as its track is added, before negotiation, so no frame leaves without it and the encoder needs no restart. It sets the receive transform on each receiver before `onTrack` hands out the track, so the transform is in place before the first frame arrives. If installing the receive transform fails, the peer stops that track's transceiver and reports the failure through `onError`; the track is neither listed in `remoteTracks()` nor delivered through `onTrack`. If the receive path cannot be stopped, the peer closes. The builders of [WHIP and WHEP](#whip-and-whep) take them too, `Whip`'s as `sendTransform(...)`, `Whep`'s as `receiveTransform(...)`. - -For end-to-end encryption, `e2ee(keys)` takes the place of both transforms: every sender gets an encryptor and every remote track a decryptor of the [key provider](/guide/advanced/end-to-end-encryption), which also takes care of key rotation, replay protection, frames that fail and key frame requests: - -```java -PeerOptions.builder() - .signaling(signaling) - .e2ee(keys) - .build(); -``` - -A single sender or remote track takes a transform of its own, in place of the peer's, and `null` removes it: - -```java -videoSender.transform(frame -> addMetadata(frame)); -remoteTrack.transform(null); // frames reach the decoder unchanged -``` - -The first transform set on a video sender that already sends restarts its encoder, which costs a key frame. A transform set on a remote track misses the frames that arrive before its future completes, so a transform that every frame needs, such as decryption, belongs in the peer options. A recording of a sender holds its frames before the transform, a recording of a remote track after it. - -A transform that changes its key needs a key frame to start over with, since the frames before depend on the old key: - -```java -videoSender.generateKeyFrame(); // the local encoder makes the next frame a key frame -remoteTrack.asVideo().requestKeyFrame(); // asks the remote sender for one -``` - -`generateKeyFrame()` does nothing for an audio sender, and nothing before the peer is negotiated, as the stream starts with a key frame anyway. `requestKeyFrame()` fails once the track ended. - -Transforms run on a thread of each sender and receiver, not on the peer's event thread. A frame a transform throws on is dropped rather than passed on, so a failed encryption never sends a frame in the clear. The first such failure of each sender or receiver goes to the peer's `onError` handlers. The [`EncryptedRecordingExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java) encrypts a call with `e2ee(keys)` and records it. - -## Data Channels - -```java -DataChannel chat = peer.createChannel("chat"); - -chat.onMessage(message -> System.out.println("Peer says: " + message.text())); -chat.send("hello"); // sent as soon as the channel opens - -peer.onChannel(channel -> { // channels the remote peer opens - if (channel.label().equals("files")) { - channel.onMessage(message -> output.write(message.bytes())); - } -}); -``` - -`Message.toString()` reports only the message type and byte length. Access content -explicitly with `text()` or `bytes()`. For diagnostics, `message.preview(64)` inspects -at most 64 payload bytes and escapes control characters. Binary previews use hex. -Use `message.preview(64, prefix -> redact(prefix))` to apply your own masking before -escaping; the masking function must handle secrets truncated at the prefix boundary. -The preview marks truncated content and also limits the masking function's output. - -- A channel may be used right away: messages sent before it opens are held until it does. -- Messages that arrive while a channel has no `onMessage` handler are kept for the next one, so none are lost while a handler is being set up. -- `send` takes a `String`, a `byte[]` or a `ByteBuffer`, and copies the data before it returns. -- A `Message` holds a copy of the data. `isText()` tells text from binary, `text()` decodes it as UTF-8, and `bytes()` returns a copy. - -The future `send` returns completes when the message is handed to the transport. While more than 1 MiB waits to be sent, further messages are held in Java, and their futures complete only as the backlog shrinks. A loop that waits for each send therefore slows down to what the network carries, instead of filling native memory: - -```java -for (byte[] chunk : chunks) { - channel.send(chunk).get(); // from a thread that is not an event thread -} -``` - -A sender that does not wait still cannot fill Java memory: each channel holds at most 16 MiB waiting to be sent, and a `send` beyond that fails at once with a `ChannelFullException`. `PeerOptions.builder().channelSendLimit(bytes)` changes the limit for every channel of a peer. `bufferedAmount()` tells how much a channel holds. - -Received messages are bounded as well. WebRTC cannot be told to stop receiving, so messages that arrive faster than the handlers take them, or while a channel has no handler, wait in Java. Each channel lets at most 16 MiB of them wait. When more arrive, the channel closes instead of dropping messages unnoticed: the peer's `onError` handlers hear why, the messages after the one over the limit are not delivered, and both sides see the channel close. `PeerOptions.builder().channelReceiveLimit(bytes)` changes the limit. A handler that takes long should hand the work to a thread of its own. - -`ChannelOptions` set how a channel delivers: - -```java -DataChannel positions = peer.createChannel("positions", ChannelOptions.builder() - .ordered(false) - .maxRetransmits(0) - .build()); -``` - -`maxRetransmits(int)` accepts 0 to 65535 retransmissions. `maxPacketLifeTime(Duration)` accepts nonnegative durations up to 65535 milliseconds; positive fractions of a millisecond are rounded down, so a duration below one millisecond becomes zero. The two limits exclude each other. Values outside these ranges are rejected before native processing. - -`negotiatedId(int)` creates a channel both peers set up themselves with the same id, without announcing it; such a channel never shows up in `onChannel`. The [Data Channels](/guide/data/data-channels) guide explains the options in detail. - -A completed `send` future means the data was handed to the transport, not that the remote application processed it. To send files, use `FileTransfers` from `dev.onvoid.webrtc.client.transfer`, which adds the receiver's own confirmations, a SHA-256 checksum, progress, cancelling and resume. See the [File Transfer](/guide/data/file-transfer) guide. - -## WHIP and WHEP - -Media servers and streaming services take streams with the WebRTC HTTP Ingestion Protocol ([WHIP](https://www.rfc-editor.org/rfc/rfc9725)) and hand them out with its egress counterpart, WHEP. Both exchange one offer and one answer over HTTP. `Whip` publishes tracks, and `Whep` receives a stream: - -```java -Whip.Session live = Whip.to(URI.create("https://example.com/live/whip")) - .bearerToken(streamKey) - .track(rtc.media().camera()) - .track(rtc.media().microphone()) - .publish(rtc) - .get(15, TimeUnit.SECONDS); - -Whep.Session viewer = Whep.from(URI.create("https://example.com/live/whep")) - .play(rtc) - .get(15, TimeUnit.SECONDS); - -viewer.peer().onTrack(track -> System.out.println("Receiving " + track.kind())); -``` - -By default, the offer goes to the endpoint at once, before any candidate is gathered, and the future completes once the peer is connected. As RFC 9725 asks, the peer bundles all media on one transport (`max-bundle`), the descriptions it sends mark each m-line `rtcp-mux-only`, and a WHIP offer only sends (`sendonly`). If the endpoint refuses the offer, or the peer does not connect within the timeout (15 seconds by default), the future fails, and the session is deleted on the server again. An endpoint that is busy, with status 503, or a WHEP endpoint whose stream has yet to start, with status 409, gets the offer again after the wait its `Retry-After` names, at least a second, as long as that fits within the timeout; without `Retry-After` the session fails at once. A WHEP endpoint may also reject the offer and counter it with one of its own, with status 406: `Whep` then takes its offer back, answers the endpoint's, receiving only, and sends the answer by `PATCH` to the session the endpoint named, before the counter-offer's `valid-until`, 30 seconds by default. The answer rejects the audio or video of the counter-offer that `receive(...)` leaves out, and the session fails if the counter-offer holds none of what it asks for. Data channels of a counter-offer are accepted. Cancelling the future, say once `get` gave up waiting, ends the session the same way. `onTrack` hands over tracks that already arrived, so a handler added once the future completed misses none. - -With the default `Gathering.TRICKLE`, only once the endpoint answered does the peer gather its candidates, with the STUN and TURN servers the endpoint names in `Link` headers with `rel="ice-server"`, unless `iceServer(...)` or `turnServer(...)` names servers of its own, which take precedence. The candidates follow the offer by `PATCH` to the session, as `application/trickle-ice-sdpfrag`, a few at a time and with the session's `ETag` as `If-Match`, as RFC 9725 describes. An endpoint that takes no such `PATCH`, with status 405, 415, 422 or 501, gets no more; it may still learn peer-reflexive candidates from incoming connectivity checks, but that does not cover every network topology. A 412 or 428 for the current ICE generation reaches the error handler and stops candidate PATCHes; responses to a generation replaced by a restart are ignored. - -For an endpoint without Trickle ICE, use `.gathering(Gathering.COMPLETE)` to include gathered candidates in the initial POST. It waits at most three seconds for gathering, within the overall session timeout, then sends the candidates available so far; any later candidates may still follow by PATCH. Configure STUN/TURN servers locally in this mode, since the endpoint's Link headers arrive after gathering. For example: - -```java -Whep.Session viewer = Whep.from(endpoint) - .turnServer("turn:turn.example.com:3478", turnUsername, turnPassword) - .gathering(Gathering.COMPLETE) - .play(rtc) - .get(15, TimeUnit.SECONDS); -``` - -Both builders support these methods. `turnServer` is separate from `iceServer(String...)`, so passing three URLs to `iceServer` still means three URLs. The gathering setting controls the initial POST only; ICE restarts continue to use PATCH and trickled candidates. - -By default, the offer holds every codec WebRTC has, in WebRTC's order, and the endpoint picks; one that follows the offer's order picks VP8 for video. Endpoints that pass the stream on unchanged, to HLS or a recording, often take only H.264 and Opus. `videoCodecs(...)` and `audioCodecs(...)` offer only the codecs named, in that order of preference, by name (`H264`, `VP8`, `VP9`, `AV1`, `opus`, `G722`, ...) or MIME type (`video/H264`), whatever the case. Retransmission, redundancy, FEC, comfort noise and DTMF go along. For WHEP, they decide what the endpoint sends, also in the answer to a counter-offer. A name WebRTC has no codec of fails the session with a `ConfigurationException` before the offer goes out. - -Some endpoints, Amazon IVS among them, refuse a WHIP offer without video, even for audio alone. `alwaysOfferVideo()` then adds video that only sends, in the stream of the tracks, without a track, so that nothing is sent on it; with a video track, it changes nothing: - -```java -Whip.Session live = Whip.to(URI.create("https://example.com/live/whip")) - .bearerToken(token) - .videoCodecs("H264") - .audioCodecs("opus") - .track(rtc.media().microphone()) - .alwaysOfferVideo() - .publish(rtc) - .get(15, TimeUnit.SECONDS); -``` - -`simulcast()` sends the video track of a WHIP session in three layers, as simulcast (RFC 8853), in full, half and quarter resolution, with the rids `f`, `h` and `q`. The offer negotiates them, as RFC 9725 asks, and an endpoint that forwards the stream can then give each viewer the layer its bandwidth takes. `simulcast(Encoding...)` takes two or three layers of its own, the most WebRTC sends, from the largest down, with their resolution factor, bitrate and frame rate. An endpoint that takes no simulcast leaves it out of its answer and gets the first layer only, which is why the largest comes first. Simulcast needs a video track; `Sender.configure` on its sender changes every layer alike. - -```java -Whip.Session live = Whip.to(URI.create("https://example.com/live/whip")) - .bearerToken(token) - .track(rtc.media().camera()) - .simulcast( - Encoding.builder().maxBitrate(2_500_000).build(), - Encoding.builder().scaleDownBy(2).maxBitrate(800_000).build(), - Encoding.builder().scaleDownBy(4).maxBitrate(200_000).build()) - .publish(rtc) - .get(15, TimeUnit.SECONDS); -``` - -A normal SDP answer must have status 201, a nonempty session `Location`, and Content-Type `application/sdp` (parameters are allowed). Invalid responses fail the session; a resource already named by the server is deleted even when its SDP response cannot be used. The message of a failure an endpoint answered with an error names the status, and what the endpoint says of it: the title and detail of problem details (RFC 9457, `application/problem+json`), as in `status 403: Forbidden: The stream key has expired`, or else the start of the body. The failure is an `HttpStatusException`, whose `status()`, `title()`, `detail()` and `problemType()` give the same to a program; for a deletion or for candidates, it is the cause of the failure reported. As RFC 9725 asks, read no specific cause into a status beyond its class, since endpoints use them differently: - -```java -try { - session = Whip.to(url).bearerToken(key).track(camera).publish(rtc).get(15, TimeUnit.SECONDS); -} -catch (ExecutionException e) { - if (e.getCause() instanceof HttpStatusException) { - HttpStatusException refused = (HttpStatusException) e.getCause(); - System.err.println("Refused, " + refused.status() + ": " + refused.detail()); - } -} -``` - -When the connection fails, or stays disconnected for two seconds, the session restarts ICE: the peer gathers new candidates, and only the new ICE username fragment and password go to the session, by `PATCH` with `If-Match: *`, while the media go on as negotiated. The session's answer, with status 200 and Content-Type `application/trickle-ice-sdpfrag`, brings its own ICE information and a new `ETag`. Its candidate set replaces the old one completely, including when it is empty. Three restarts in a row that do not bring the connection back, or an endpoint that takes no restarts, end the attempts, which the error handler hears of. `session.restartIce()` restarts ICE at once, say after the network changed; a restart that fails leaves the ICE session before as it was. Restarts also work after a WHEP counter-offer; the player keeps its answerer role and updates only the ICE information. - -The timeout bounds the requests as a whole, not each read, so a server that sends its answer a byte at a time cannot stretch it. A request still out when the session times out or is cancelled is cut off at once; one still waiting for the head of its answer only from Java 16 on, and before that at its timeout. Once the server has named the session, in the head of its answer, it is deleted all the same; a session the server created for an answer that had not come yet is left for the server to time out. - -Failures and reports name the URL of an endpoint or session by its scheme, host and port only, since the rest may be a secret: services take stream keys and signatures in the query, and some, such as Cloudflare Stream, in the path. A failure can go to a log as it is, apart from what the endpoint itself says in an error, which it passes on unchanged. - -`session.close()` deletes the session on the server, which frees what the server holds for it, and closes the peer. Both start at once, so a slow server does not keep the peer open, and the deletion takes at most the session's timeout. The tracks a WHIP session published stay open. `session.peer()` is a regular peer, for stats and anything else. A deletion the server refuses or that fails goes to the runtime's error handler, and also fails the future of `closeAsync()`, which completes once both are done; a peer that did not close fails it too. - -`session.extensions()` gives the protocol extensions the endpoint offers for the session (RFC 9725, 4.9): the URL of each by its relation type, such as `urn:ietf:params:whip:ext:example:server-sent-events` (`urn:ietf:params:whep:ext:...` for WHEP), as the endpoint named them in `Link` header fields of its answer, resolved against it. Of a relation type named twice, the first counts. Links of other relation types are ignored, as WHIP and WHEP ask, apart from `ice-server`. The session uses none of the extensions itself; an application uses those it knows, and the map is empty if the endpoint named none. - -`Whep.from(url).detect(rtc)` asks a URL with `HEAD` whether it is a WHEP endpoint, as a player may before it plays a URL it was given (draft-ietf-wish-whep-05, 4.2). Its future holds `true` when the endpoint answers 200 with the Content-Type `application/sdp` and a `Link` to the profile `urn:ietf:params:whep`; the Content-Type alone does not count. It holds `false` for any other answer, such as that of an endpoint that takes no `HEAD`, and fails when the request does. The request carries the credentials of the builder, takes its timeout at most, and follows the redirects the offer follows: - -```java -if (Whep.from(url).detect(rtc).get(15, TimeUnit.SECONDS)) { - Whep.Session viewer = Whep.from(url).play(rtc).get(15, TimeUnit.SECONDS); -} -``` - -Endpoints have to be `https` URLs, as WHIP and WHEP ask, except on the machine the application runs on: there plain `http` will do, for a media server beside it, at `localhost`, an address in 127.0.0.0/8 or `::1`. A name that merely resolves to one of them does not count, since it may resolve elsewhere later. The same holds for the session an endpoint names and for where a redirect points. - -The credentials of a session, its bearer token or its Basic user and password, go to the endpoint, and along with the deletion to the session the endpoint names in its `Location` header. That session may be on another host, but an `https` endpoint cannot name a plain `http` one. The offer follows redirects with status 307 and 308, as WHIP and WHEP ask of clients, such as those of a load balancer: up to five of them, all within the timeout, and the credentials go along. A redirect from `https` to plain `http` fails the session, and so does one with status 301, 302 or 303, which would turn the offer into a `GET`. The deletion follows no redirect; one fails it with its status. - -Servers that ask for a user and password rather than a token, such as MediaMTX, take them by HTTP Basic authentication (RFC 7617): `basicAuth("user", "password")` sends `Authorization: Basic` with both, in UTF-8, on every request of the session, without waiting to be asked. The user must not contain a colon. Of `bearerToken` and `basicAuth`, the one called last counts. MediaMTX also takes them as a token, the way OBS Studio sends them, `bearerToken("user:password")`. Other schemes, such as Digest, are not supported. - -Requests go through Java's `HttpClient`, with the default `SSLContext` and the HTTP proxy Java is configured with, the first the default `ProxySelector` names: `-Dhttps.proxyHost` and `-Dhttps.proxyPort`, `-Dhttp.proxyHost`, or `-Djava.net.useSystemProxies=true`. `HttpClient` takes no SOCKS proxies. A proxy that asks for credentials gets Basic ones from the default `Authenticator`, but only for sessions without credentials of their own: `HttpClient` drops the Authorization header of a request once it answers for credentials itself. As in Java, Basic is not used for the tunnel of an `https` request unless `jdk.http.auth.tunneling.disabledSchemes` allows it. Other schemes, such as Digest, NTLM and Negotiate, are not supported. - -| Builder method | What it sets | -|---|---| -| `bearerToken(String)` | The token sent as `Authorization: Bearer`; for many services, the stream key. | -| `basicAuth(String, String)` | A user and password sent as `Authorization: Basic`, as MediaMTX takes them; replaces a bearer token, and the other way round. | -| `iceServer(String...)` | A STUN or TURN server without credentials, overriding endpoint-provided servers. | -| `turnServer(String, String, String)` | A TURN URL, username and password, overriding endpoint-provided servers. | -| `gathering(Gathering)` | When the initial offer is sent: `TRICKLE` by default, or `COMPLETE` to gather first. | -| `track(LocalTrack)` | A track to publish, WHIP only. The tracks share one stream, which holds one audio and one video track at most. | -| `receive(MediaKind...)` | What to receive, WHEP only; video and audio by default. | -| `videoCodecs(String...)`, `audioCodecs(String...)` | The only codecs to negotiate, in order of preference; every codec WebRTC has by default. | -| `alwaysOfferVideo()` | Video in the offer even without a video track, WHIP only, for endpoints that refuse an offer without video. | -| `simulcast()`, `simulcast(Encoding...)` | The video track in layers, as simulcast, WHIP only: three by default (`f`, `h`, `q`), or two or three given, the largest first. | -| `sendTransform(RTCEncodedFrameTransformer)` | The [encoded transform](#encoded-transforms) of every frame sent, WHIP only, in place before the offer goes out. | -| `e2ee(KeyProvider)` | [End-to-end encryption](/guide/advanced/end-to-end-encryption) of every frame sent (WHIP) or received (WHEP), in place of the transforms. | -| `receiveTransform(RTCEncodedFrameTransformer)` | The [encoded transform](#encoded-transforms) of every frame received, WHEP only, in place before the first frame, also for a counter-offer. | -| `timeout(Duration)` | How long the session may take to be connected, and its deletion to be answered. | - -## Manual Negotiation - -SFUs and other servers dictate an offer and answer exchange of their own. A peer built with `Negotiation.MANUAL` leaves it to the application, through `peer.negotiator()`; `Whip` and `Whep` are built on it. Each step creates and applies a description in one go, and `Gathering.COMPLETE` waits until the description holds all local candidates, for servers that take no trickled candidates. A STUN or TURN server that cannot be reached, as on networks that block UDP, holds up the end of gathering for a long time, so `COMPLETE` waits at most for the peer's gathering timeout and then goes with the candidates it has. This WHEP client does by hand what `Whep` does: - -```java -Peer viewer = rtc.createPeer(PeerOptions.builder() - .negotiation(Negotiation.MANUAL) - .receive(MediaKind.VIDEO, MediaKind.AUDIO) - .build()); - -viewer.onTrack(track -> System.out.println("Receiving " + track.kind())); - -viewer.negotiator().createOffer(Gathering.COMPLETE) - .thenApplyAsync(offer -> http.postSdp(whepUrl, offer.sdp), ioPool) // blocking HTTP off the event thread - .thenCompose(sdp -> viewer.negotiator().applyAnswer(new RTCSessionDescription(RTCSdpType.ANSWER, sdp))) - .thenCompose(applied -> viewer.whenConnected()) - .get(10, TimeUnit.SECONDS); -``` - -| Method | What it does | -|---|---| -| `createOffer()`, `createOffer(Gathering)` | Creates the local offer and applies it. | -| `answer(offer)`, `answer(offer, Gathering)` | Applies the remote offer, then creates the answer and applies it. | -| `applyAnswer(answer)` | Applies the remote answer to the local offer. | -| `addCandidate(candidate)` | Adds a remote candidate, or holds it until there is a remote description to add it to. The future completes once the candidate is added or held; a held candidate that WebRTC rejects later reaches the error handler. | - -A manual peer with a `Signaling` still sends its local candidates through it. `peer.negotiator()` throws an `IllegalStateException` on a peer that negotiates automatically. - -## Statistics - -`peer.stats()` returns the numbers people look for first, taken from the full [RTC stats](/guide/monitoring/rtc-stats) report: - -```java -peer.onStats(Duration.ofSeconds(1), stats -> System.out.printf( - "%s candidate, rtt %s, %d kbit/s out, %d kbit/s in, %.1f%% lost%n", - stats.candidateType(), stats.roundTripTime(), - stats.outgoingBitrate() / 1000, stats.incomingBitrate() / 1000, - stats.recentPacketLoss() * 100)); -``` - -| Method | Value | -|---|---| -| `transport()` | The transport in use, whose candidate type and round trip time the sample reports. Null while no transport has a candidate pair in use. | -| `transports()` | Every transport of the peer, ordered by id, each with its id, the id of its candidate pair in use, candidate type, round trip time and bytes. | -| `candidateType()` | The type of the local candidate in use on `transport()`: `host`, `srflx`, `prflx` or `relay`. Null while not connected. | -| `roundTripTime()` | The current round trip time on `transport()`, or null while not measured. | -| `bytesSent()`, `bytesReceived()` | The bytes sent and received on all transports so far. | -| `timestamp()` | When WebRTC measured the statistics. | -| `outgoingBitrate()`, `incomingBitrate()` | Bits per second between the measurement times of the previous sample and this one, or -1 when there is none to compare with: for a peer's first sample, after the clock went back, or after the byte counter started again. A report WebRTC hands out again within its short cache lifetime repeats the previous bitrates. | -| `packetLoss()` | The share of received packets lost so far, 0 to 1. After a long session it barely moves. | -| `recentPacketLoss()` | The share of packets lost of those expected since the previous sample, 0 to 1, and 0 if none were expected; -1 for the first sample and after the clock went back. A stream that came up or whose counters started again counts all it has. | -| `outboundVideo()`, `inboundVideo()` | Codec, resolution and frame rate of each video stream, and what limits a sent one. | -| `inboundAudio()`, `outboundAudio()` | Audio streams ordered by statistics id: codec, jitter, concealment counters, audio level and sample counters. Unavailable fields are null; received PCM callbacks are observed only while an `onAudio` sink is attached. | -| `diagnostics()` | The current state and confirmed `QualityCondition` causes. | -| `diagnosticReport()` | A JSON snapshot with allowlisted diagnostics, media measurements, codec names, OS/architecture, Java version, library version and processor count. | -| `report()` | The whole `RTCStatsReport`, including potentially sensitive connection information. | - -A peer bundles its media on one transport by default. When the remote side refuses to bundle, each m-line has a transport of its own. `transport()` is then the one with a candidate pair in use that carried the most bytes, sent and received, and of transports that carried as many, the one with the lowest id. Read `transports()` to watch each connection by itself. - -### Diagnosis and quality events - -```java -Subscription diagnosis = peer.onDiagnostics(Duration.ofSeconds(1), status -> { - System.out.println(status.state()); - status.conditions().forEach(cause -> System.out.println(cause.reason())); -}); - -// Export on demand, for example from a support button. File I/O belongs on -// an application worker, outside the peer's event executor. -peer.stats().thenAccept(stats -> supportExecutor.execute(() -> { - try { - Files.write(Paths.get("diagnostics.json"), - stats.diagnosticReport().getBytes(StandardCharsets.UTF_8)); - } - catch (IOException error) { - error.printStackTrace(); - } -})); -``` - -The export example uses `java.nio.file.Files`, `java.nio.file.Paths`, -`java.nio.charset.StandardCharsets` and `java.io.IOException`; `supportExecutor` -is an application-owned executor. - -`onDiagnostics` emits the initial diagnosis and then only changes to its state or -causes, including recovery. Handlers run on the peer's event executor; closing the -subscription stops sampling. All `stats`, `onStats` and `onDiagnostics` calls share -one history per peer. A captured `PeerStats` and its diagnosis stay immutable. - -There is no combined score. `OBSERVING` means there is no continuous measurement -interval yet (including after a clock rollback or a gap longer than five seconds). -`HEALTHY` means no supported problem has been confirmed; it is not a guarantee -that all media paths work. `DEGRADED` includes the following confirmed causes: - -| Condition | Evidence | -|---|---| -| `VIDEO_CPU_LIMITED` | The same outgoing video RTP stream consistently reports `qualityLimitationReason=cpu`. | -| `HIGH_PACKET_LOSS` | Recent inbound packet loss is at least 5%; recovery needs at most 2%. Values between those thresholds retain the current state and restart the confirmation window. | -| `AUDIO_PACKETS_WITHOUT_PCM` | The same received audio RTP stream gains packets while no PCM callbacks reach its attached `onAudio` sinks. This observes sink ingress, before application queues and handlers; it does not diagnose speakers or slow application handlers. | - -Each problem and its recovery need at least three distinct measurements spanning -at least three seconds. Polling once a second is recommended; cached reports do -not advance the window. A missing stream or required measurement, a changed sink -set, or a reset audio counter discards that stream's evidence instead of guessing. -Sampling gaps longer than five seconds and backwards timestamps discard temporal -evidence. Diagnostics do not add audio sinks: without an application subscription, -PCM availability is unknown. Silence (audio level zero) alone is not a fault. - -`TURN_IN_USE` is immediate information when either selected candidate on any -transport is a relay. It does not make a peer degraded. `TransportStats.relayed()` -also exposes this fact, while `candidateType()` continues to describe the local -candidate. - -`AudioStats.jitter()` is a `Duration`, `audioLevel()` is linear (0 to 1), and -`concealedSamples()`, `concealmentEvents()` and `totalSamplesReceived()` are -cumulative counters. Outbound level is read from the media source when available; -receive-only measurements on outbound audio remain null. `pcmCallbacks()` counts -sink callbacks across subscriptions, so multiple subscriptions can count the same -audio more than once. - -The JSON export includes the confirmation policy and cause explanations. It uses -schema version 1 and an explicit allowlist, including a -bounded set of codec MIME names; unknown names become `other`. It excludes raw -statistics, track/stream/transport/candidate identifiers, addresses, ports, URLs, -ICE credentials, SDP, certificates and codec parameters. Missing measurements -become JSON null. The raw `report()` and its string representation are not -sanitized diagnostic exports. - -## Checking Before a Call - -Diagnostics watch a call that is under way. `checkCall` answers the question before it: am I ready -for the call? It checks that the camera delivers frames, that the microphone delivers audio that is -neither silent nor clipped, and that a connection to a test peer comes up, all at the same time: - -```java -LocalCamera camera = rtc.media().camera(); -LocalAudio microphone = rtc.media().microphone(); - -CallCheck check = CallCheck.builder() - .video(camera) - .audio(microphone) // ask the user to say something meanwhile - .network(PeerOptions.builder() - .iceServer("turn:turn.example.com", "user", "secret") - .relayOnly(true) // only a relayed connection counts - .signaling(testPeerSignaling) - .build()) - .build(); - -rtc.checkCall(check).thenAccept(result -> { - if (result.isReady()) { - System.out.println("Ready: " + result); - } - result.problems().forEach(problem -> System.out.println(problem.reason())); -}); -``` - -Every part is optional, but a check needs at least one. Each waits up to the check's `timeout` -(10 seconds by default) for the first frame, the first audio or the connection, then measures for -its `duration` (3 seconds by default). The future completes on an event thread of the runtime's -local media, so the rule against `join()` holds there. The tracks stay the application's: the check -neither closes nor mutes them, and the same tracks can go into the call once it is done. - -| Problem | Found when | -|---|---| -| `NO_VIDEO` | The video track delivered no frame within the timeout. | -| `DARK_VIDEO` | No frame had an average luma above 8% of white: a covered lens, a closed privacy shutter or no light. The brightest frame counts, since a camera's first frames are often dark while its exposure settles. A muted track sends black frames, so it is dark too. | -| `NO_AUDIO` | The audio track delivered no audio within the timeout. | -| `SILENT_AUDIO` | The audio never rose above -50 dBFS: a muted microphone, or nobody spoke. | -| `CLIPPED_AUDIO` | More than 0.1% of the samples were at full scale (99% of it or above): the input is too loud and distorts. | -| `NO_CONNECTION` | No connection to the test peer came up within the timeout. `network().failure()` tells why: a `TimeoutException`, or the failure of the peer. | - -Besides the problems, the result has the measurements behind them: frames, frame rate, resolution and -brightness of the video; how much audio arrived, with its peak and RMS level and the share of -clipped samples; and for the network the time it took to connect, the round trip time, the local -candidate type and whether the connection is relayed. `isReady()` means that no part found a -problem, not that the call is sure to work. - -Audio is measured as the track sends it, after audio processing. WebRTC records a microphone only -while it sends it, so for a microphone the check sends the track to a peer of its own, in the same -runtime, which drops every encoded frame before it goes out: nothing is played back, and the user -does not hear themselves. That peer, and the one of the network check, show among `rtc.peers()` -while the check runs and are closed before the result goes out. - -Whether the network lets a call through cannot be told from the local configuration, only from a -connection that is made. The network part therefore connects to a real test peer: one the -application runs for this, such as an echo bot behind its own signaling, or any other peer that -answers the offer. The check creates a peer with the given options, opens a data channel so that it -offers, reads the connection's statistics once it is up and closes the peer. With `relayOnly(true)` -the connection can only go through the TURN servers of the options, so a connection proves that they -work; without it, a direct connection may come up instead. - ## Closing You close what you created, and closing an owner closes what it owns: @@ -1116,7 +113,7 @@ You close what you created, and closing an owner closes what it owns: `WebRtc.close()` is the one close that waits, for up to the close timeout, which makes it fit for try-with-resources. Once it returns, every peer's `state()` is `CLOSED`; the `whenClosed` futures and `CLOSED` events follow on the peers' event threads, since waiting for them could never end when `close()` is called on an event thread. -When a peer or track does not close within the close timeout, or a track or its source could not be released, the runtime leaves its factory undisposed rather than free what WebRTC's threads may still use. A track's `closeAsync()` fails in that case too. `WebRtc.closeAsync()` then fails with a `WebRtcException` naming the cause, and the error handler hears of it as well, which is how `close()` reports it. +A close that fails is described under [Details](#a-close-that-fails). ## Errors @@ -1161,8 +158,18 @@ The function passed to `withNative` runs after the operations called before it a ## Example -The [`ClientApiExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java) connects two peers in one process through a signaling that carries JSON text. It sends pushed video, audio and chat messages from one to the other, and prints statistics once a second. It needs no audio or video hardware: +The [`ClientApiExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java) connects two peers in one process through a signaling that carries JSON text. It sends pushed video, audio and chat messages from one to the other, and prints statistics once a second. It needs no audio or video hardware: ```bash mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.ClientApiExample" -D"exec.args=10" ``` + +## Details + +### Timeouts + +Every timeout of the client API, the operation and close timeouts as well as the [gathering timeout](/guide/client/peers#manual-negotiation) and the timeouts of [`Whip` and `Whep`](/guide/client/whip-whep), must be positive; none of them can be switched off. A timeout counts in milliseconds, rounded up, so one under a millisecond waits a millisecond. One too long for milliseconds to count waits as long as they can count, about 292 million years. + +### A Close That Fails + +When a peer or track does not close within the close timeout, or a track or its source could not be released, the runtime leaves its factory undisposed rather than free what WebRTC's threads may still use. A track's `closeAsync()` fails in that case too. `WebRtc.closeAsync()` then fails with a `WebRtcException` naming the cause, and the error handler hears of it as well, which is how `close()` reports it. diff --git a/docs/guide/client/data-channels.md b/docs/guide/client/data-channels.md new file mode 100644 index 000000000..fb51a7e88 --- /dev/null +++ b/docs/guide/client/data-channels.md @@ -0,0 +1,71 @@ +# Data Channels + +A data channel carries text and binary messages between two peers. `peer.createChannel` opens one, and `peer.onChannel` hands over those the remote peer opens: + +```java +DataChannel chat = peer.createChannel("chat"); + +chat.onMessage(message -> System.out.println("Peer says: " + message.text())); +chat.send("hello"); // sent as soon as the channel opens + +peer.onChannel(channel -> { // channels the remote peer opens + if (channel.label().equals("files")) { + channel.onMessage(message -> output.write(message.bytes())); + } +}); +``` + +- A channel may be used right away: messages sent before it opens are held until it does. +- Messages that arrive while a channel has no `onMessage` handler are kept for the next one, so none are lost while a handler is being set up. +- `send` takes a `String`, a `byte[]` or a `ByteBuffer`, and copies the data before it returns. +- A `Message` holds a copy of the data. `isText()` tells text from binary, `text()` decodes it as UTF-8, and `bytes()` returns a copy. + +What a message shows in logs is described under [Details](#message-previews). + +## Flow Control + +The future `send` returns completes when the message is handed to the transport. While more than 1 MiB waits to be sent, further messages are held in Java, and their futures complete only as the backlog shrinks. A loop that waits for each send therefore slows down to what the network carries, instead of filling native memory: + +```java +for (byte[] chunk : chunks) { + channel.send(chunk).get(); // from a thread that is not an event thread +} +``` + +A sender that does not wait still cannot fill Java memory: each channel holds at most 16 MiB waiting to be sent, and a `send` beyond that fails at once with a `ChannelFullException`. `PeerOptions.builder().channelSendLimit(bytes)` changes the limit for every channel of a peer. `bufferedAmount()` tells how much a channel holds. + +Received messages are bounded as well. WebRTC cannot be told to stop receiving, so messages that arrive faster than the handlers take them, or while a channel has no handler, wait in Java. Each channel lets at most 16 MiB of them wait. When more arrive, the channel closes instead of dropping messages unnoticed: the peer's `onError` handlers hear why, the messages after the one over the limit are not delivered, and both sides see the channel close. `PeerOptions.builder().channelReceiveLimit(bytes)` changes the limit. A handler that takes long should hand the work to a thread of its own. + +## Channel Options + +`ChannelOptions` set how a channel delivers: + +```java +DataChannel positions = peer.createChannel("positions", ChannelOptions.builder() + .ordered(false) + .maxRetransmits(0) + .build()); +``` + +`negotiatedId(int)` creates a channel both peers set up themselves with the same id, without announcing it; such a channel never shows up in `onChannel`. The [Data Channels](/guide/data/data-channels) guide explains the options in detail. + +The ranges the options accept are listed under [Details](#option-ranges). + +## Sending Files + +A completed `send` future means the data was handed to the transport, not that the remote application processed it. To send files, use `FileTransfers` from `dev.onvoid.webrtc.client.transfer`, which adds the receiver's own confirmations, a SHA-256 checksum, progress, cancelling and resume. See the [File Transfer](/guide/data/file-transfer) guide. + +## Details + +### Message Previews + +`Message.toString()` reports only the message type and byte length. Access content +explicitly with `text()` or `bytes()`. For diagnostics, `message.preview(64)` inspects +at most 64 payload bytes and escapes control characters. Binary previews use hex. +Use `message.preview(64, prefix -> redact(prefix))` to apply your own masking before +escaping; the masking function must handle secrets truncated at the prefix boundary. +The preview marks truncated content and also limits the masking function's output. + +### Option Ranges + +`maxRetransmits(int)` accepts 0 to 65535 retransmissions. `maxPacketLifeTime(Duration)` accepts nonnegative durations up to 65535 milliseconds; positive fractions of a millisecond are rounded down, so a duration below one millisecond becomes zero. The two limits exclude each other. Values outside these ranges are rejected before native processing. diff --git a/docs/guide/client/peers.md b/docs/guide/client/peers.md new file mode 100644 index 000000000..c22b12e85 --- /dev/null +++ b/docs/guide/client/peers.md @@ -0,0 +1,239 @@ +# Peers and Signaling + +`rtc.createPeer(options)` creates a connection to one remote peer. It is the one call of the client API that waits, briefly, for the connection to be created, so that a configuration WebRTC refuses fails right there with a `ConfigurationException`. + +```java +Peer peer = rtc.createPeer(PeerOptions.builder() + .name("alice") + .iceServer("stun:stun.l.google.com:19302") + .iceServer("turn:turn.example.com:3478", "user", "secret") + .polite(true) + .signaling(message -> socket.send(message.toJson())) + .build()); +``` + +| Method | What it sets | +|---|---| +| `name(String)` | The name in thread names and error messages. | +| `iceServer(String...)` | A STUN or TURN server without credentials. | +| `iceServer(String, String, String)` | A TURN server with a user name and password. | +| `relayOnly(boolean)` | Connects through TURN servers only, which hides this side's addresses. | +| `signaling(Signaling)` | What carries messages to the remote peer. Required unless negotiation is manual. | +| `polite(boolean)` | The role in perfect negotiation, see below. | +| `negotiation(Negotiation)` | `AUTOMATIC` by default, or `MANUAL`, see [Manual Negotiation](#manual-negotiation). | +| `receive(MediaKind...)` | Receives audio or video without sending any, as a viewer does. | +| `gatheringTimeout(Duration)` | How long `Gathering.COMPLETE` waits for ICE gathering, 3 seconds by default. See [Manual Negotiation](#manual-negotiation). | +| `channelSendLimit(long)` | How many bytes each data channel may hold waiting to be sent, 16 MiB by default. See [Data Channels](/guide/client/data-channels). | +| `channelReceiveLimit(long)` | How many bytes of received messages each data channel may hold that its handlers are not done with, 16 MiB by default. See [Data Channels](/guide/client/data-channels). | +| `configure(Consumer)` | Anything else the [peer connection configuration](/guide/networking/peer-connection-config) has. | + +## Signaling + +The library has no transport of its own. A `Signaling` sends a peer's messages to the remote peer, and messages from the remote peer are handed to `peer.signal(...)`, in the order they were sent: + +```java +// Outgoing: called on the peer's event thread, in order. +Signaling signaling = message -> socket.send(message.toJson()); + +// Incoming: from any thread. +socket.onText(text -> peer.signal(SignalMessage.fromJson(text))); +``` + +A transport class can implement both directions by overriding `Signaling.attach(Consumer)`, which the peer calls once when it is created, with the consumer to push received messages into. It returns a `Subscription` that removes that consumer, and only that one: the peer closes it when it closes, after its last `send`, on its event thread. The transport itself stays open, so a WebSocket can outlive the peer and take the next one, or serve several peers at once. A transport that keeps the consumer must drop it there, or the closed peer and the runtime with it stay reachable: + +```java +@Override +public synchronized Subscription attach(Consumer inbound) { + this.inbound = inbound; + + return () -> { + synchronized (this) { + // A peer closed after the next one attached leaves that one in place. + if (this.inbound == inbound) { + this.inbound = null; + } + } + }; +} +``` + +Each end of `Signaling.loopback()` takes one peer at a time; once that peer has closed, the next one can attach, and receives what was sent to the end in between. + +A `SignalMessage` is a session description (`OFFER`, `ANSWER`) or an ICE candidate (`CANDIDATE`). `toJson()` writes the shapes the browser API uses, so a JavaScript peer can pass them to `setRemoteDescription` and `addIceCandidate` as they are: + +```json +{"type":"offer","sdp":"v=0\r\no=- 4611731400430051336 2 IN IP4 127.0.0.1\r\n..."} +{"candidate":"candidate:842163049 1 udp 1677729535 203.0.113.7 49203 typ srflx raddr 0.0.0.0 rport 0","sdpMid":"0","sdpMLineIndex":0} +``` + +`SignalMessage.fromJson(String)` reads these, what a browser's `JSON.stringify(pc.localDescription)` and `JSON.stringify(candidate)` write, and the `{"description": ...}` and `{"candidate": {...}}` envelopes of the perfect negotiation examples. What it rejects is listed under [Details](#parsing-signal-messages). Routing, such as room names or peer ids, belongs in an envelope of your own around this JSON. + +A browser peer that only answers needs no more than this: + +```js +socket.onmessage = async ({ data }) => { + const message = JSON.parse(data); + + if (message.type) { + await pc.setRemoteDescription(message); + + if (message.type === "offer") { + await pc.setLocalDescription(); + socket.send(JSON.stringify(pc.localDescription)); + } + } + else if (message.candidate) { + await pc.addIceCandidate(message); + } +}; + +pc.onicecandidate = ({ candidate }) => candidate && socket.send(JSON.stringify(candidate)); +``` + +A browser peer that adds tracks of its own sends offers too, and then has to implement perfect negotiation as well, as the [W3C example](https://w3c.github.io/webrtc-pc/#perfect-negotiation-example) does. + +For tests and demos, `Signaling.loopback()` links two peers in the same process: + +```java +SignalingPair link = Signaling.loopback(); + +Peer caller = rtc.createPeer(PeerOptions.builder().signaling(link.first()).build()); +Peer callee = rtc.createPeer(PeerOptions.builder().signaling(link.second()).polite(true).build()); +``` + +## Polite and Impolite Peers + +With automatic negotiation, a peer sends an offer whenever its tracks or channels change. When both peers send one at the same moment, the **polite** peer drops its own offer and answers the other one, and the impolite peer ignores the offer it got. Exactly one of two peers must be polite. Peers are impolite by default, so set `polite(true)` on one side, for example on the peer that joined second. + +When two impolite peers collide, neither gives way. An impolite peer that waited 5 seconds for an answer reports a `NegotiationException` that says to make one of them polite. + +## Peer State + +`peer.state()` returns the last state the connection reported: + +| State | Meaning | +|---|---| +| `NEW` | Created, nothing negotiated yet. | +| `CONNECTING` | ICE and DTLS are establishing the connection. | +| `CONNECTED` | Media and data can flow. | +| `DISCONNECTED` | Connectivity was lost. It may come back by itself. | +| `FAILED` | The connection failed. `peer.restartIce()` or the configured recovery policy tries to recover it. | +| `CLOSED` | The peer was closed. This state is final. | + +```java +peer.onStateChange(state -> statusLabel.setText(state.toString())); + +peer.whenConnected().thenAccept(p -> System.out.println("Connected")); +peer.whenClosed().thenRun(() -> System.out.println("Closed")); +``` + +`onStateChange` calls a new handler with the current state first, so a handler added late still learns where the peer stands. `whenConnected()` fails if the connection fails or closes first. + +## Automatic ICE Recovery + +Automatic peers can opt into a `RecoveryPolicy`. Without one, the application still +calls `peer.restartIce()` itself. The policy waits through short disconnections, +restarts ICE on the same peer, and bounds retries with exponential backoff: + +```java +RecoveryPolicy recovery = RecoveryPolicy.builder() + .grace(Duration.ofSeconds(2)) + .maxAttempts(3) + .backoff(Duration.ofSeconds(1), Duration.ofSeconds(30)) + .attemptTimeout(Duration.ofSeconds(15)) + .onEvent(event -> System.out.println(event.state() + " attempt " + event.attempt())) + .build(); + +Peer peer = rtc.createPeer(PeerOptions.builder() + .signaling(signaling) + .recovery(recovery) + .build()); +``` + +`DISCONNECTED` starts the grace period; `FAILED` skips it. Repeated failure +notifications during an attempt do not start parallel restarts. Events are +`WAITING`, `ATTEMPTING`, `RETRYING`, `RECOVERED` and `EXHAUSTED`, delivered on the +peer's event executor. `attempt()` counts attempts in the current outage, starting +at one; a recovery during the grace period reports zero. `cause()` gives the last +failure on retry or exhaustion. Successful signaling alone does not count as +recovery: the peer must report `CONNECTED`. That resets the budget, including +after exhaustion. Exhaustion leaves the peer open and creates no further automatic +attempts until it connects again. `whenConnected()` keeps its existing first-connection +semantics; observe `RECOVERED` for later recovery. + +Before an attempt, `iceServers(attempt -> completionStage)` can obtain fresh TURN +credentials asynchronously. It runs on the event executor and must return promptly; +the resulting list replaces **all** ICE servers, so include STUN servers that should +remain available. The library copies the supplied servers; failed, null or invalid +results fail the attempt. For example, with an application's asynchronous credential +service returning `CompletionStage>`: + +```java +RecoveryPolicy recovery = RecoveryPolicy.builder() + .iceServers(attempt -> credentials.fetchIceServers()) + .onEvent(event -> { + if (event.state() == RecoveryEvent.State.EXHAUSTED) { + showReconnectAction(event.cause()); + } + }) + .build(); +``` + +ICE recovery requires working signaling. Reconnecting a WebSocket or refreshing +HTTP authorization is the application's separate responsibility. A policy does not +create a replacement peer or session. If the application chooses that fallback, it +must close the old session, attach handlers to the new peer, republish local tracks, +and handle new remote tracks and data channels explicitly; their old handles are +not transferred. + +`Whip.to(endpoint).recovery(recovery)` and `Whep.from(endpoint).recovery(recovery)` +use the same policy with the session's ICE `PATCH` exchange. This replaces their +built-in restart rule for that session. Keep using `session.restartIce()` for a +manual HTTP restart; `session.peer().restartIce()` does not perform the HTTP +exchange. For a manually negotiated peer outside WHIP/WHEP, the application owns +recovery and its protocol-specific offer/answer exchange. + +How attempts are timed and cancelled is described under [Details](#ice-recovery-timing). + +## Manual Negotiation + +SFUs and other servers dictate an offer and answer exchange of their own. A peer built with `Negotiation.MANUAL` leaves it to the application, through `peer.negotiator()`; `Whip` and `Whep` are built on it. Each step creates and applies a description in one go, and `Gathering.COMPLETE` waits until the description holds all local candidates, for servers that take no trickled candidates. A STUN or TURN server that cannot be reached, as on networks that block UDP, holds up the end of gathering for a long time, so `COMPLETE` waits at most for the peer's gathering timeout and then goes with the candidates it has. This WHEP client does by hand what `Whep` does: + +```java +Peer viewer = rtc.createPeer(PeerOptions.builder() + .negotiation(Negotiation.MANUAL) + .receive(MediaKind.VIDEO, MediaKind.AUDIO) + .build()); + +viewer.onTrack(track -> System.out.println("Receiving " + track.kind())); + +viewer.negotiator().createOffer(Gathering.COMPLETE) + .thenApplyAsync(offer -> http.postSdp(whepUrl, offer.sdp), ioPool) // blocking HTTP off the event thread + .thenCompose(sdp -> viewer.negotiator().applyAnswer(new RTCSessionDescription(RTCSdpType.ANSWER, sdp))) + .thenCompose(applied -> viewer.whenConnected()) + .get(10, TimeUnit.SECONDS); +``` + +| Method | What it does | +|---|---| +| `createOffer()`, `createOffer(Gathering)` | Creates the local offer and applies it. | +| `answer(offer)`, `answer(offer, Gathering)` | Applies the remote offer, then creates the answer and applies it. | +| `applyAnswer(answer)` | Applies the remote answer to the local offer. | +| `addCandidate(candidate)` | Adds a remote candidate, or holds it until there is a remote description to add it to. The future completes once the candidate is added or held; a held candidate that WebRTC rejects later reaches the error handler. | + +A manual peer with a `Signaling` still sends its local candidates through it. `peer.negotiator()` throws an `IllegalStateException` on a peer that negotiates automatically. + +## Details + +### Parsing Signal Messages + +Anything else throws an `IllegalArgumentException` that says what is wrong and where, as does text longer than a MiB or nested deeper than 64 levels, so a message from the remote side cannot exhaust the stack. The JSON must be strict (RFC 8259): no raw line breaks or other control characters inside strings, no member name twice in one object, no numbers like `01`, `1.` or `1e999`. An `sdpMLineIndex` must be an integer from 0 to 65535; `1.9` is rejected rather than truncated to 1. + +### ICE Recovery Timing + +The attempt deadline covers credentials, signaling and waiting for connectivity. +An HTTP restart already in flight finishes or reaches its own operation timeout +before another starts. Late credentials and completions after recovery or `close()` +are ignored; the credential service owns cancellation of its requests. Closing the +peer or runtime cancels scheduled retries, including one whose timer has fired but +whose control task has not run yet. Event notifications already queued may still arrive. diff --git a/docs/guide/client/receiving-media.md b/docs/guide/client/receiving-media.md new file mode 100644 index 000000000..2eceedeb2 --- /dev/null +++ b/docs/guide/client/receiving-media.md @@ -0,0 +1,163 @@ +# Receiving Media + +`peer.onTrack` is called for every track the remote peer sends. Tracks that already arrived are handed to a handler added later. + +```java +peer.onTrack(track -> { + System.out.println("Receiving " + track.kind() + " in " + track.streamIds()); + + if (track.isVideo()) { + track.asVideo().onFrame(frame -> view.render(frame)); + } + else { + track.asAudio().onAudio(chunk -> recognizer.feed(chunk.samples())); + } + + track.onMute(muted -> System.out.println("Muted: " + muted)); + track.onEnded(() -> System.out.println("Track ended")); +}); +``` + +A track ends when the remote peer stops sending it or the peer closes. + +## Video Frames + +Each `onFrame` subscription has a thread of its own, so a slow renderer holds up neither WebRTC's decoder nor the peer's other events. With `frameThreads(n)` the subscriptions share `n` threads and take turns, see [Limiting Threads](/guide/client/threads#limiting-threads). By default the handler always gets the newest frame, and the frames it had no time for are dropped. `FrameDelivery.queue(n)` keeps up to `n` frames instead: + +```java +video.onFrame(FrameDelivery.queue(10), frame -> encoder.encode(frame)); +``` + +A frame is valid while the handler runs. Call `frame.retain()` to keep it longer, and `frame.release()` when done. + +## Showing Video + +Handlers that touch UI components need the toolkit's thread as event executor, see [Handlers on the UI Thread](/guide/client/threads#handlers-on-the-ui-thread). + +The `webrtc-java-swing` module has a `VideoView` in `dev.onvoid.webrtc.client.swing` that shows video in a Swing window. The core `webrtc-java` module does not depend on `java.desktop`, so headless applications and trimmed runtime images need neither AWT nor Swing: + +```xml + + dev.onvoid.webrtc + webrtc-java-swing + {{ VERSION }} + +``` + +The view converts each frame on the frame's own thread and leaves only the painting to the EDT. It keeps the picture's aspect ratio and rotation, and can mirror a local camera preview: + +```java +import dev.onvoid.webrtc.client.swing.VideoView; + +VideoView remoteView = new VideoView(); +VideoView preview = new VideoView(); +preview.setMirrored(true); + +peer.onTrack(track -> { + if (track.isVideo()) { + track.asVideo().onFrame(remoteView); + } +}); +camera.onFrame(preview); + +frame.add(remoteView, BorderLayout.CENTER); +``` + +`snapshot()` returns a copy of the picture shown, for saving a still or for tests. How much memory the views take is described under [Details](#video-views-and-memory). + +For JavaFX, the `webrtc-java-javafx` module has a `VideoView` node in `dev.onvoid.webrtc.client.javafx` that works the same way. It needs Java 11 and the JavaFX your application already uses: + +```xml + + dev.onvoid.webrtc + webrtc-java-javafx + {{ VERSION }} + +``` + +```java +import dev.onvoid.webrtc.client.javafx.VideoView; + +WebRtc rtc = WebRtc.builder() + .eventExecutor(Platform::runLater) // handlers on the JavaFX Application Thread + .build(); + +VideoView remoteView = new VideoView(); +peer.onTrack(track -> { + if (track.isVideo()) { + track.asVideo().onFrame(remoteView); + } +}); + +stage.setScene(new Scene(new StackPane(remoteView), 960, 540)); +``` + +The view converts frames on their own thread and paints at most once per JavaFX pulse. `setMirrored(true)`, or its `mirroredProperty()`, flips a camera preview, and `snapshot()`, on the JavaFX Application Thread, returns a copy of the picture. The `JavaFxCameraExample` in the [examples](/guide/examples#javafx-camera) shows a preview and a received track side by side. + +## Audio + +`onAudio` hands decoded audio to a handler in order, usually 10 ms of 16-bit PCM per `AudioChunk`. Audio reaches these handlers only while the runtime plays out, which a `HEADLESS` runtime always does and a `DEVICES` or `CUSTOM` runtime does to its speakers. `setVolume(double)` changes the playout gain of a track, from 0 to 10, where 1 plays it as received. + +A handler that falls behind gets the audio of the last 5 seconds; the oldest audio beyond that is dropped and counted in `rtc.loadStats().audio().dropped()`. `WebRtc.Builder.audioBuffer(Duration)` changes the buffer of every subscription, and `onAudio(Duration, handler)` that of one. A speech recognizer that must stay live keeps a short buffer; a recorder that can catch up after a stall keeps a long one, at about 190 KiB per second of 48 kHz stereo: + +```java +track.asAudio().onAudio(Duration.ofMillis(200), chunk -> recognizer.feed(chunk.samples())); +``` + +A local audio track tells when this side speaks. `onVoiceActivity` calls its handler with `true` when speech begins and `false` when it ends, on the event executor. The audio is analyzed in native code, and only these changes reach Java: + +```java +LocalAudio microphone = rtc.media().microphone(); +Subscription speaking = microphone.onVoiceActivity( + new VoiceActivityConfig(0.6, Duration.ofMillis(400)), + active -> System.out.println(active ? "speaking" : "quiet")); +``` + +`VoiceActivityConfig` sets the threshold, the voice probability from 0 to 1 at which audio counts as speech (0.5 by default), and the hangover, how long speech goes on once the audio fell below it (300 ms by default), which bridges the pauses between words. Which audio is analyzed is described under [Details](#speech-detection). + +## Receiving Without Sending + +A viewer that sends nothing asks for what it wants to receive, so that the offer includes it: + +```java +Peer viewer = rtc.createPeer(PeerOptions.builder() + .receive(MediaKind.VIDEO, MediaKind.AUDIO) + .signaling(signaling) + .build()); +``` + +## Recording + +The media module's `MediaRecorder` records senders and remote tracks into a file as they are, without re-encoding: + +```java +try (MediaRecorder recorder = new MediaRecorder(Paths.get("call.mkv"))) { + recorder.addTrack(videoSender); + recorder.addTrack(remoteVideo); + recorder.start(); + // ... + recorder.stop(); +} +``` + +Closing the peer, or removing a sender, during a recording ends that track in the file, and `stop()` still finishes it. The [Media Recording](/guide/media/media-recording) guide covers containers and the recorder's events. + +## Details + +### Video Views and Memory + +Both views support resolutions such as 8K without a fixed pixel limit. They reject +nonpositive dimensions and frames whose four-byte pixel data cannot fit in a Java +buffer, using overflow-safe arithmetic. Conversion buffers allocate on demand, +grow geometrically when necessary, and retain their capacity across smaller frames +and `clear()`. Swing reuses one direct buffer; JavaFX reuses up to two. +JavaFX reserves a buffer only while copying it into the displayed image; a waiting +frame may be replaced by a newer frame to avoid building a queue. Buffer +memory is released when the view becomes unreachable. Memory usage follows the +largest frames received and includes displayed images, conversion buffers, and +replacement allocations awaiting garbage collection. Snapshots, input frames and +toolkit rendering resources use additional memory. + +### Speech Detection + +Pushed audio is analyzed as pushed. A microphone's audio is analyzed after audio processing, with echo cancellation, noise suppression and gain control applied as configured, so the remote peer's voice from the speakers does not count as this side speaking. It flows while WebRTC records, which it does while a peer sends a microphone track. Every microphone track of a runtime hears the same, as WebRTC records one microphone for all of them. diff --git a/docs/guide/client/sending-media.md b/docs/guide/client/sending-media.md new file mode 100644 index 000000000..5c322bfb7 --- /dev/null +++ b/docs/guide/client/sending-media.md @@ -0,0 +1,177 @@ +# Sending Media + +`rtc.media()` creates the tracks a runtime sends, and `peer.addTrack(track)` sends one. With automatic negotiation, the peer renegotiates by itself. + +```java +LocalMedia media = rtc.media(); + +LocalCamera camera = media.camera(CameraOptions.builder() + .device(media.devices().cameras().get(0)) + .resolution(1280, 720) + .frameRate(30) + .build()); +LocalAudio microphone = media.microphone(); + +Sender videoSender = peer.addTrack(camera); +Sender audioSender = peer.addTrack(microphone); +``` + +| Method | Track | +|---|---| +| `camera()`, `camera(CameraOptions)` | A camera, as a `LocalCamera` that can switch cameras. The supported capability nearest to the requested resolution and frame rate is used. | +| `microphone()`, `microphone(AudioOptions)` | The microphone, with echo cancellation, noise suppression and gain control unless `AudioOptions` say otherwise. `DEVICES` mode only. | +| `screen(ScreenOptions)` | A screen or a window, as the [Desktop Video](/guide/video/desktop-capture) guide describes. | +| `customVideo(String)` | Frames you push. | +| `customAudio(String)` | Audio you push, in every mode. | +| `videoTrack(String, VideoTrackSource)`, `audioTrack(String, AudioTrackSource)` | A source of the core API, such as one the [media module](/guide/media/media-files) feeds, or a [`SystemAudioSource`](/guide/audio/system-audio-source) for the audio the system plays. The source stays yours. | + +```java +Devices devices = media.devices(); +LocalVideo screen = media.screen(ScreenOptions.screen(devices.screens().get(0)).frameRate(15).build()); +``` + +A content hint tells the senders of a video track what to keep when the bandwidth or the CPU runs short. It can be changed at any time, also while the track is sent, without renegotiating: + +```java +screen.setContentHint(VideoContentHint.TEXT); // source code: keep the resolution +peer.addTrack(screen); + +screen.setContentHint(VideoContentHint.MOTION); // a video starts in the shared window: keep the frame rate +``` + +| Hint | Suits | Gives up first | +|---|---|---| +| `DEFAULT` | Anything | What the source suggests: the frame rate of screens, the resolution of other video | +| `MOTION` | Cameras, videos, animations | Resolution | +| `DETAIL` | Drawings, slides, photos | Frame rate | +| `TEXT` | Source code, terminals, tables | Frame rate; WebRTC treats it as `DETAIL` for now | + +The hint belongs to the track, so it applies on every peer the track is sent on, and a camera keeps it when it switches. Screens keep their resolution by default already, so the hint pays off where the content changes, or for a camera that shows a whiteboard. No hint keeps text readable at any bitrate. + +Tracks added without stream ids share one stream per peer, so that a browser plays their audio and video in sync. Pass ids to `addTrack(track, streamIds...)` to group them yourself. + +One track may be sent on any number of peers. Opening a camera that is already open shares its capture when the options are the same, and fails with a `MediaException` when they differ. + +## Sending a Media File + +The media module plays files and RTSP streams into sources of the core API. `videoTrack` and `audioTrack` put them on peers like any other track: + +```java +WebRtc rtc = WebRtc.builder().audio(AudioMode.HEADLESS).build(); // first: it loads the native library +MediaFileSource file = new MediaFileSource(Paths.get("movie.mp4")); + +peer.addTrack(rtc.media().videoTrack("movie", file.getVideoSource())); +peer.addTrack(rtc.media().audioTrack("movie", file.getAudioSource())); +file.play(); +``` + +Create the runtime before the file: the runtime loads the native library that the file's sources need. The sources belong to the `MediaFileSource`, not to the tracks, so close the runtime, or the tracks, before the file: `track.closeAsync()` returns a future that completes once the track let go of its source. The file's audio source is pushed audio, so the runtime needs the `CUSTOM` or `HEADLESS` audio mode. + +## Pushing Video and Audio + +```java +CustomVideo video = media.customVideo("render"); + +VideoFrame frame = new VideoFrame(buffer, System.nanoTime()); +video.push(frame); +frame.release(); +``` + +`push(frame)` treats the frame as captured now. A source with timing of its own, such as a file, passes a timestamp with `push(frame, timestampMicros)`; see [Custom Video Source](/guide/video/custom-video-source) for why that matters. + +```java +CustomAudio voice = media.customAudio("voice"); + +// 16-bit little-endian PCM, any length, from any thread. +voice.push(pcm, PcmFormat.of(48000, 1)); +``` + +`CustomAudio` takes PCM in pieces of any length and from any thread. It cuts the audio into the 10 ms chunks WebRTC works with and sends one every 10 ms from a thread of its own, so pushing a whole second at once is as fine as pushing every 10 ms. `queuedMillis()` tells how much is still waiting, and `clear()` drops it. + +A producer faster than real time would otherwise fill memory and push the delay up without end, so the queue holds at most 10 seconds, and a `push` beyond that fails with an `AudioQueueFullException`, keeping none of its audio. `queueLimit(max, overflow)` sets another limit and what happens beyond it: + +```java +// Live audio: stay at most 200 ms behind, dropping the oldest audio. +voice.queueLimit(Duration.ofMillis(200), AudioOverflow.DROP_OLDEST); + +// A file, which must arrive whole: refuse the push, to try it again later. +voice.queueLimit(Duration.ofSeconds(2), AudioOverflow.REJECT); +``` + +With `REJECT` a refused push can be pushed again once `queuedMillis()` has shrunk; with `DROP_OLDEST` every push is taken, and the audio sent lags at most the limit behind. + +## Controlling Senders and Tracks + +```java +camera.setEnabled(false); // sends black frames, the camera stays open + +videoSender.configure(Encoding.builder() + .maxBitrate(1_500_000) + .maxFrameRate(30) + .build()); + +videoSender.replaceTrack(screen); // no renegotiation +audioSender.remove(); // renegotiates; the track stays open + +camera.close(); // leaves every peer and stops the camera +``` + +Closing a track stops sending it on every peer, without renegotiating, and then releases the capture. The senders stay, and can be given another track with `replaceTrack`. + +An audio sender also sends DTMF tones, for telephone systems and voice menus. The future completes once the tones have been sent, and `onTone` reports each tone as it starts, then an empty string at the end: + +```java +audioSender.onTone(tone -> System.out.println(tone.isEmpty() ? "Done" : "Tone " + tone)); +audioSender.sendTones("1234#"); // 100 ms each +audioSender.sendTones("*99,1", Duration.ofMillis(200), Duration.ofMillis(100)); +``` + +The characters 0 to 9, A to D, `#` and `*` are tones, and a comma is a pause of 2 seconds. What happens to queued and stopped tones is described under [Details](#dtmf-tones), and the [DTMF Sender](/guide/audio/dtmf-sender) guide has more. + +## Switching Cameras + +A `LocalCamera` switches to another camera without renegotiating. Every peer that sends it keeps sending it, and preview handlers keep getting frames: + +```java +LocalCamera camera = media.camera(); +peer.addTrack(camera); + +camera.switchCamera(media.devices().cameras().get(1)) + .thenRun(() -> System.out.println("Now sending " + camera.device().getName())); +``` + +The new camera keeps the resolution and frame rate of the old one, as far as it supports them. The switch opens the new camera before it closes the old one, so that the peers see no gap. Some devices cannot capture with two cameras at once, such as the two sensors of a laptop camera; then the old camera closes first, and the switch takes about half a second. What happens when a switch fails is described under [Details](#when-a-camera-switch-fails). + +## Devices + +`media.devices()` returns the cameras, microphones and speakers. The first call reads them from the system; after that, the lists are snapshots kept current in the background, so reading them is cheap on any thread. How the lists are read, and how a query fails, is described under [Details](#device-lists). + +```java +Devices devices = media.devices(); + +devices.onChange(changed -> System.out.println("Cameras: " + changed.cameras())); +devices.useMicrophone(devices.microphones().get(1)); +devices.useSpeaker(devices.speakers().get(0)); +``` + +`useMicrophone` and `useSpeaker` switch devices also for tracks that are sent or played now. A device that was disconnected since the list was read fails the returned future, and the device in use stays as it was. On Linux, where WebRTC's audio device module cannot tell devices apart, the default device is used whichever is given. + +A track does not end by itself when its device is disconnected or a captured window closes. `LocalTrack.onEnded` follows only from closing the track, or from a camera switch that can open neither camera. `onChange` tells of a device that went away; closing the track, or switching a camera to another device, is up to you. Nothing tells of a captured window that closes. + +## Details + +### Timing of Pushed Audio + +That thread does nothing else, so a slow push into WebRTC delays only the audio, not the timeouts of the runtime. Should it stall, as in a long garbage collection pause, it makes up for at most the last 50 ms instead of sending the whole pause at once; the queued audio then stays behind by the rest, until `clear()` drops it. + +### DTMF Tones + +Tones still queued from an earlier call are replaced, and that call's future is cancelled. A video sender, or one whose remote peer accepts no telephone events, fails the future with an `IllegalStateException`. So do tones WebRTC stops sending midway, for example because the sender was paused with `configure`, a few seconds after their planned end. + +### When a Camera Switch Fails + +If the new camera cannot be opened, the future fails with a `MediaException`, and the track keeps sending the camera it had. If a peer does not take the new camera within the operation timeout, the future fails with a `WebRtcException`; the track goes back to the old camera, or keeps the new one when the old one had to close first. A track that closed its camera first and can open neither camera ends. + +### Device Lists + +Where the system reports no device changes, as on a Linux machine without PulseAudio, they are read on each call instead. `screens()` and `windows()` ask the system on each call. A query that fails, including an unavailable audio backend, throws a `MediaException`, so an empty list means there are no such devices. Its cause is an `AudioBackendUnavailableException` when the audio backend is missing or unreachable, or a `DeviceException` for other native device failures. diff --git a/docs/guide/client/statistics.md b/docs/guide/client/statistics.md new file mode 100644 index 000000000..ac362b12c --- /dev/null +++ b/docs/guide/client/statistics.md @@ -0,0 +1,180 @@ +# Statistics and Call Checks + +`peer.stats()` returns the numbers people look for first, taken from the full [RTC stats](/guide/monitoring/rtc-stats) report: + +```java +peer.onStats(Duration.ofSeconds(1), stats -> System.out.printf( + "%s candidate, rtt %s, %d kbit/s out, %d kbit/s in, %.1f%% lost%n", + stats.candidateType(), stats.roundTripTime(), + stats.outgoingBitrate() / 1000, stats.incomingBitrate() / 1000, + stats.recentPacketLoss() * 100)); +``` + +| Method | Value | +|---|---| +| `transport()` | The transport in use, whose candidate type and round trip time the sample reports. Null while no transport has a candidate pair in use. | +| `transports()` | Every transport of the peer, ordered by id, each with its id, the id of its candidate pair in use, candidate type, round trip time and bytes. | +| `candidateType()` | The type of the local candidate in use on `transport()`: `host`, `srflx`, `prflx` or `relay`. Null while not connected. | +| `roundTripTime()` | The current round trip time on `transport()`, or null while not measured. | +| `bytesSent()`, `bytesReceived()` | The bytes sent and received on all transports so far. | +| `timestamp()` | When WebRTC measured the statistics. | +| `outgoingBitrate()`, `incomingBitrate()` | Bits per second between the measurement times of the previous sample and this one, or -1 when there is none to compare with: for a peer's first sample, after the clock went back, or after the byte counter started again. A report WebRTC hands out again within its short cache lifetime repeats the previous bitrates. | +| `packetLoss()` | The share of received packets lost so far, 0 to 1. After a long session it barely moves. | +| `recentPacketLoss()` | The share of packets lost of those expected since the previous sample, 0 to 1, and 0 if none were expected; -1 for the first sample and after the clock went back. A stream that came up or whose counters started again counts all it has. | +| `outboundVideo()`, `inboundVideo()` | Codec, resolution and frame rate of each video stream, and what limits a sent one. | +| `inboundAudio()`, `outboundAudio()` | Audio streams ordered by statistics id: codec, jitter, concealment counters, audio level and sample counters. Unavailable fields are null; received PCM callbacks are observed only while an `onAudio` sink is attached. | +| `diagnostics()` | The current state and confirmed `QualityCondition` causes. | +| `diagnosticReport()` | A JSON snapshot with allowlisted diagnostics, media measurements, codec names, OS/architecture, Java version, library version and processor count. | +| `report()` | The whole `RTCStatsReport`, including potentially sensitive connection information. | + +How `transport()` is chosen when media do not share one transport is described under [Details](#separate-transports). + +## Diagnosis and Quality Events + +```java +Subscription diagnosis = peer.onDiagnostics(Duration.ofSeconds(1), status -> { + System.out.println(status.state()); + status.conditions().forEach(cause -> System.out.println(cause.reason())); +}); + +// Export on demand, for example from a support button. File I/O belongs on +// an application worker, outside the peer's event executor. +peer.stats().thenAccept(stats -> supportExecutor.execute(() -> { + try { + Files.write(Paths.get("diagnostics.json"), + stats.diagnosticReport().getBytes(StandardCharsets.UTF_8)); + } + catch (IOException error) { + error.printStackTrace(); + } +})); +``` + +The export example uses `java.nio.file.Files`, `java.nio.file.Paths`, +`java.nio.charset.StandardCharsets` and `java.io.IOException`; `supportExecutor` +is an application-owned executor. + +`onDiagnostics` emits the initial diagnosis and then only changes to its state or +causes, including recovery. Handlers run on the peer's event executor; closing the +subscription stops sampling. All `stats`, `onStats` and `onDiagnostics` calls share +one history per peer. A captured `PeerStats` and its diagnosis stay immutable. + +There is no combined score. `OBSERVING` means there is no continuous measurement +interval yet (including after a clock rollback or a gap longer than five seconds). +`HEALTHY` means no supported problem has been confirmed; it is not a guarantee +that all media paths work. `DEGRADED` includes the following confirmed causes: + +| Condition | Evidence | +|---|---| +| `VIDEO_CPU_LIMITED` | The same outgoing video RTP stream consistently reports `qualityLimitationReason=cpu`. | +| `HIGH_PACKET_LOSS` | Recent inbound packet loss is at least 5%; recovery needs at most 2%. Values between those thresholds retain the current state and restart the confirmation window. | +| `AUDIO_PACKETS_WITHOUT_PCM` | The same received audio RTP stream gains packets while no PCM callbacks reach its attached `onAudio` sinks. This observes sink ingress, before application queues and handlers; it does not diagnose speakers or slow application handlers. | + +`TURN_IN_USE` is immediate information when either selected candidate on any +transport is a relay. It does not make a peer degraded. `TransportStats.relayed()` +also exposes this fact, while `candidateType()` continues to describe the local +candidate. + +How a condition is confirmed, the audio statistics and what the JSON export holds are described under [Details](#confirming-a-condition). + +## Checking Before a Call + +Diagnostics watch a call that is under way. `checkCall` answers the question before it: am I ready +for the call? It checks that the camera delivers frames, that the microphone delivers audio that is +neither silent nor clipped, and that a connection to a test peer comes up, all at the same time: + +```java +LocalCamera camera = rtc.media().camera(); +LocalAudio microphone = rtc.media().microphone(); + +CallCheck check = CallCheck.builder() + .video(camera) + .audio(microphone) // ask the user to say something meanwhile + .network(PeerOptions.builder() + .iceServer("turn:turn.example.com", "user", "secret") + .relayOnly(true) // only a relayed connection counts + .signaling(testPeerSignaling) + .build()) + .build(); + +rtc.checkCall(check).thenAccept(result -> { + if (result.isReady()) { + System.out.println("Ready: " + result); + } + result.problems().forEach(problem -> System.out.println(problem.reason())); +}); +``` + +Every part is optional, but a check needs at least one. Each waits up to the check's `timeout` +(10 seconds by default) for the first frame, the first audio or the connection, then measures for +its `duration` (3 seconds by default). The future completes on an event thread of the runtime's +local media, so the rule against `join()` holds there. The tracks stay the application's: the check +neither closes nor mutes them, and the same tracks can go into the call once it is done. + +| Problem | Found when | +|---|---| +| `NO_VIDEO` | The video track delivered no frame within the timeout. | +| `DARK_VIDEO` | No frame had an average luma above 8% of white: a covered lens, a closed privacy shutter or no light. The brightest frame counts, since a camera's first frames are often dark while its exposure settles. A muted track sends black frames, so it is dark too. | +| `NO_AUDIO` | The audio track delivered no audio within the timeout. | +| `SILENT_AUDIO` | The audio never rose above -50 dBFS: a muted microphone, or nobody spoke. | +| `CLIPPED_AUDIO` | More than 0.1% of the samples were at full scale (99% of it or above): the input is too loud and distorts. | +| `NO_CONNECTION` | No connection to the test peer came up within the timeout. `network().failure()` tells why: a `TimeoutException`, or the failure of the peer. | + +Besides the problems, the result has the measurements behind them: frames, frame rate, resolution and +brightness of the video; how much audio arrived, with its peak and RMS level and the share of +clipped samples; and for the network the time it took to connect, the round trip time, the local +candidate type and whether the connection is relayed. `isReady()` means that no part found a +problem, not that the call is sure to work. + +Whether the network lets a call through cannot be told from the local configuration, only from a +connection that is made. The network part therefore connects to a real test peer: one the +application runs for this, such as an echo bot behind its own signaling, or any other peer that +answers the offer. The check creates a peer with the given options, opens a data channel so that it +offers, reads the connection's statistics once it is up and closes the peer. With `relayOnly(true)` +the connection can only go through the TURN servers of the options, so a connection proves that they +work; without it, a direct connection may come up instead. + +How the microphone is checked without being heard is described under [Details](#how-the-microphone-is-checked). + +## Details + +### Separate Transports + +A peer bundles its media on one transport by default. When the remote side refuses to bundle, each m-line has a transport of its own. `transport()` is then the one with a candidate pair in use that carried the most bytes, sent and received, and of transports that carried as many, the one with the lowest id. Read `transports()` to watch each connection by itself. + +### Confirming a Condition + +Each problem and its recovery need at least three distinct measurements spanning +at least three seconds. Polling once a second is recommended; cached reports do +not advance the window. A missing stream or required measurement, a changed sink +set, or a reset audio counter discards that stream's evidence instead of guessing. +Sampling gaps longer than five seconds and backwards timestamps discard temporal +evidence. Diagnostics do not add audio sinks: without an application subscription, +PCM availability is unknown. Silence (audio level zero) alone is not a fault. + +### Audio Statistics + +`AudioStats.jitter()` is a `Duration`, `audioLevel()` is linear (0 to 1), and +`concealedSamples()`, `concealmentEvents()` and `totalSamplesReceived()` are +cumulative counters. Outbound level is read from the media source when available; +receive-only measurements on outbound audio remain null. `pcmCallbacks()` counts +sink callbacks across subscriptions, so multiple subscriptions can count the same +audio more than once. + +### The JSON Export + +The JSON export includes the confirmation policy and cause explanations. It uses +schema version 1 and an explicit allowlist, including a +bounded set of codec MIME names; unknown names become `other`. It excludes raw +statistics, track/stream/transport/candidate identifiers, addresses, ports, URLs, +ICE credentials, SDP, certificates and codec parameters. Missing measurements +become JSON null. The raw `report()` and its string representation are not +sanitized diagnostic exports. + +### How the Microphone Is Checked + +Audio is measured as the track sends it, after audio processing. WebRTC records a microphone only +while it sends it, so for a microphone the check sends the track to a peer of its own, in the same +runtime, which drops every encoded frame before it goes out: nothing is played back, and the user +does not hear themselves. That peer, and the one of the network check, show among `rtc.peers()` +while the check runs and are closed before the result goes out. diff --git a/docs/guide/client/threads.md b/docs/guide/client/threads.md new file mode 100644 index 000000000..180b02daa --- /dev/null +++ b/docs/guide/client/threads.md @@ -0,0 +1,102 @@ +# Threads + +Every method of the client API may be called from any thread, including from inside a handler. Methods that change a peer return at once and take effect in the order they were called. + +| Code | Runs on | May it block? | +|---|---|---| +| Your calls into the API | your thread | They don't block, with these exceptions. `WebRtc.createPeer`, the `LocalMedia` methods that make a track and `LocalTrack.setEnabled` and `LocalVideo.setContentHint` wait briefly for WebRTC. `camera()` and `screen()` also open their capture on your thread, which can take longer, so a UI thread should leave them to another thread. The first `media.devices()`, `screens()` and `windows()` ask the system on your thread. `WebRtc.close()` waits up to the close timeout. | +| Peer, track and channel handlers, future callbacks, `Signaling.send` | the peer's event thread | Yes, but that delays only the events of this peer. They must not wait for a future of this API, see below. | +| `onFrame` and `onAudio` handlers | a thread of each subscription | Better not. A slow video handler gets the newest frame next, and the frames in between are dropped; a slow audio handler loses the oldest audio beyond its buffer. | +| [Encoded transforms](/guide/client/transforms) | a thread of each sender and receiver | Better not. A slow transform delays only its own frames, and one that falls several seconds behind has frames dropped. | +| `LocalTrack.onEnded` and `Devices.onChange` handlers, callbacks of the futures of local tracks and devices | the event thread of the runtime's local media | Yes, but that delays the other events of local media. They must not wait for a future of this API either. A track is released before its `onEnded` handlers run, so they never hold up a release. | +| WebRTC's own callbacks | WebRTC's threads | Not your code. The library only copies what they report and queues it. | + +The handlers of one peer never run at the same time, and they run in the order of the events. The handlers of different peers may run at the same time. With the default executor, the event thread is named after its peer, such as `webrtc-client-events-alice`, which shows up in thread dumps. + +An executor of your own that rejects work is described under [Details](#executors-that-reject-work). + +## Limiting Threads + +By default the runtime starts a thread for each peer whose events wait and for each subscription whose frames wait, so a thousand peers with slow handlers take a thousand threads. `eventThreads(n)` and `frameThreads(n)` cap them: + +```java +WebRtc rtc = WebRtc.builder() + .eventThreads(8) // handlers of all peers, future callbacks, Signaling.send + .frameThreads(4) // onFrame and onAudio handlers, local previews included + .build(); +``` + +With a limit, the peers take turns, and so do the subscriptions: one that has work runs up to 64 handlers, for at most 10 ms, then goes to the back of the line. A slow handler thus delays the others but never starves them, and a peer's events still run one at a time and in order. The limit rejects nothing: events wait for their turn and are never dropped, so no future stays open, and once the runtime closes, the futures still open fail after the events queued before them. Media that waits is bounded by its subscription instead, by the `FrameDelivery` of a video handler and the audio buffer of an audio handler, which drop the oldest frames and audio when they are full. + +A limit has one cost: a handler that waits for what another peer's handler does, say on a latch, can wait forever once all threads are taken by such waits. Handlers that wait for something outside the API should hand the work to a thread of their own. + +The control threads of the library, which call into WebRTC, are not limited. Their tasks wait for one another, such as a camera switch for every peer to take the new track. + +The HTTP requests of `Whip` and `Whep` sessions take no thread while they wait, so a slow server holds up no peer: they go through Java's `HttpClient`, one of each runtime. + +`rtc.loadStats()` tells how much work waits and runs, at this moment and cheaply enough to be read every second: + +```java +LoadStats load = rtc.loadStats(); + +LoadStats.Work events = load.events(); +metrics.gauge("webrtc.events.queued", events.queued()); // events waiting for a thread +metrics.gauge("webrtc.events.oldest_ms", events.oldestWaiting().toMillis()); // how long the oldest has waited +metrics.gauge("webrtc.events.active", events.active()); // handlers running now +metrics.gauge("webrtc.video.dropped", load.video().dropped()); // frames dropped since start +metrics.gauge("webrtc.audio.dropped", load.audio().dropped()); // audio chunks dropped since start +metrics.gauge("webrtc.event_threads", load.eventThreads()); +``` + +| Part | What waits | What `dropped()` counts | +|---|---|---| +| `control()` | tasks of the library that call into WebRTC | tasks the control pool rejected while running, which it never does | +| `events()` | events of all peers, for handlers, future callbacks and `Signaling.send` | events an `eventExecutor` of your own rejected | +| `video()` | frames for `onFrame` handlers | frames a handler had no time for | +| `audio()` | chunks for `onAudio` handlers | chunks beyond the audio buffer | + +A queue that keeps growing, or an oldest wait that keeps getting longer, tells that handlers do not keep up. A higher limit helps handlers that wait for I/O; handlers that compute only spread the delay over more threads. + +## Handlers on the UI Thread + +Pass the UI toolkit's executor, and every handler may touch UI components directly: + +```java +WebRtc rtc = WebRtc.builder() + .eventExecutor(SwingUtilities::invokeLater) // or Platform::runLater for JavaFX + .build(); + +peer.onStateChange(state -> statusLabel.setText(state.toString())); +hangUpButton.addActionListener(event -> peer.close()); // never blocks the EDT +``` + +The order of each peer's events is kept with any executor, including a thread pool. Frame handlers stay on their own threads even then, since frames come far too often for a UI thread. + +To show video in Swing or JavaFX, see [Showing Video](/guide/client/receiving-media#showing-video). + +## Don't Wait in a Handler + +A future of the client API completes on the event thread of its peer. A handler that waits for one would wait for its own thread, so `join()` and `get()` there throw a `BlockingCallException` instead of hanging. Chain the next step instead: + +```java +peer.onChannel(channel -> { + // channel.whenOpen().join() would throw here. + channel.whenOpen().thenCompose(open -> open.send("hello")); +}); +``` + +A future that is already done may be read anywhere, and any thread that is not an event thread, such as `main`, may wait. + +Each call returns a future of its own, also of `closeAsync()`, `whenClosed()`, `whenConnected()` and `whenOpen()`. Cancelling one only stops waiting: the close goes on, and the futures of other callers, and of the runtime's own close, are not affected. + +Futures derived from them with `thenApply` and the like refuse to be waited on as well. + +::: warning +A UI thread used as event executor does not refuse: waiting on it for a future of this API hangs the UI, since the future completes on it. +::: + +## Details + +### Executors That Reject Work + +An executor of your own that rejects work, such as a bounded pool that is full, loses the events it rejects, and the rejection goes to the error handler. The futures those events would have completed complete all the same, on the thread that found them rejected. Once the runtime is closed, every future of this API that is not done fails with `PeerClosedException`, including those of calls made after the close. diff --git a/docs/guide/client/transforms.md b/docs/guide/client/transforms.md new file mode 100644 index 000000000..39b5f5944 --- /dev/null +++ b/docs/guide/client/transforms.md @@ -0,0 +1,56 @@ +# Transforms and Encryption + +An [encoded transform](/guide/advanced/encoded-transforms) sees every encoded frame between the encoder and the network, or between the network and the decoder, which is where end-to-end encryption goes. The transforms of the peer options apply to every sender and every remote track of the peer: + +```java +Peer peer = rtc.createPeer(PeerOptions.builder() + .signaling(signaling) + .sendTransform(frame -> encrypt(frame)) + .receiveTransform(frame -> decrypt(frame)) + .build()); +``` + +The peer sets the send transform on each sender as its track is added, before negotiation, so no frame leaves without it and the encoder needs no restart. It sets the receive transform on each receiver before `onTrack` hands out the track, so the transform is in place before the first frame arrives. What happens when it cannot be installed is described under [Details](#installing-the-receive-transform). The builders of [WHIP and WHEP](/guide/client/whip-whep) take them too, `Whip`'s as `sendTransform(...)`, `Whep`'s as `receiveTransform(...)`. + +## End-to-End Encryption + +For end-to-end encryption, `e2ee(keys)` takes the place of both transforms: every sender gets an encryptor and every remote track a decryptor of the [key provider](/guide/advanced/end-to-end-encryption), which also takes care of key rotation, replay protection, frames that fail and key frame requests: + +```java +PeerOptions.builder() + .signaling(signaling) + .e2ee(keys) + .build(); +``` + +## One Sender or Track + +A single sender or remote track takes a transform of its own, in place of the peer's, and `null` removes it: + +```java +videoSender.transform(frame -> addMetadata(frame)); +remoteTrack.transform(null); // frames reach the decoder unchanged +``` + +The first transform set on a video sender that already sends restarts its encoder, which costs a key frame. A transform set on a remote track misses the frames that arrive before its future completes, so a transform that every frame needs, such as decryption, belongs in the peer options. A recording of a sender holds its frames before the transform, a recording of a remote track after it. + +## Key Frames + +A transform that changes its key needs a key frame to start over with, since the frames before depend on the old key: + +```java +videoSender.generateKeyFrame(); // the local encoder makes the next frame a key frame +remoteTrack.asVideo().requestKeyFrame(); // asks the remote sender for one +``` + +`generateKeyFrame()` does nothing for an audio sender, and nothing before the peer is negotiated, as the stream starts with a key frame anyway. `requestKeyFrame()` fails once the track ended. + +## Threads and Failures + +Transforms run on a thread of each sender and receiver, not on the peer's event thread. A frame a transform throws on is dropped rather than passed on, so a failed encryption never sends a frame in the clear. The first such failure of each sender or receiver goes to the peer's `onError` handlers. The [`EncryptedRecordingExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java) encrypts a call with `e2ee(keys)` and records it. + +## Details + +### Installing the Receive Transform + +If installing the receive transform fails, the peer stops that track's transceiver and reports the failure through `onError`; the track is neither listed in `remoteTracks()` nor delivered through `onTrack`. If the receive path cannot be stopped, the peer closes. diff --git a/docs/guide/client/whip-whep.md b/docs/guide/client/whip-whep.md new file mode 100644 index 000000000..e74cac199 --- /dev/null +++ b/docs/guide/client/whip-whep.md @@ -0,0 +1,176 @@ +# WHIP and WHEP + +Media servers and streaming services take streams with the WebRTC HTTP Ingestion Protocol ([WHIP](https://www.rfc-editor.org/rfc/rfc9725)) and hand them out with its egress counterpart, WHEP. Both exchange one offer and one answer over HTTP. `Whip` publishes tracks, and `Whep` receives a stream: + +```java +Whip.Session live = Whip.to(URI.create("https://example.com/live/whip")) + .bearerToken(streamKey) + .track(rtc.media().camera()) + .track(rtc.media().microphone()) + .publish(rtc) + .get(15, TimeUnit.SECONDS); + +Whep.Session viewer = Whep.from(URI.create("https://example.com/live/whep")) + .play(rtc) + .get(15, TimeUnit.SECONDS); + +viewer.peer().onTrack(track -> System.out.println("Receiving " + track.kind())); +``` + +## Connecting + +By default, the offer goes to the endpoint at once, before any candidate is gathered, and the future completes once the peer is connected. If the endpoint refuses the offer, or the peer does not connect within the timeout (15 seconds by default), the future fails, and the session is deleted on the server again. Cancelling the future, say once `get` gave up waiting, ends the session the same way. `onTrack` hands over tracks that already arrived, so a handler added once the future completed misses none. Retries and counter-offers are described under [Details](#retries-and-counter-offers). + +## Candidates + +With the default `Gathering.TRICKLE`, only once the endpoint answered does the peer gather its candidates, with the STUN and TURN servers the endpoint names in `Link` headers with `rel="ice-server"`, unless `iceServer(...)` or `turnServer(...)` names servers of its own, which take precedence. The candidates follow the offer by `PATCH` to the session, as `application/trickle-ice-sdpfrag`, a few at a time and with the session's `ETag` as `If-Match`, as RFC 9725 describes. + +For an endpoint without Trickle ICE, use `.gathering(Gathering.COMPLETE)` to include gathered candidates in the initial POST. It waits at most three seconds for gathering, within the overall session timeout, then sends the candidates available so far; any later candidates may still follow by PATCH. Configure STUN/TURN servers locally in this mode, since the endpoint's Link headers arrive after gathering. For example: + +```java +Whep.Session viewer = Whep.from(endpoint) + .turnServer("turn:turn.example.com:3478", turnUsername, turnPassword) + .gathering(Gathering.COMPLETE) + .play(rtc) + .get(15, TimeUnit.SECONDS); +``` + +Both builders support these methods. `turnServer` is separate from `iceServer(String...)`, so passing three URLs to `iceServer` still means three URLs. The gathering setting controls the initial POST only; ICE restarts continue to use PATCH and trickled candidates. + +How the session treats an endpoint that refuses candidates is described under [Details](#trickle-ice-responses). + +## Codecs + +By default, the offer holds every codec WebRTC has, in WebRTC's order, and the endpoint picks; one that follows the offer's order picks VP8 for video. Endpoints that pass the stream on unchanged, to HLS or a recording, often take only H.264 and Opus. `videoCodecs(...)` and `audioCodecs(...)` offer only the codecs named, in that order of preference, by name (`H264`, `VP8`, `VP9`, `AV1`, `opus`, `G722`, ...) or MIME type (`video/H264`), whatever the case. Retransmission, redundancy, FEC, comfort noise and DTMF go along. For WHEP, they decide what the endpoint sends, also in the answer to a counter-offer. A name WebRTC has no codec of fails the session with a `ConfigurationException` before the offer goes out. + +## Endpoints That Require Video + +Some endpoints, Amazon IVS among them, refuse a WHIP offer without video, even for audio alone. `alwaysOfferVideo()` then adds video that only sends, in the stream of the tracks, without a track, so that nothing is sent on it; with a video track, it changes nothing: + +```java +Whip.Session live = Whip.to(URI.create("https://example.com/live/whip")) + .bearerToken(token) + .videoCodecs("H264") + .audioCodecs("opus") + .track(rtc.media().microphone()) + .alwaysOfferVideo() + .publish(rtc) + .get(15, TimeUnit.SECONDS); +``` + +## Simulcast + +`simulcast()` sends the video track of a WHIP session in three layers, as simulcast (RFC 8853), in full, half and quarter resolution, with the rids `f`, `h` and `q`. The offer negotiates them, as RFC 9725 asks, and an endpoint that forwards the stream can then give each viewer the layer its bandwidth takes. `simulcast(Encoding...)` takes two or three layers of its own, the most WebRTC sends, from the largest down, with their resolution factor, bitrate and frame rate. An endpoint that takes no simulcast leaves it out of its answer and gets the first layer only, which is why the largest comes first. Simulcast needs a video track; `Sender.configure` on its sender changes every layer alike. + +```java +Whip.Session live = Whip.to(URI.create("https://example.com/live/whip")) + .bearerToken(token) + .track(rtc.media().camera()) + .simulcast( + Encoding.builder().maxBitrate(2_500_000).build(), + Encoding.builder().scaleDownBy(2).maxBitrate(800_000).build(), + Encoding.builder().scaleDownBy(4).maxBitrate(200_000).build()) + .publish(rtc) + .get(15, TimeUnit.SECONDS); +``` + +## Errors + +If an endpoint answers with an error, the message of the failure names the status, and what the endpoint says of it: the title and detail of problem details (RFC 9457, `application/problem+json`), as in `status 403: Forbidden: The stream key has expired`, or else the start of the body. The failure is an `HttpStatusException`, whose `status()`, `title()`, `detail()` and `problemType()` give the same to a program; for a deletion or for candidates, it is the cause of the failure reported. As RFC 9725 asks, read no specific cause into a status beyond its class, since endpoints use them differently: + +```java +try { + session = Whip.to(url).bearerToken(key).track(camera).publish(rtc).get(15, TimeUnit.SECONDS); +} +catch (ExecutionException e) { + if (e.getCause() instanceof HttpStatusException) { + HttpStatusException refused = (HttpStatusException) e.getCause(); + System.err.println("Refused, " + refused.status() + ": " + refused.detail()); + } +} +``` + +## ICE Restarts + +When the connection fails, or stays disconnected for two seconds, the session restarts ICE: the peer gathers new candidates, and only the new ICE username fragment and password go to the session, by `PATCH` with `If-Match: *`, while the media go on as negotiated. The session's answer, with status 200 and Content-Type `application/trickle-ice-sdpfrag`, brings its own ICE information and a new `ETag`. Its candidate set replaces the old one completely, including when it is empty. Three restarts in a row that do not bring the connection back, or an endpoint that takes no restarts, end the attempts, which the error handler hears of. `session.restartIce()` restarts ICE at once, say after the network changed; a restart that fails leaves the ICE session before as it was. Restarts also work after a WHEP counter-offer; the player keeps its answerer role and updates only the ICE information. + +## Authentication + +Servers that ask for a user and password rather than a token, such as MediaMTX, take them by HTTP Basic authentication (RFC 7617): `basicAuth("user", "password")` sends `Authorization: Basic` with both, in UTF-8, on every request of the session, without waiting to be asked. The user must not contain a colon. Of `bearerToken` and `basicAuth`, the one called last counts. MediaMTX also takes them as a token, the way OBS Studio sends them, `bearerToken("user:password")`. Other schemes, such as Digest, are not supported. + +How requests reach the endpoint, over `https`, redirects and proxies, is described under [Details](#https-and-redirects). + +## Detecting a WHEP Endpoint + +`Whep.from(url).detect(rtc)` asks a URL with `HEAD` whether it is a WHEP endpoint, as a player may before it plays a URL it was given (draft-ietf-wish-whep-05, 4.2). Its future holds `true` when the endpoint answers 200 with the Content-Type `application/sdp` and a `Link` to the profile `urn:ietf:params:whep`; the Content-Type alone does not count. It holds `false` for any other answer, such as that of an endpoint that takes no `HEAD`, and fails when the request does. The request carries the credentials of the builder, takes its timeout at most, and follows the redirects the offer follows: + +```java +if (Whep.from(url).detect(rtc).get(15, TimeUnit.SECONDS)) { + Whep.Session viewer = Whep.from(url).play(rtc).get(15, TimeUnit.SECONDS); +} +``` + +## Closing a Session + +`session.close()` deletes the session on the server, which frees what the server holds for it, and closes the peer. Both start at once, so a slow server does not keep the peer open, and the deletion takes at most the session's timeout. The tracks a WHIP session published stay open. `session.peer()` is a regular peer, for stats and anything else. A deletion the server refuses or that fails goes to the runtime's error handler, and also fails the future of `closeAsync()`, which completes once both are done; a peer that did not close fails it too. + +## Builder Methods + +| Builder method | What it sets | +|---|---| +| `bearerToken(String)` | The token sent as `Authorization: Bearer`; for many services, the stream key. | +| `basicAuth(String, String)` | A user and password sent as `Authorization: Basic`, as MediaMTX takes them; replaces a bearer token, and the other way round. | +| `iceServer(String...)` | A STUN or TURN server without credentials, overriding endpoint-provided servers. | +| `turnServer(String, String, String)` | A TURN URL, username and password, overriding endpoint-provided servers. | +| `gathering(Gathering)` | When the initial offer is sent: `TRICKLE` by default, or `COMPLETE` to gather first. | +| `track(LocalTrack)` | A track to publish, WHIP only. The tracks share one stream, which holds one audio and one video track at most. | +| `receive(MediaKind...)` | What to receive, WHEP only; video and audio by default. | +| `videoCodecs(String...)`, `audioCodecs(String...)` | The only codecs to negotiate, in order of preference; every codec WebRTC has by default. | +| `alwaysOfferVideo()` | Video in the offer even without a video track, WHIP only, for endpoints that refuse an offer without video. | +| `simulcast()`, `simulcast(Encoding...)` | The video track in layers, as simulcast, WHIP only: three by default (`f`, `h`, `q`), or two or three given, the largest first. | +| `sendTransform(RTCEncodedFrameTransformer)` | The [encoded transform](/guide/client/transforms) of every frame sent, WHIP only, in place before the offer goes out. | +| `e2ee(KeyProvider)` | [End-to-end encryption](/guide/advanced/end-to-end-encryption) of every frame sent (WHIP) or received (WHEP), in place of the transforms. | +| `receiveTransform(RTCEncodedFrameTransformer)` | The [encoded transform](/guide/client/transforms) of every frame received, WHEP only, in place before the first frame, also for a counter-offer. | +| `timeout(Duration)` | How long the session may take to be connected, and its deletion to be answered. | + +## Details + +### Offers + +As RFC 9725 asks, the peer bundles all media on one transport (`max-bundle`), the descriptions it sends mark each m-line `rtcp-mux-only`, and a WHIP offer only sends (`sendonly`). + +### Retries and Counter-Offers + +An endpoint that is busy, with status 503, or a WHEP endpoint whose stream has yet to start, with status 409, gets the offer again after the wait its `Retry-After` names, at least a second, as long as that fits within the timeout; without `Retry-After` the session fails at once. + +A WHEP endpoint may also reject the offer and counter it with one of its own, with status 406: `Whep` then takes its offer back, answers the endpoint's, receiving only, and sends the answer by `PATCH` to the session the endpoint named, before the counter-offer's `valid-until`, 30 seconds by default. The answer rejects the audio or video of the counter-offer that `receive(...)` leaves out, and the session fails if the counter-offer holds none of what it asks for. Data channels of a counter-offer are accepted. + +### Trickle ICE Responses + +An endpoint that takes no such `PATCH`, with status 405, 415, 422 or 501, gets no more; it may still learn peer-reflexive candidates from incoming connectivity checks, but that does not cover every network topology. A 412 or 428 for the current ICE generation reaches the error handler and stops candidate PATCHes; responses to a generation replaced by a restart are ignored. + +### Responses + +A normal SDP answer must have status 201, a nonempty session `Location`, and Content-Type `application/sdp` (parameters are allowed). Invalid responses fail the session; a resource already named by the server is deleted even when its SDP response cannot be used. + +### Timeouts + +The timeout bounds the requests as a whole, not each read, so a server that sends its answer a byte at a time cannot stretch it. A request still out when the session times out or is cancelled is cut off at once; one still waiting for the head of its answer only from Java 16 on, and before that at its timeout. Once the server has named the session, in the head of its answer, it is deleted all the same; a session the server created for an answer that had not come yet is left for the server to time out. + +### URLs in Failures + +Failures and reports name the URL of an endpoint or session by its scheme, host and port only, since the rest may be a secret: services take stream keys and signatures in the query, and some, such as Cloudflare Stream, in the path. A failure can go to a log as it is, apart from what the endpoint itself says in an error, which it passes on unchanged. + +### HTTPS and Redirects + +Endpoints have to be `https` URLs, as WHIP and WHEP ask, except on the machine the application runs on: there plain `http` will do, for a media server beside it, at `localhost`, an address in 127.0.0.0/8 or `::1`. A name that merely resolves to one of them does not count, since it may resolve elsewhere later. The same holds for the session an endpoint names and for where a redirect points. + +The credentials of a session, its bearer token or its Basic user and password, go to the endpoint, and along with the deletion to the session the endpoint names in its `Location` header. That session may be on another host, but an `https` endpoint cannot name a plain `http` one. The offer follows redirects with status 307 and 308, as WHIP and WHEP ask of clients, such as those of a load balancer: up to five of them, all within the timeout, and the credentials go along. A redirect from `https` to plain `http` fails the session, and so does one with status 301, 302 or 303, which would turn the offer into a `GET`. The deletion follows no redirect; one fails it with its status. + +### Proxies + +Requests go through Java's `HttpClient`, with the default `SSLContext` and the HTTP proxy Java is configured with, the first the default `ProxySelector` names: `-Dhttps.proxyHost` and `-Dhttps.proxyPort`, `-Dhttp.proxyHost`, or `-Djava.net.useSystemProxies=true`. `HttpClient` takes no SOCKS proxies. A proxy that asks for credentials gets Basic ones from the default `Authenticator`, but only for sessions without credentials of their own: `HttpClient` drops the Authorization header of a request once it answers for credentials itself. As in Java, Basic is not used for the tunnel of an `https` request unless `jdk.http.auth.tunneling.disabledSchemes` allows it. Other schemes, such as Digest, NTLM and Negotiate, are not supported. + +### Protocol Extensions + +`session.extensions()` gives the protocol extensions the endpoint offers for the session (RFC 9725, 4.9): the URL of each by its relation type, such as `urn:ietf:params:whip:ext:example:server-sent-events` (`urn:ietf:params:whep:ext:...` for WHEP), as the endpoint named them in `Link` header fields of its answer, resolved against it. Of a relation type named twice, the first counts. Links of other relation types are ignored, as WHIP and WHEP ask, apart from `ice-server`. The session uses none of the extensions itself; an application uses those it knows, and the map is empty if the endpoint named none. diff --git a/docs/guide/core-api.md b/docs/guide/core-api.md new file mode 100644 index 000000000..ec4cf57c5 --- /dev/null +++ b/docs/guide/core-api.md @@ -0,0 +1,300 @@ +# Core API + +The core API in `dev.onvoid.webrtc` mirrors WebRTC's own classes one to one. It leaves negotiation, threads and the lifetime of native objects to you. The steps below make the call of [Getting Started](/guide/get-started#your-first-call) with it; the [Client API](/guide/client-api) does all of this for you. + +## Initialize the WebRTC Library + +First, create a `PeerConnectionFactory` which is the entry point for most WebRTC operations: + +```java +import dev.onvoid.webrtc.PeerConnectionFactory; + +// Create a peer connection factory +PeerConnectionFactory factory = new PeerConnectionFactory(); +``` + +## Create a Peer Connection + +To establish communication with another peer, you need to create a `RTCPeerConnection`: + +```java +import dev.onvoid.webrtc.RTCConfiguration; +import dev.onvoid.webrtc.RTCIceCandidate; +import dev.onvoid.webrtc.RTCIceServer; +import dev.onvoid.webrtc.RTCPeerConnection; +import dev.onvoid.webrtc.PeerConnectionObserver; + +// Configure ICE servers (STUN/TURN) +RTCConfiguration config = new RTCConfiguration(); +RTCIceServer iceServer = new RTCIceServer(); +iceServer.urls.add("stun:stun.l.google.com:19302"); +config.iceServers.add(iceServer); + +// Create a peer connection with an observer to handle events +RTCPeerConnection peerConnection = factory.createPeerConnection(config, new PeerConnectionObserver() { + @Override + public void onIceCandidate(RTCIceCandidate candidate) { + // Send the candidate to the remote peer, see below. The other + // methods are optional. + } +}); +``` + +## Signaling + +WebRTC requires a signaling mechanism to exchange connection information between peers. The library doesn't provide this, so you'll need to implement it using your preferred method (WebSockets, HTTP, etc.). + +Here's a simplified example of the signaling process: + +```java +// Create an offer +RTCOfferOptions options = new RTCOfferOptions(); + +peerConnection.createOffer(options, new CreateSessionDescriptionObserver() { + @Override + public void onSuccess(RTCSessionDescription description) { + // Set local description + peerConnection.setLocalDescription(description, new SetSessionDescriptionObserver() { + @Override + public void onSuccess() { + // Send the offer to the remote peer via your signaling channel + signalingChannel.send(description); + } + + @Override + public void onFailure(String error) { + System.err.println("Failed to set local description: " + error); + } + }); + } + + @Override + public void onFailure(String error) { + System.err.println("Failed to create offer: " + error); + } +}); + +// When you receive an answer from the remote peer via your signaling channel +signalingChannel.onMessage(message -> { + RTCSessionDescription remoteDescription = parseSessionDescription(message); + peerConnection.setRemoteDescription(remoteDescription, new SetSessionDescriptionObserver() { + @Override + public void onSuccess() { + System.out.println("Remote description set successfully"); + } + + @Override + public void onFailure(String error) { + System.err.println("Failed to set remote description: " + error); + } + }); +}); + +// Handle ICE candidates +PeerConnectionObserver connectionObserver = new PeerConnectionObserver() { + @Override + public void onIceCandidate(RTCIceCandidate candidate) { + // Send the ICE candidate to the remote peer via your signaling channel + signalingChannel.send(candidate); + } +}; + +// When you receive an ICE candidate from the remote peer +signalingChannel.onIceCandidate(candidateMessage -> { + RTCIceCandidate candidate = parseIceCandidate(candidateMessage); + peerConnection.addIceCandidate(candidate); +}); +``` + +### Letting the Peer Connection Create the Description + +`setLocalDescription` can also be called with only an observer. It then creates the offer or the answer, whichever the signaling state calls for, and applies it in one step, so nothing can change in between. That makes it the simpler way to implement "perfect negotiation". Read the result with `getLocalDescription()` once the observer reports success: + +```java +peerConnection.setLocalDescription(new SetSessionDescriptionObserver() { + @Override + public void onSuccess() { + signalingChannel.send(peerConnection.getLocalDescription()); + } + + @Override + public void onFailure(String error) { + System.err.println("Failed to set local description: " + error); + } +}); +``` + +### Adding Candidates and Knowing the Outcome + +`addIceCandidate(candidate)` reports nothing and fails silently when the remote description is not set yet. The form that takes an `AddIceCandidateObserver` is queued behind any pending offer/answer operation, as the standard requires, so a candidate can be added right after `setRemoteDescription()` without waiting for it, and the observer says whether it was accepted: + +```java +peerConnection.addIceCandidate(candidate, new AddIceCandidateObserver() { + @Override + public void onSuccess() { + } + + @Override + public void onFailure(String error) { + // e.g. "[INVALID_STATE] The remote description was null" + System.err.println("Candidate rejected: " + error); + } +}); +``` + +A local candidate can also go away, for example when its network interface does. The observer's `onIceCandidateRemoved(RTCIceCandidate)` reports it; send it to the remote peer, which passes it to `removeIceCandidate(candidate)`. + +Once ICE has picked the candidate pair it uses, `onSelectedCandidatePairChanged(RTCCandidatePairChangeEvent)` reports both candidates of the pair and why it was chosen. It fires again whenever ICE switches pairs: + +```java +@Override +public void onSelectedCandidatePairChanged(RTCCandidatePairChangeEvent event) { + System.out.println("Using " + event.getLocal().sdp + " <-> " + event.getRemote().sdp + + " (" + event.getReason() + ")"); +} +``` + +## Media Streams + +### Accessing Media Devices + +To query media devices (cameras and microphones): + +```java +import dev.onvoid.webrtc.media.MediaDevices; +import dev.onvoid.webrtc.media.video.VideoDevice; +import dev.onvoid.webrtc.media.audio.AudioDevice; + +// Get available video devices +List videoDevices = MediaDevices.getVideoCaptureDevices(); +for (VideoDevice device : videoDevices) { + System.out.println("Video device: " + device.getName()); +} + +// Get available audio devices +List audioDevices = MediaDevices.getAudioCaptureDevices(); +for (AudioDevice device : audioDevices) { + System.out.println("Audio device: " + device.getName()); +} +``` + +### Creating Media Tracks + +To create audio and video tracks: + +```java +import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.video.VideoDeviceSource; +import dev.onvoid.webrtc.media.audio.AudioOptions; +import dev.onvoid.webrtc.media.audio.AudioTrackSource; + +// Create a video source and track +VideoDeviceSource videoSource = new VideoDeviceSource(); +videoSource.setVideoCaptureDevice(videoDevices.get(0)); // Use the first available camera +VideoTrack videoTrack = factory.createVideoTrack("video0", videoSource); + +// Create an audio source and track +AudioOptions audioOptions = new AudioOptions(); +audioOptions.echoCancellation = true; +audioOptions.autoGainControl = true; +audioOptions.noiseSuppression = true; + +// Create an audio source using the default audio device +AudioTrackSource audioSource = factory.createAudioSource(audioOptions); +AudioTrack audioTrack = factory.createAudioTrack("audio0", audioSource); +``` + +::: info +Audio devices can be specified via the `AudioDeviceModule`, see the guide on [Audio Devices][audio-devices] for more details. +::: + +### Adding Tracks to Peer Connection + +```java +import java.util.ArrayList; +import java.util.List; +import dev.onvoid.webrtc.RTCRtpSender; + +// Add tracks to the peer connection +List streamIds = new ArrayList<>(); +streamIds.add("stream1"); +RTCRtpSender videoSender = peerConnection.addTrack(videoTrack, streamIds); +RTCRtpSender audioSender = peerConnection.addTrack(audioTrack, streamIds); +``` + +### Receiving Tracks + +A remote track arrives in `onTrack` with the transceiver that carries it. To learn when its media actually starts flowing, for example to show a video only once there is something to show, set an `RTCRtpReceiverObserver`; an `RTCRtpSenderObserver` does the same for the first packet a sender sends. Either is called right away when that has already happened: + +```java +@Override +public void onTrack(RTCRtpTransceiver transceiver) { + RTCRtpReceiver receiver = transceiver.getReceiver(); + + receiver.setObserver(mediaType -> System.out.println("First " + mediaType + " packet received")); + + MediaStreamTrack track = receiver.getTrack(); + + if (track instanceof AudioTrack) { + // Playout volume of this remote track, 0 to 10; 1 plays it as received. + ((AudioTrack) track).setVolume(0.5); + } +} +``` + +An observer belongs to the `RTCRtpReceiver` or `RTCRtpSender` object it was set through, and is removed when that object is disposed. `getId()` identifies a sender or receiver, and `RTCRtpSender.getStreams()` returns the stream IDs its track was added with. + +## Cleanup + +Always properly dispose of resources when you're done: + +```java +// Dispose of senders (not owned by the peer connection) +videoSender.dispose(); +audioSender.dispose(); + +// Dispose of tracks +videoTrack.dispose(); +audioTrack.dispose(); + +// Dispose of sources +videoSource.dispose(); +audioSource.dispose(); + +// Close peer connection and release resources +peerConnection.close(); + +// Dispose of factory +factory.dispose(); +``` + +A track may be disposed, and a listener removed, from within one of its listeners: the listeners the notification has not reached yet are then not called, and what goes is freed once the notification is over. An observer registered on one thread while the object is disposed on another is freed once and never left registered. A factory disposed while a closed connection is still being freed is released once that connection is. + +`dispose()` and `close()` may be called from any thread and more than once: only the first call frees anything, later ones do nothing. Once an object is disposed, its methods throw an `IllegalStateException` (a closed connection still reports its state). A method that another thread is running at that moment keeps the native object until it returns, and what the dispose frees is freed then. + +Apart from the media sources, which only have `dispose()`, objects with a `dispose()` method also have a `close()` that does the same. A try-with-resources statement therefore releases them, also when an exception leaves the block: + +```java +try (AudioConverter converter = new AudioConverter(48000, 1, 16000, 1)) { + converter.convert(input, output); +} +``` + +`RTCDataChannel` is the exception: its `close()` closes the channel, which then still has to be disposed. + +### Finding Objects That Were Never Disposed + +An object that becomes unreachable without being disposed or closed leaks its native object. To find such objects, start the JVM with `-Dwebrtc.java.leak.detection=true`: each one is then reported, with its class, once the garbage collector finds it unreachable. Add `-Dwebrtc.java.leak.trace=true` to have the report tell where the object was created, at the cost of a stack trace per object. + +```shell +java -Dwebrtc.java.leak.detection=true -Dwebrtc.java.leak.trace=true -jar app.jar +``` + +The reports are warnings through `Logging`, which writes nothing unless the application asks for it: + +```java +Logging.logToDebug(Logging.Severity.WARNING); +``` + +Only objects that own a native object and have a `dispose()` or `close()` method are watched: not views such as the track of `RTCRtpSender.getTrack()`, which own nothing, and not reference-counted objects such as video frame buffers. The detector only reports; it never frees anything itself, since a native object may still be in use on a WebRTC thread. While it is off, which is the default, nothing is watched. diff --git a/docs/guide/data/data-channels.md b/docs/guide/data/data-channels.md index ca01903c6..7b5e966be 100644 --- a/docs/guide/data/data-channels.md +++ b/docs/guide/data/data-channels.md @@ -1,5 +1,7 @@ # Data Channels + + This guide explains how to use WebRTC data channels with the webrtc-java library. Data channels provide a bidirectional communication mechanism that can be used to send arbitrary data between peers. ## Overview diff --git a/docs/guide/examples.md b/docs/guide/examples.md index c304b5f0a..82452ea70 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -4,7 +4,7 @@ This section provides an overview of the example applications included in the `w ## Client API -The [`ClientApiExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java) connects two peers in one process with the [Client API](/guide/client-api), and sends video, audio and chat messages from one to the other. It needs no audio or video hardware and no server. +The [`ClientApiExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java) connects two peers in one process with the [Client API](/guide/client-api), and sends video, audio and chat messages from one to the other. It needs no audio or video hardware and no server. **Key features demonstrated:** - Creating a headless `WebRtc` runtime, which needs no microphone or speakers @@ -24,7 +24,7 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.ClientApiExample" -D" ## Native Image -The [`NativeImageExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/NativeImageExample.java) is made to be compiled into a [GraalVM native image](/guide/advanced/native-image), a single executable that needs no installed Java runtime. It connects two peers in one process and checks itself: it exits with status 0 only if everything arrived. +The [`NativeImageExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/NativeImageExample.java) is made to be compiled into a [GraalVM native image](/guide/advanced/native-image), a single executable that needs no installed Java runtime. It connects two peers in one process and checks itself: it exits with status 0 only if everything arrived. **Key features demonstrated:** - Loading the native library that the image carries inside itself @@ -41,7 +41,7 @@ webrtc-examples/target/webrtc-native-image-example ## File Transfer -The [`FileTransferExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/FileTransferExample.java) connects two peers in one process with the [Client API](/guide/client-api), and sends a file from one to the other with [File Transfer](/guide/data/file-transfer). It needs no audio or video hardware and no server. +The [`FileTransferExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/FileTransferExample.java) connects two peers in one process with the [Client API](/guide/client-api), and sends a file from one to the other with [File Transfer](/guide/data/file-transfer). It needs no audio or video hardware and no server. **Key features demonstrated:** - Running `FileTransfers` over a data channel on both sides @@ -57,7 +57,7 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.FileTransferExample" ## PeerConnection -The [`PeerConnectionExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/PeerConnectionExample.java) sends the camera and the microphone from one peer to another in the same process, with the [Client API](/guide/client-api), and logs the video frames and audio that arrive. +The [`PeerConnectionExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/PeerConnectionExample.java) sends the camera and the microphone from one peer to another in the same process, with the [Client API](/guide/client-api), and logs the video frames and audio that arrive. **Key features demonstrated:** - Creating a `WebRtc` runtime that captures from and plays to the audio devices @@ -75,7 +75,7 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.PeerConnectionExample ## WHEP -The [`WhepExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java) implements a client of the WebRTC HTTP Egress Protocol (WHEP), which receives a stream from a server, with the [Client API](/guide/client-api). +The [`WhepExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/WhepExample.java) implements a client of the WebRTC HTTP Egress Protocol (WHEP), which receives a stream from a server, with the [Client API](/guide/client-api). **Key features demonstrated:** - Receiving a stream from a WHEP endpoint with `Whep` @@ -90,7 +90,7 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.WhepExample" -D"exec. ## Codec List -The [`CodecListExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/CodecListExample.java) demonstrates how to list all supported codecs with the WebRTC peer-connection-factory. +The [`CodecListExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/CodecListExample.java) demonstrates how to list all supported codecs with the WebRTC peer-connection-factory. **Key features demonstrated:** - Creating a `PeerConnectionFactory` with the `HardwareVideoEncoderFactory` and `HardwareVideoDecoderFactory` @@ -102,7 +102,7 @@ This example is useful for understanding what codecs are available on the curren ## Desktop Video -The [`DesktopVideoExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopVideoExample.java) shares a screen or a window from one peer to another in the same process, with the [Client API](/guide/client-api). +The [`DesktopVideoExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/DesktopVideoExample.java) shares a screen or a window from one peer to another in the same process, with the [Client API](/guide/client-api). **Key features demonstrated:** - Listing the screens and windows that can be captured @@ -119,7 +119,7 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.DesktopVideoExample" ## System Audio -The [`SystemAudioExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/SystemAudioExample.java) captures the audio the system plays on its default output with a [`SystemAudioSource`](/guide/audio/system-audio-source), and counts the frames that reach a sink of its track, without sending or playing anything. +The [`SystemAudioExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/SystemAudioExample.java) captures the audio the system plays on its default output with a [`SystemAudioSource`](/guide/audio/system-audio-source), and counts the frames that reach a sink of its track, without sending or playing anything. **Key features demonstrated:** - Capturing the system output as a local audio track @@ -134,7 +134,7 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.SystemAudioExample" ## Screen Share with Audio -The [`ScreenShareAudioExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ScreenShareAudioExample.java) shares a screen together with the microphone and the system audio, from one runtime and over one peer, with the [Client API](/guide/client-api). +The [`ScreenShareAudioExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ScreenShareAudioExample.java) shares a screen together with the microphone and the system audio, from one runtime and over one peer, with the [Client API](/guide/client-api). **Key features demonstrated:** - Sending a screen, the microphone and the system output as three tracks of one stream @@ -150,7 +150,7 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.ScreenShareAudioExamp ## Media File -The [`MediaFileExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/MediaFileExample.java) demonstrates how to send a media file over a peer connection, in place of a camera and a microphone. See the [Media Files](/guide/media/media-files) guide for the API it uses. +The [`MediaFileExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/MediaFileExample.java) demonstrates how to send a media file over a peer connection, in place of a camera and a microphone. See the [Media Files](/guide/media/media-files) guide for the API it uses. **Key features demonstrated:** - Opening a media file, or a live RTSP stream, with a `MediaFileSource` @@ -173,7 +173,7 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.MediaFileExample" -D" ## Media File Player -The [`MediaFilePlayerExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/MediaFilePlayerExample.java) is a Swing application built on the [Client API](/guide/client-api). It sends a media file, or a live RTSP stream, from one peer to another in the same process, then shows the received video and plays the received audio in sync, and records what is received. Start and Stop buttons set up and tear down the whole session, and Record starts and stops a recording. +The [`MediaFilePlayerExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/MediaFilePlayerExample.java) is a Swing application built on the [Client API](/guide/client-api). It sends a media file, or a live RTSP stream, from one peer to another in the same process, then shows the received video and plays the received audio in sync, and records what is received. Start and Stop buttons set up and tear down the whole session, and Record starts and stops a recording. **Key features demonstrated:** - Sending the sources of a `MediaFileSource` as client API tracks, with `videoTrack` and `audioTrack` @@ -197,7 +197,7 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.MediaFilePlayerExampl ## JavaFX Camera -The [`JavaFxCameraExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/JavaFxCameraExample.java) is a JavaFX application built on the [Client API](/guide/client-api). It sends the camera from one peer to another in the same process, and shows the camera's preview beside the video that arrives. +The [`JavaFxCameraExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/JavaFxCameraExample.java) is a JavaFX application built on the [Client API](/guide/client-api). It sends the camera from one peer to another in the same process, and shows the camera's preview beside the video that arrives. **Key features demonstrated:** - Running the peers' handlers on the JavaFX Application Thread, with `Platform::runLater` as the event executor @@ -213,7 +213,7 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.JavaFxCameraExample" ## Encrypted Recording -The [`EncryptedRecordingExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java) encrypts a call end to end with a key provider of [End-to-End Encryption](/guide/advanced/end-to-end-encryption), with the [Client API](/guide/client-api#encoded-transforms), and records what the receiving side decrypted into a media file. See the [Encoded Transforms](/guide/advanced/encoded-transforms) and [Media Recording](/guide/media/media-recording) guides for the APIs it uses. +The [`EncryptedRecordingExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java) encrypts a call end to end with a key provider of [End-to-End Encryption](/guide/advanced/end-to-end-encryption), with the [Client API](/guide/client/transforms), and records what the receiving side decrypted into a media file. See the [Encoded Transforms](/guide/advanced/encoded-transforms) and [Media Recording](/guide/media/media-recording) guides for the APIs it uses. **Key features demonstrated:** - Encrypting every encoded audio and video frame in the SFrame format with `e2ee(keys)` of the peer options, and decrypting it, from the first frame on @@ -231,7 +231,7 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.EncryptedRecordingExa ## Web Client -The [`WebClientExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/WebClientExample.java) calls a browser from Java over WebSocket signaling, with the [Client API](/guide/client-api). The browser client is served by the example's `WebServer`. +The [`WebClientExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/WebClientExample.java) calls a browser from Java over WebSocket signaling, with the [Client API](/guide/client-api). The browser client is served by the example's `WebServer`. **Key features demonstrated:** - Connecting to a signaling server using WebSockets diff --git a/docs/guide/get-started.md b/docs/guide/get-started.md index a17862043..6630400da 100644 --- a/docs/guide/get-started.md +++ b/docs/guide/get-started.md @@ -1,6 +1,6 @@ -# Get Started +# Getting Started -This guide will help you get started with webrtc-java quickly. We'll cover installation, a first call with the client API, and the same steps with the core API. +This guide will help you get started with webrtc-java quickly. We'll cover installation and a first call with the client API. The [Core API](/guide/core-api) page makes the same call with the core API. ## Supported Platforms Maven Central artifacts contain native libraries that can be loaded on the following platforms: @@ -23,7 +23,6 @@ apt-get install libx11-6 libxext6 libxfixes3 libxdamage1 libxtst6 libxrandr2 lib Since 0.17.0, PulseAudio, udev and D-Bus are opened at runtime only when audio devices, cameras or the screen saver inhibition are used, so the library loads without `libpulse0`, `libudev1` and `libdbus-1-3`. Earlier versions link them and need those three packages as well. ::: - ## Installation ### Maven @@ -135,7 +134,7 @@ The Linux libraries are built against glibc. They do not load on a system with t ## Your First Call -webrtc-java has two APIs. The [Client API](/guide/client-api) in `dev.onvoid.webrtc.client` negotiates by itself, keeps your code off WebRTC's threads, and closes everything it created with one call. The core API in `dev.onvoid.webrtc` mirrors WebRTC's own classes one to one, for full control. Start with the client API; [Using the Core API](#using-the-core-api) below shows the same steps with the core API. +webrtc-java has two APIs. The [Client API](/guide/client-api) in `dev.onvoid.webrtc.client` negotiates by itself, keeps your code off WebRTC's threads, and closes everything it created with one call. The core API in `dev.onvoid.webrtc` mirrors WebRTC's own classes one to one, for full control. Start with the client API; the [Core API](/guide/core-api) page shows the same steps with the core API. ### Create the Runtime @@ -207,315 +206,14 @@ peer.whenConnected().get(10, TimeUnit.SECONDS); peer.close(); ``` -The [Client API](/guide/client-api) guide covers everything else: pushing your own audio and video, manual negotiation for WHIP and WHEP, statistics and errors. The [`ClientApiExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java) runs a whole call between two peers in one process, without any hardware. - -## Using the Core API - -The core API leaves negotiation, threads and the lifetime of native objects to you. The steps below make the same call with it. - - -### Initialize the WebRTC Library - -First, create a `PeerConnectionFactory` which is the entry point for most WebRTC operations: - -```java -import dev.onvoid.webrtc.PeerConnectionFactory; - -// Create a peer connection factory -PeerConnectionFactory factory = new PeerConnectionFactory(); -``` - -### Create a Peer Connection - -To establish communication with another peer, you need to create a `RTCPeerConnection`: - -```java -import dev.onvoid.webrtc.RTCConfiguration; -import dev.onvoid.webrtc.RTCIceCandidate; -import dev.onvoid.webrtc.RTCIceServer; -import dev.onvoid.webrtc.RTCPeerConnection; -import dev.onvoid.webrtc.PeerConnectionObserver; - -// Configure ICE servers (STUN/TURN) -RTCConfiguration config = new RTCConfiguration(); -RTCIceServer iceServer = new RTCIceServer(); -iceServer.urls.add("stun:stun.l.google.com:19302"); -config.iceServers.add(iceServer); - -// Create a peer connection with an observer to handle events -RTCPeerConnection peerConnection = factory.createPeerConnection(config, new PeerConnectionObserver() { - @Override - public void onIceCandidate(RTCIceCandidate candidate) { - // Send the candidate to the remote peer, see below. The other - // methods are optional. - } -}); -``` - -### Signaling - -WebRTC requires a signaling mechanism to exchange connection information between peers. The library doesn't provide this, so you'll need to implement it using your preferred method (WebSockets, HTTP, etc.). - -Here's a simplified example of the signaling process: - -```java -// Create an offer -RTCOfferOptions options = new RTCOfferOptions(); - -peerConnection.createOffer(options, new CreateSessionDescriptionObserver() { - @Override - public void onSuccess(RTCSessionDescription description) { - // Set local description - peerConnection.setLocalDescription(description, new SetSessionDescriptionObserver() { - @Override - public void onSuccess() { - // Send the offer to the remote peer via your signaling channel - signalingChannel.send(description); - } - - @Override - public void onFailure(String error) { - System.err.println("Failed to set local description: " + error); - } - }); - } - - @Override - public void onFailure(String error) { - System.err.println("Failed to create offer: " + error); - } -}); - -// When you receive an answer from the remote peer via your signaling channel -signalingChannel.onMessage(message -> { - RTCSessionDescription remoteDescription = parseSessionDescription(message); - peerConnection.setRemoteDescription(remoteDescription, new SetSessionDescriptionObserver() { - @Override - public void onSuccess() { - System.out.println("Remote description set successfully"); - } - - @Override - public void onFailure(String error) { - System.err.println("Failed to set remote description: " + error); - } - }); -}); - -// Handle ICE candidates -PeerConnectionObserver connectionObserver = new PeerConnectionObserver() { - @Override - public void onIceCandidate(RTCIceCandidate candidate) { - // Send the ICE candidate to the remote peer via your signaling channel - signalingChannel.send(candidate); - } -}; - -// When you receive an ICE candidate from the remote peer -signalingChannel.onIceCandidate(candidateMessage -> { - RTCIceCandidate candidate = parseIceCandidate(candidateMessage); - peerConnection.addIceCandidate(candidate); -}); -``` - -#### Letting the Peer Connection Create the Description - -`setLocalDescription` can also be called with only an observer. It then creates the offer or the answer, whichever the signaling state calls for, and applies it in one step, so nothing can change in between. That makes it the simpler way to implement "perfect negotiation". Read the result with `getLocalDescription()` once the observer reports success: - -```java -peerConnection.setLocalDescription(new SetSessionDescriptionObserver() { - @Override - public void onSuccess() { - signalingChannel.send(peerConnection.getLocalDescription()); - } - - @Override - public void onFailure(String error) { - System.err.println("Failed to set local description: " + error); - } -}); -``` - -#### Adding Candidates and Knowing the Outcome - -`addIceCandidate(candidate)` reports nothing and fails silently when the remote description is not set yet. The form that takes an `AddIceCandidateObserver` is queued behind any pending offer/answer operation, as the standard requires, so a candidate can be added right after `setRemoteDescription()` without waiting for it, and the observer says whether it was accepted: - -```java -peerConnection.addIceCandidate(candidate, new AddIceCandidateObserver() { - @Override - public void onSuccess() { - } - - @Override - public void onFailure(String error) { - // e.g. "[INVALID_STATE] The remote description was null" - System.err.println("Candidate rejected: " + error); - } -}); -``` - -A local candidate can also go away, for example when its network interface does. The observer's `onIceCandidateRemoved(RTCIceCandidate)` reports it; send it to the remote peer, which passes it to `removeIceCandidate(candidate)`. - -Once ICE has picked the candidate pair it uses, `onSelectedCandidatePairChanged(RTCCandidatePairChangeEvent)` reports both candidates of the pair and why it was chosen. It fires again whenever ICE switches pairs: - -```java -@Override -public void onSelectedCandidatePairChanged(RTCCandidatePairChangeEvent event) { - System.out.println("Using " + event.getLocal().sdp + " <-> " + event.getRemote().sdp - + " (" + event.getReason() + ")"); -} -``` - -### Media Streams - -#### Accessing Media Devices - -To query media devices (cameras and microphones): - -```java -import dev.onvoid.webrtc.media.MediaDevices; -import dev.onvoid.webrtc.media.video.VideoDevice; -import dev.onvoid.webrtc.media.audio.AudioDevice; - -// Get available video devices -List videoDevices = MediaDevices.getVideoCaptureDevices(); -for (VideoDevice device : videoDevices) { - System.out.println("Video device: " + device.getName()); -} - -// Get available audio devices -List audioDevices = MediaDevices.getAudioCaptureDevices(); -for (AudioDevice device : audioDevices) { - System.out.println("Audio device: " + device.getName()); -} -``` - -#### Creating Media Tracks - -To create audio and video tracks: - -```java -import dev.onvoid.webrtc.media.video.VideoTrack; -import dev.onvoid.webrtc.media.audio.AudioTrack; -import dev.onvoid.webrtc.media.video.VideoDeviceSource; -import dev.onvoid.webrtc.media.audio.AudioOptions; -import dev.onvoid.webrtc.media.audio.AudioTrackSource; - -// Create a video source and track -VideoDeviceSource videoSource = new VideoDeviceSource(); -videoSource.setVideoCaptureDevice(videoDevices.get(0)); // Use the first available camera -VideoTrack videoTrack = factory.createVideoTrack("video0", videoSource); - -// Create an audio source and track -AudioOptions audioOptions = new AudioOptions(); -audioOptions.echoCancellation = true; -audioOptions.autoGainControl = true; -audioOptions.noiseSuppression = true; - -// Create an audio source using the default audio device -AudioTrackSource audioSource = factory.createAudioSource(audioOptions); -AudioTrack audioTrack = factory.createAudioTrack("audio0", audioSource); -``` - -::: info -Audio devices can be specified via the `AudioDeviceModule`, see the guide on [Audio Devices][audio-devices] for more details. -::: - -#### Adding Tracks to Peer Connection - -```java -import java.util.ArrayList; -import java.util.List; -import dev.onvoid.webrtc.RTCRtpSender; - -// Add tracks to the peer connection -List streamIds = new ArrayList<>(); -streamIds.add("stream1"); -RTCRtpSender videoSender = peerConnection.addTrack(videoTrack, streamIds); -RTCRtpSender audioSender = peerConnection.addTrack(audioTrack, streamIds); -``` - -#### Receiving Tracks - -A remote track arrives in `onTrack` with the transceiver that carries it. To learn when its media actually starts flowing, for example to show a video only once there is something to show, set an `RTCRtpReceiverObserver`; an `RTCRtpSenderObserver` does the same for the first packet a sender sends. Either is called right away when that has already happened: - -```java -@Override -public void onTrack(RTCRtpTransceiver transceiver) { - RTCRtpReceiver receiver = transceiver.getReceiver(); - - receiver.setObserver(mediaType -> System.out.println("First " + mediaType + " packet received")); - - MediaStreamTrack track = receiver.getTrack(); - - if (track instanceof AudioTrack) { - // Playout volume of this remote track, 0 to 10; 1 plays it as received. - ((AudioTrack) track).setVolume(0.5); - } -} -``` - -An observer belongs to the `RTCRtpReceiver` or `RTCRtpSender` object it was set through, and is removed when that object is disposed. `getId()` identifies a sender or receiver, and `RTCRtpSender.getStreams()` returns the stream IDs its track was added with. - -### Cleanup - -Always properly dispose of resources when you're done: - -```java -// Dispose of senders (not owned by the peer connection) -videoSender.dispose(); -audioSender.dispose(); - -// Dispose of tracks -videoTrack.dispose(); -audioTrack.dispose(); - -// Dispose of sources -videoSource.dispose(); -audioSource.dispose(); - -// Close peer connection and release resources -peerConnection.close(); - -// Dispose of factory -factory.dispose(); -``` - -A track may be disposed, and a listener removed, from within one of its listeners: the listeners the notification has not reached yet are then not called, and what goes is freed once the notification is over. An observer registered on one thread while the object is disposed on another is freed once and never left registered. A factory disposed while a closed connection is still being freed is released once that connection is. - -`dispose()` and `close()` may be called from any thread and more than once: only the first call frees anything, later ones do nothing. Once an object is disposed, its methods throw an `IllegalStateException` (a closed connection still reports its state). A method that another thread is running at that moment keeps the native object until it returns, and what the dispose frees is freed then. - -Apart from the media sources, which only have `dispose()`, objects with a `dispose()` method also have a `close()` that does the same. A try-with-resources statement therefore releases them, also when an exception leaves the block: - -```java -try (AudioConverter converter = new AudioConverter(48000, 1, 16000, 1)) { - converter.convert(input, output); -} -``` - -`RTCDataChannel` is the exception: its `close()` closes the channel, which then still has to be disposed. - -#### Finding Objects That Were Never Disposed - -An object that becomes unreachable without being disposed or closed leaks its native object. To find such objects, start the JVM with `-Dwebrtc.java.leak.detection=true`: each one is then reported, with its class, once the garbage collector finds it unreachable. Add `-Dwebrtc.java.leak.trace=true` to have the report tell where the object was created, at the cost of a stack trace per object. - -```shell -java -Dwebrtc.java.leak.detection=true -Dwebrtc.java.leak.trace=true -jar app.jar -``` - -The reports are warnings through `Logging`, which writes nothing unless the application asks for it: - -```java -Logging.logToDebug(Logging.Severity.WARNING); -``` - -Only objects that own a native object and have a `dispose()` or `close()` method are watched: not views such as the track of `RTCRtpSender.getTrack()`, which own nothing, and not reference-counted objects such as video frame buffers. The detector only reports; it never frees anything itself, since a native object may still be in use on a WebRTC thread. While it is off, which is the default, nothing is watched. +The [Client API](/guide/client-api) guide covers everything else: pushing your own audio and video, manual negotiation for WHIP and WHEP, statistics and errors. The [`ClientApiExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ClientApiExample.java) runs a whole call between two peers in one process, without any hardware. ## Next Steps Now that you have a basic understanding of webrtc-java, you can explore more advanced features: - [Client API][client-api] - Everything the client API does, and how its threads work +- [Core API][core-api] - The same call with the core API, which leaves negotiation, threads and cleanup to you - [Data Channels][data-channels] - Learn more about data channels - [RTC Stats][rtc-stats] - Monitor connection quality - [Desktop Capture][desktop-capture] - Share screens and windows @@ -524,6 +222,7 @@ Now that you have a basic understanding of webrtc-java, you can explore more adv For a complete API reference, check the [JavaDoc][javadoc]. [client-api]: /guide/client-api +[core-api]: /guide/core-api [audio-devices]: /guide/audio/audio-devices [data-channels]: /guide/data/data-channels [rtc-stats]: /guide/monitoring/rtc-stats diff --git a/docs/guide/index.md b/docs/guide/index.md index 8d32c6e1c..003430cef 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -1,38 +1,57 @@ # Guide Overview -This section provides detailed guides for various features of the webrtc-java library. +Every guide of the webrtc-java library, in the order of the sidebar. New to the library? Start with [Getting Started](/guide/get-started). + +## Introduction + +- [Introduction](/guide/introduction) - What the library is, and which of its two APIs to use +- [Getting Started](/guide/get-started) - Installation and a first call with the client API +- [Core API](/guide/core-api) - The same call with the core API, which leaves negotiation, threads and cleanup to you + +## Client API + +- [Client API](/guide/client-api) - The runtime, closing, errors and reaching the core API +- [Peers and Signaling](/guide/client/peers) - Creating peers, carrying their messages, connection state and ICE recovery +- [Threads](/guide/client/threads) - Where handlers run, limiting threads and the UI thread +- [Sending Media](/guide/client/sending-media) - Cameras, microphones, screens, files and pushed audio and video +- [Receiving Media](/guide/client/receiving-media) - Remote tracks, video frames, video views, audio and recording +- [Transforms and Encryption](/guide/client/transforms) - Encoded transforms and end-to-end encryption of a peer's media +- [Data Channels](/guide/client/data-channels) - Messages between peers, flow control and channel options +- [WHIP and WHEP](/guide/client/whip-whep) - Publishing to and playing from media servers +- [Statistics and Call Checks](/guide/client/statistics) - Connection statistics, diagnosis and checks before a call ## Media Basics -- [Media Devices](/guide/media/media-devices) - Working with audio and video devices -- [Bitrate and Framerate Constraints](/guide/media/constraints) - Controlling media quality -- [Send-only and Receive-only](/guide/media/directionality) - Configure transceiver directions (send-only, receive-only or inactive) +- [Media Devices](/guide/media/media-devices) - Listing microphones, speakers and cameras, and following hotplug events +- [Media Constraints](/guide/media/constraints) - Bitrate, frame rate, resolution, simulcast and priorities of what is sent +- [Media Directionality](/guide/media/directionality) - Send-only, receive-only and inactive transceivers - [Media Files](/guide/media/media-files) - Sending video and audio files instead of a camera and microphone - [Media Recording](/guide/media/media-recording) - Recording what a call sends or receives into a media file, into rotating segments, or into a replay buffer, without re-encoding ## Audio -- [Audio Device Selection](/guide/audio/audio-devices) - Selecting and configuring audio devices -- [Audio Processing](/guide/audio/audio-processing) - Voice processing components -- [Custom Audio Source](/guide/audio/custom-audio-source) - Using custom audio sources with WebRTC +- [Audio Devices](/guide/audio/audio-devices) - Selecting and configuring audio devices +- [Audio Processing](/guide/audio/audio-processing) - Echo cancellation, noise suppression, gain control and other voice processing +- [Custom Audio Source](/guide/audio/custom-audio-source) - Sending audio your application produces +- [System Audio Source](/guide/audio/system-audio-source) - Sending what the system plays, beside a shared screen - [Headless Audio](/guide/audio/headless-audio) - Playout pull without touching real OS audio devices - [DTMF Sender](/guide/audio/dtmf-sender) - Sending DTMF tones in a call ## Video -- [Camera Capture](/guide/video/camera-capture) - Capturing video from cameras -- [Desktop Capture](/guide/video/desktop-capture) - Capturing and sharing screens and windows -- [Custom Video Source](/guide/video/custom-video-source) - Using custom video sources with WebRTC +- [Camera Video](/guide/video/camera-capture) - Capturing video from cameras +- [Desktop Video](/guide/video/desktop-capture) - Capturing and sharing screens and windows +- [Custom Video Source](/guide/video/custom-video-source) - Sending video frames your application produces ## Data Communication -- [Data Channels](/guide/data/data-channels) - Sending and receiving arbitrary data between peers +- [Data Channels](/guide/data/data-channels) - Sending and receiving arbitrary data between peers with the core API - [File Transfer](/guide/data/file-transfer) - Sending files with progress, confirmation, checksum and resume ## Networking and ICE - [Peer Connection Configuration](/guide/networking/peer-connection-config) - Candidate gathering, ICE checks, TURN, SRTP ciphers and other connection settings -- [Port Allocator Config](/guide/networking/port-allocator-config) - Restrict ICE port ranges and control candidate gathering behavior +- [Port Allocator Configuration](/guide/networking/port-allocator-config) - Restricting ICE port ranges and controlling candidate gathering ## Monitoring and Debugging @@ -43,9 +62,18 @@ This section provides detailed guides for various features of the webrtc-java li - [Field Trials](/guide/advanced/field-trials) - Enabling experimental features and tuning WebRTC internals - [Encoded Transforms](/guide/advanced/encoded-transforms) - Reading and changing encoded frames, e.g. for end-to-end encryption -- [Video Codecs](/guide/advanced/video-codecs) - Adding video codecs implemented in Java, or limiting the built-in ones +- [End-to-End Encryption](/guide/advanced/end-to-end-encryption) - Encrypting a call so that servers forward media they cannot read +- [Video Codecs](/guide/advanced/video-codecs) - Hardware codecs, adding video codecs implemented in Java, or limiting the built-in ones - [Audio Codecs](/guide/advanced/audio-codecs) - Limiting and ordering the built-in audio codecs +- [GraalVM Native Image](/guide/advanced/native-image) - A single executable that needs no installed Java runtime + +## Background + +- [Client API Design](/guide/client-api-design) - The decisions behind the client API, its threading model and its full API surface +- [Build Notes](/guide/build) - Building the library from source -## Additional Resources +## More -For a complete API reference, check the [JavaDoc](https://javadoc.io/doc/dev.onvoid.webrtc/webrtc-java/latest/index.html). \ No newline at end of file +- [Tools](/tools/) - Small helpers that work without a peer connection, such as audio players, converters and capturers +- [Examples](/guide/examples) - The example applications of the `webrtc-examples` module +- [JavaDoc](https://javadoc.io/doc/dev.onvoid.webrtc/webrtc-java/latest/index.html) - The complete API reference diff --git a/docs/guide/introduction.md b/docs/guide/introduction.md index 8b35fedeb..3a681e62f 100644 --- a/docs/guide/introduction.md +++ b/docs/guide/introduction.md @@ -4,23 +4,29 @@ webrtc-java is a Java wrapper for the [WebRTC Native API](https://webrtc.github. The library provides a comprehensive set of Java classes that map to the WebRTC C++ API, making it possible to establish peer-to-peer connections, transmit audio and video, share screens, and exchange arbitrary data between applications. -It has two APIs. The [Client API](/guide/client-api) negotiates connections by itself, keeps application code off WebRTC's threads, and releases everything it created with one call, which makes it the place to start. The core API maps WebRTC's classes one to one, for applications that need full control. +## Which API? + +The library has two APIs: + +| | Client API | Core API | +|---|---|---| +| Package | `dev.onvoid.webrtc.client` | `dev.onvoid.webrtc` | +| Negotiation | Runs by itself | Your code exchanges offers, answers and candidates | +| Threads | Your handlers never run on a WebRTC thread | Callbacks run on WebRTC's threads | +| Cleanup | Closing the runtime closes everything | You dispose every native object | +| Use it for | Most applications: start here | Full control over WebRTC's classes | + +The [Client API](/guide/client-api) is built on the core API and can reach it where it needs to. Most guides under Media, Audio, Video and Networking show the core API; a note at their top names the client API page for the same task. ## Overview For more detailed information, check out the documentation: -- [Get Started](/guide/get-started) - Get up and running quickly with webrtc-java +- [Getting Started](/guide/get-started) - Get up and running quickly with webrtc-java - [Guides](/guide/) - Comprehensive documentation on using the library - [Examples](/guide/examples) - Sample code demonstrating various features - [Build Notes](/guide/build) - Instructions for building the library from source ## License -Copyright (c) 2019 Alex Andres - -Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - -[http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) - -Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. \ No newline at end of file +webrtc-java is licensed under the [Apache License, Version 2.0](https://github.com/devopvoid/webrtc-java/blob/main/LICENSE). diff --git a/docs/guide/media/constraints.md b/docs/guide/media/constraints.md index c9fc5c332..aa18000cc 100644 --- a/docs/guide/media/constraints.md +++ b/docs/guide/media/constraints.md @@ -1,5 +1,7 @@ # Media Constraints + + This guide explains how to set bitrate and framerate constraints for MediaStreamTrack of RTCRtpSender. It covers: - Understanding RTCRtpSender Parameters diff --git a/docs/guide/media/directionality.md b/docs/guide/media/directionality.md index c96d40a56..1f0c6e175 100644 --- a/docs/guide/media/directionality.md +++ b/docs/guide/media/directionality.md @@ -1,5 +1,7 @@ # Media Directionality + + This guide explains how to configure WebRTC media to be receive-only or send-only using the webrtc-java API. It also shows the equivalent configuration on the JavaScript (browser) side. You will learn how to: - Create receive-only audio/video transceivers diff --git a/docs/guide/media/media-devices.md b/docs/guide/media/media-devices.md index 2d54dc059..8e58b6136 100644 --- a/docs/guide/media/media-devices.md +++ b/docs/guide/media/media-devices.md @@ -1,5 +1,7 @@ # Media Devices + + This guide explains how to work with media devices (microphones, speakers, and cameras). It covers: - Querying available media devices diff --git a/docs/guide/media/media-recording.md b/docs/guide/media/media-recording.md index a45225163..2e934f6a8 100644 --- a/docs/guide/media/media-recording.md +++ b/docs/guide/media/media-recording.md @@ -232,4 +232,4 @@ KeyProvider keys = KeyProvider.builder(myId) - A transform of your own allows it unless its `allowsPlaintextObservers()` returns false. -A recording of an encrypted call holds the decrypted media, outside the protection of the call. [`EncryptedRecordingExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java) shows both together. +A recording of an encrypted call holds the decrypted media, outside the protection of the call. [`EncryptedRecordingExample`](https://github.com/devopvoid/webrtc-java/blob/main/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/EncryptedRecordingExample.java) shows both together. diff --git a/docs/guide/monitoring/rtc-stats.md b/docs/guide/monitoring/rtc-stats.md index 9a687da18..760f6686b 100644 --- a/docs/guide/monitoring/rtc-stats.md +++ b/docs/guide/monitoring/rtc-stats.md @@ -1,5 +1,7 @@ # RTC Stats + + This guide explains how to use WebRTC statistics (RTC Stats) with the webrtc-java library. RTC Stats provide detailed metrics and information about the state and performance of your WebRTC connections. ## Overview diff --git a/docs/guide/networking/peer-connection-config.md b/docs/guide/networking/peer-connection-config.md index ade5e4bc5..b9000eeae 100644 --- a/docs/guide/networking/peer-connection-config.md +++ b/docs/guide/networking/peer-connection-config.md @@ -1,5 +1,7 @@ # Peer Connection Configuration + + `RTCConfiguration` holds the settings a peer connection is created with: ICE servers and policies, how candidates are gathered, how ICE checks connections, and how media is protected. This guide covers the settings beyond the ICE servers, which most applications leave at their defaults, but which matter for fast connecting, restrictive networks, and mobile or multi-homed hosts. Every setting described here is unset by default, which keeps WebRTC's default. `getConfiguration()` returns the values in effect, defaults included. diff --git a/docs/guide/networking/port-allocator-config.md b/docs/guide/networking/port-allocator-config.md index fa153388c..81349199d 100644 --- a/docs/guide/networking/port-allocator-config.md +++ b/docs/guide/networking/port-allocator-config.md @@ -1,4 +1,6 @@ -# Port Allocator Configuration (ICE) +# Port Allocator Configuration + + This guide explains how to configure the ICE port allocator using `dev.onvoid.webrtc.PortAllocatorConfig` and how to use it with `RTCConfiguration` when creating a peer connection. diff --git a/docs/guide/video/camera-capture.md b/docs/guide/video/camera-capture.md index 088959683..5061c370c 100644 --- a/docs/guide/video/camera-capture.md +++ b/docs/guide/video/camera-capture.md @@ -1,5 +1,7 @@ # Camera Video + + This guide focuses on setting up a peer connection with camera video capture capabilities, which allows you to capture and stream video from connected cameras in your WebRTC connection. ## Camera Video Source Selection diff --git a/docs/guide/video/custom-video-source.md b/docs/guide/video/custom-video-source.md index 0873707b5..18178dc7b 100644 --- a/docs/guide/video/custom-video-source.md +++ b/docs/guide/video/custom-video-source.md @@ -1,5 +1,7 @@ # Custom Video Source + + This guide explains how to use the `CustomVideoSource` class to push video frames from external sources directly to the WebRTC video pipeline. ## Overview diff --git a/docs/guide/video/desktop-capture.md b/docs/guide/video/desktop-capture.md index 9712ed7be..09cf14210 100644 --- a/docs/guide/video/desktop-capture.md +++ b/docs/guide/video/desktop-capture.md @@ -1,5 +1,7 @@ # Desktop Video + + This guide focuses on setting up a peer connection with desktop video capture capabilities, which allows you to capture and stream content from your screens or application windows in your WebRTC connection. ## Desktop Video Source Selection diff --git a/docs/tools/audio/audio-converter.md b/docs/tools/audio/audio-converter.md index 5603aa270..eae2c068c 100644 --- a/docs/tools/audio/audio-converter.md +++ b/docs/tools/audio/audio-converter.md @@ -105,4 +105,4 @@ finally { - [Audio Processing](/guide/audio/audio-processing) - [Headless Audio](/guide/audio/headless-audio) -- [Voice Activity Detector](/tools/audio/voice-activity-detector) +- [Voice Activity Detection](/tools/audio/voice-activity-detector) diff --git a/docs/tools/audio/audio-player.md b/docs/tools/audio/audio-player.md index 362295b0b..b8af6a87d 100644 --- a/docs/tools/audio/audio-player.md +++ b/docs/tools/audio/audio-player.md @@ -8,7 +8,7 @@ API: `dev.onvoid.webrtc.media.audio.AudioPlayer` - You want to render raw PCM audio (generated or decoded by your app) to an OS output device. - You need a simple, high‑level start/stop wrapper around WebRTC’s audio playout. -See also: [Audio Device Selection](/guide/audio/audio-devices), [Custom Audio Source](/guide/audio/custom-audio-source), [Headless Audio](/guide/audio/headless-audio). +See also: [Audio Devices](/guide/audio/audio-devices), [Custom Audio Source](/guide/audio/custom-audio-source), [Headless Audio](/guide/audio/headless-audio). ## Key concepts - Device selection: Provide an `AudioDevice` representing the output device (speaker) before starting. diff --git a/docs/tools/audio/audio-recorder.md b/docs/tools/audio/audio-recorder.md index b0113b68a..1fb19078a 100644 --- a/docs/tools/audio/audio-recorder.md +++ b/docs/tools/audio/audio-recorder.md @@ -8,7 +8,7 @@ API: `dev.onvoid.webrtc.media.audio.AudioRecorder` - You want a simple way to record microphone input without wiring a full PeerConnection. - You need raw 16‑bit PCM frames (10 ms) delivered to your code for analysis, file writing, or custom processing. -See also: [Audio Device Selection](/guide/audio/audio-devices), [Audio Processing](/guide/audio/audio-processing), [Custom Audio Source](/guide/audio/custom-audio-source). +See also: [Audio Devices](/guide/audio/audio-devices), [Audio Processing](/guide/audio/audio-processing), [Custom Audio Source](/guide/audio/custom-audio-source). ## Key concepts - Device selection: Provide an `AudioDevice` representing the input device (microphone) before starting. @@ -57,7 +57,7 @@ public class MicRecorderExample { ## Tips - Apply echo cancellation, AGC, and noise suppression via the [Audio Processing](/guide/audio/audio-processing) guide if needed. - If you need to resample or remix, use the [Audio Converter](/tools/audio/audio-converter). -- Device selection details and best practices are covered in [Audio Device Selection](/guide/audio/audio-devices). +- Device selection details and best practices are covered in [Audio Devices](/guide/audio/audio-devices). ## API reference - `setAudioDevice(AudioDevice device)` – choose input device diff --git a/docs/tools/audio/voice-activity-detector.md b/docs/tools/audio/voice-activity-detector.md index bb4206e81..f1fdd913a 100644 --- a/docs/tools/audio/voice-activity-detector.md +++ b/docs/tools/audio/voice-activity-detector.md @@ -157,7 +157,7 @@ factory, after echo cancellation and noise suppression, so the remote peer's voice from the speakers does not count as this side speaking: `PeerConnectionFactory.addVoiceActivityListener`, see [Audio Processing](/guide/audio/audio-processing#speech-in-the-processed-microphone-audio). -With the [Client API](/guide/client-api#audio), `LocalAudio.onVoiceActivity` +With the [Client API](/guide/client/receiving-media#audio), `LocalAudio.onVoiceActivity` covers both, for the microphone and for pushed audio: ```java diff --git a/docs/tools/desktop/power-management.md b/docs/tools/desktop/power-management.md index 690dfce17..c34bf59bf 100644 --- a/docs/tools/desktop/power-management.md +++ b/docs/tools/desktop/power-management.md @@ -58,5 +58,5 @@ Exact mechanisms are handled by the native layer; your Java code remains the sam ## Related guides -- [Desktop Capture](/guide/video/desktop-capture) +- [Desktop Video](/guide/video/desktop-capture) - [Logging](/guide/monitoring/logging) diff --git a/docs/tools/desktop/screen-capturer.md b/docs/tools/desktop/screen-capturer.md index 67b9219b1..8bc91f4aa 100644 --- a/docs/tools/desktop/screen-capturer.md +++ b/docs/tools/desktop/screen-capturer.md @@ -108,5 +108,5 @@ sc.dispose(); ## Related guides -- [Desktop Capture](/guide/video/desktop-capture) +- [Desktop Video](/guide/video/desktop-capture) - [Power Management](/tools/desktop/power-management) diff --git a/docs/tools/desktop/window-capturer.md b/docs/tools/desktop/window-capturer.md index 6f533b5c6..e33c898ec 100644 --- a/docs/tools/desktop/window-capturer.md +++ b/docs/tools/desktop/window-capturer.md @@ -101,5 +101,5 @@ wc.dispose(); ## Related guides -- [Desktop Capture](/guide/video/desktop-capture) +- [Desktop Video](/guide/video/desktop-capture) - [Power Management](/tools/desktop/power-management) diff --git a/docs/tools/index.md b/docs/tools/index.md index 9e8a7f465..67685c53c 100644 --- a/docs/tools/index.md +++ b/docs/tools/index.md @@ -9,7 +9,7 @@ Use these pages to pick the right tool and jump straight to code examples. - [Audio Converter](/tools/audio/audio-converter) — Resample and remix 10 ms PCM frames between rates and channel layouts. - [Audio Recorder](/tools/audio/audio-recorder) — Capture microphone input and receive 10 ms PCM frames via an `AudioSink`. - [Audio Player](/tools/audio/audio-player) — Play PCM audio to an output device by supplying frames via an `AudioSource`. -- [Voice Activity Detector](/tools/audio/voice-activity-detector) — Estimate speech probability for short PCM frames. +- [Voice Activity Detection](/tools/audio/voice-activity-detector) — Estimate speech probability for short PCM frames. ## Video @@ -31,10 +31,10 @@ Use these pages to pick the right tool and jump straight to code examples. ## Related guides -- [Camera Capture](/guide/video/camera-capture) -- [Desktop Capture](/guide/video/desktop-capture) +- [Camera Video](/guide/video/camera-capture) +- [Desktop Video](/guide/video/desktop-capture) - [Custom Video Source](/guide/video/custom-video-source) -- [Audio Device Selection](/guide/audio/audio-devices) +- [Audio Devices](/guide/audio/audio-devices) - [Audio Processing](/guide/audio/audio-processing) - [RTC Stats](/guide/monitoring/rtc-stats) diff --git a/docs/tools/video/video-capturer.md b/docs/tools/video/video-capturer.md index 86474798f..18d6c2aeb 100644 --- a/docs/tools/video/video-capturer.md +++ b/docs/tools/video/video-capturer.md @@ -106,7 +106,7 @@ If you need pixel format conversion, see `VideoBufferConverter`. ## Related guides -- [Camera Capture](/guide/video/camera-capture) +- [Camera Video](/guide/video/camera-capture) - [Custom Video Source](/guide/video/custom-video-source) - [Screen Capturer](/tools/desktop/screen-capturer) - [Window Capturer](/tools/desktop/window-capturer)