diff --git a/README.md b/README.md index 7f395d08f..aafde85c9 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ The library provides a comprehensive set of Java classes that map to the WebRTC - **[Cameras and microphones](https://jrtc.dev/guide/media/media-devices)** - Capture from any local device - **[Screen sharing](https://jrtc.dev/guide/video/desktop-capture)** - Share the full desktop or single application windows +- **[System audio](https://jrtc.dev/guide/audio/system-audio-source)** - Send what the system plays beside a shared screen and the microphone - **[Video processing](https://jrtc.dev/guide/video/camera-capture#changing-frames-before-they-are-sent)** - Blur a background, draw an overlay or blank out part of the screen before frames are sent - **[Video views](https://jrtc.dev/guide/client-api#handlers-on-the-ui-thread)** - Ready-made components that show video in Swing with the `webrtc-java-swing` module, or in JavaFX with the `webrtc-java-javafx` module diff --git a/docs/guide/audio/system-audio-source.md b/docs/guide/audio/system-audio-source.md index 295f710ab..f6dc45dd9 100644 --- a/docs/guide/audio/system-audio-source.md +++ b/docs/guide/audio/system-audio-source.md @@ -52,6 +52,16 @@ audioTrack.dispose(); systemAudio.dispose(); // or close(), also supported by try-with-resources ``` +With the [Client API](/guide/client-api), pass the source to `audioTrack()`. It +stays yours, so close it only after the track, or the runtime that closes it: + +```java +SystemAudioSource systemOutput = new SystemAudioSource(); +LocalAudio systemAudio = rtc.media().audioTrack("system-audio", systemOutput); +peer.addTrack(systemAudio, "presentation"); +systemOutput.start(); +``` + Use `try/finally` in application code to release these resources on errors too. `stop()` and `dispose()` are idempotent. `stop()` keeps the source and its tracks usable for `start()` again; disposal is permanent. A stop or disposal from a local @@ -106,7 +116,9 @@ playout is excluded by ScreenCaptureKit; remote audio played in another process can still be captured. Run `SystemAudioExample` from `webrtc-examples` for a capture-only example that -counts received PCM frames without playing them back. The opt-in +counts received PCM frames without playing them back, and +[`ScreenShareAudioExample`](/guide/examples#screen-share-with-audio) to send a +screen, the microphone and the system audio over one peer. The opt-in `SystemAudioSourceIntegrationTest` (`-Dtest.systemAudio=true`) needs audio playing on the default output and recording permission; lifecycle tests use deterministic backends without hardware. diff --git a/docs/guide/client-api.md b/docs/guide/client-api.md index df64ba08c..2eef44fdd 100644 --- a/docs/guide/client-api.md +++ b/docs/guide/client-api.md @@ -476,7 +476,7 @@ Sender audioSender = peer.addTrack(microphone); | `screen(ScreenOptions)` | A screen or a window, as the [Desktop Video](/guide/video/desktop-capture) guide describes. | | `customVideo(String)` | Frames you push. | | `customAudio(String)` | Audio you push, in every mode. | -| `videoTrack(String, VideoTrackSource)`, `audioTrack(String, AudioTrackSource)` | A source of the core API, such as one the [media module](/guide/media/media-files) feeds. The source stays yours. | +| `videoTrack(String, VideoTrackSource)`, `audioTrack(String, AudioTrackSource)` | A source of the core API, such as one the [media module](/guide/media/media-files) feeds, or a [`SystemAudioSource`](/guide/audio/system-audio-source) for the audio the system plays. The source stays yours. | ```java Devices devices = media.devices(); diff --git a/docs/guide/examples.md b/docs/guide/examples.md index 4011bc6a7..2a236fbf2 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -84,6 +84,37 @@ mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.DesktopVideoExample" mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.DesktopVideoExample" -D"exec.args=window" ``` +## System Audio + +The [`SystemAudioExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/SystemAudioExample.java) captures the audio the system plays on its default output with a [`SystemAudioSource`](/guide/audio/system-audio-source), and counts the frames that reach a sink of its track, without sending or playing anything. + +**Key features demonstrated:** +- Capturing the system output as a local audio track +- Reading the captured audio through a sink of the track +- Stopping the capture and checking it for a failure + +Play audio in another application while it runs, and press Enter to stop. On macOS, grant screen and audio recording permission when asked. + +```bash +mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.SystemAudioExample" +``` + +## Screen Share with Audio + +The [`ScreenShareAudioExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ScreenShareAudioExample.java) shares a screen together with the microphone and the system audio, from one runtime and over one peer, with the [Client API](/guide/client-api). + +**Key features demonstrated:** +- Sending a screen, the microphone and the system output as three tracks of one stream +- Sending a `SystemAudioSource` through `audioTrack()`, beside the microphone +- Telling the received audio tracks apart by their IDs +- Counting received audio without playing it back, which the system audio capture would record again + +Each audio sender gets the audio of its own track, so the microphone keeps its echo cancellation and the system output is sent as it plays. Play audio in another application and speak into the microphone while it runs, and press Enter to stop. + +```bash +mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.ScreenShareAudioExample" +``` + ## Media File The [`MediaFileExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/MediaFileExample.java) demonstrates how to send a media file over a peer connection, in place of a camera and a microphone. See the [Media Files](/guide/media/media-files) guide for the API it uses. diff --git a/docs/guide/video/desktop-capture.md b/docs/guide/video/desktop-capture.md index da9127079..9a5d3c483 100644 --- a/docs/guide/video/desktop-capture.md +++ b/docs/guide/video/desktop-capture.md @@ -193,6 +193,29 @@ Desktop capture is particularly useful for screen sharing applications, remote a ## Sharing system audio -Add a separate [`SystemAudioSource`](/guide/audio/system-audio-source) audio track -to share output audio alongside desktop video. Window selection applies only to -video; the audio source captures system output independently. +Add a [`SystemAudioSource`](/guide/audio/system-audio-source) audio track to share +the audio the system plays alongside the desktop video, and the microphone as a +third track if the presenter speaks. All three can come from one factory and go +over one peer connection; put them into one stream so the receiver keeps them in +sync. With the [Client API](/guide/client-api): + +```java +LocalVideo screen = rtc.media().screen(ScreenOptions.screen(source).build()); +LocalAudio microphone = rtc.media().microphone(); + +// The source is yours: close it after the runtime, which closes its track. +SystemAudioSource systemOutput = new SystemAudioSource(); +LocalAudio systemAudio = rtc.media().audioTrack("system-audio", systemOutput); + +peer.addTrack(screen, "presentation"); +peer.addTrack(microphone, "presentation"); +peer.addTrack(systemAudio, "presentation"); + +systemOutput.start(); // on a background thread in an application with a UI +``` + +Window selection applies only to video; the audio source captures the whole +system output. Remote call audio played on the captured output is captured too on +Windows and Linux, so route it to another output. The +[`ScreenShareAudioExample`](/guide/examples#screen-share-with-audio) runs this +end to end. diff --git a/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ScreenShareAudioExample.java b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ScreenShareAudioExample.java new file mode 100644 index 000000000..e049a8498 --- /dev/null +++ b/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/ScreenShareAudioExample.java @@ -0,0 +1,193 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.examples; + +import java.time.Duration; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +import dev.onvoid.webrtc.client.AudioChunk; +import dev.onvoid.webrtc.client.LocalAudio; +import dev.onvoid.webrtc.client.LocalVideo; +import dev.onvoid.webrtc.client.Peer; +import dev.onvoid.webrtc.client.PeerOptions; +import dev.onvoid.webrtc.client.RemoteAudio; +import dev.onvoid.webrtc.client.ScreenOptions; +import dev.onvoid.webrtc.client.Signaling; +import dev.onvoid.webrtc.client.SignalingPair; +import dev.onvoid.webrtc.client.WebRtc; +import dev.onvoid.webrtc.media.audio.SystemAudioSource; +import dev.onvoid.webrtc.media.video.desktop.DesktopSource; + +/** + * Example demonstrating how to share a screen together with the microphone and + * the audio the system plays, from one runtime and over one peer, with the + * client API. + *

+ * This example shows how to: + *

+ *

+ * Each audio sender gets the audio of its own track: the microphone, with echo + * cancellation, noise suppression and gain control, on one, and the system + * output, unprocessed, on the other. Both peers run in this process, so the + * call connects without a server. The viewer does not play what it receives: + * on Windows and Linux the system audio capture would record that playout and + * send it again. + *

+ * Play audio in another application, speak into the microphone, run the + * example and press Enter to stop. On macOS, grant screen and audio recording + * permission when asked: + *

+ * mvn exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.ScreenShareAudioExample"
+ * 
+ * + * @author Alex Andres + */ +public class ScreenShareAudioExample { + + /** A 16-bit sample above this is counted as sound rather than silence. */ + private static final int AUDIBLE = 300; + + + public static void main(String[] args) throws Exception { + // AudioMode.DEVICES, the default: the microphone is captured, and pushed + // audio such as the system output can be sent beside it. + WebRtc rtc = WebRtc.create(); + SystemAudioSource systemOutput = null; + + try { + List screens = rtc.media().devices().screens(); + + if (screens.isEmpty()) { + System.out.println("There is no screen to share"); + return; + } + + DesktopSource source = screens.get(0); + LocalVideo screen = rtc.media().screen(ScreenOptions.screen(source) + .frameRate(30) + .maxSize(1920, 1080) + .build()); + LocalAudio microphone = rtc.media().microphone(); + + // The source stays the application's: it must outlive its track, + // which the runtime closes. + systemOutput = new SystemAudioSource(); + LocalAudio systemAudio = rtc.media().audioTrack("system-audio", systemOutput); + + SignalingPair signaling = Signaling.loopback(); + + Peer sharer = rtc.createPeer(PeerOptions.builder() + .name("sharer") + .signaling(signaling.first()) + .build()); + Peer viewer = rtc.createPeer(PeerOptions.builder() + .name("viewer") + .polite(true) + .signaling(signaling.second()) + .build()); + + AtomicInteger frames = new AtomicInteger(); + Map audibleChunks = new ConcurrentHashMap<>(); + + viewer.onTrack(track -> { + if (track.isVideo()) { + track.asVideo().onFrame(frame -> frames.incrementAndGet()); + return; + } + + String name = track.id().equals(systemAudio.id()) ? "system audio" : "microphone"; + AtomicLong audible = audibleChunks.computeIfAbsent(name, key -> new AtomicLong()); + RemoteAudio audio = track.asAudio(); + + // Count the audio instead of playing it, which the system audio + // capture would pick up again. + audio.setVolume(0); + audio.onAudio(chunk -> { + if (isAudible(chunk)) { + audible.incrementAndGet(); + } + }); + }); + + // One stream for all three tracks, so the viewer lines them up. + sharer.addTrack(screen, "presentation"); + sharer.addTrack(microphone, "presentation"); + sharer.addTrack(systemAudio, "presentation"); + + // Waits for the platform to open the output, and on macOS for the + // permission prompt. An application with a user interface calls it + // on a background thread. + systemOutput.start(); + + // The main thread may wait; a handler must not. + sharer.whenConnected().get(15, TimeUnit.SECONDS); + + System.out.println("Sharing " + source.title + " with the microphone and the system audio"); + + sharer.onStats(Duration.ofSeconds(5), stats -> { + System.out.printf("Viewer got %d frames, audible 10 ms chunks: %s%n", + frames.get(), audibleChunks); + }); + + System.out.println("Press Enter to stop sharing..."); + System.in.read(); + + systemOutput.stop(); + + if (systemOutput.getFailure() != null) { + System.out.println("System audio capture failed: " + systemOutput.getFailure()); + } + } + finally { + // Closing the runtime closed the peers and the tracks, so the system + // audio source can go now. + rtc.close(); + + if (systemOutput != null) { + systemOutput.close(); + } + } + + System.out.println("Closed"); + } + + private static boolean isAudible(AudioChunk chunk) { + byte[] samples = chunk.samples(); + + for (int i = 0; i + 1 < samples.length; i += 2) { + // 16-bit little-endian PCM. + int sample = (short) ((samples[i] & 0xff) | (samples[i + 1] << 8)); + + if (Math.abs(sample) > AUDIBLE) { + return true; + } + } + + return false; + } +}