/** * An implementation of the AVClient which uses the simple-peer library and the Foundry socket server for signaling. * Credit to bekit#4213 for identifying simple-peer as a viable technology and providing a POC implementation. * @extends {AVClient} */ class SimplePeerAVClient extends AVClient { /** * The local Stream which captures input video and audio * @type {MediaStream} */ localStream = null; /** * The dedicated audio stream used to measure volume levels for voice activity detection. * @type {MediaStream} */ levelsStream = null; /** * A mapping of connected peers * @type {Map} */ peers = new Map(); /** * A mapping of connected remote streams * @type {Map} */ remoteStreams = new Map(); /** * Has the client been successfully initialized? * @type {boolean} * @private */ _initialized = false; /** * Is outbound broadcast of local audio enabled? * @type {boolean} */ audioBroadcastEnabled = false; /** * The polling interval ID for connected users that might have unexpectedly dropped out of our peer network. * @type {number|null} */ _connectionPoll = null; /* -------------------------------------------- */ /* Required AVClient Methods */ /* -------------------------------------------- */ /** @override */ async connect() { await this._connect(); clearInterval(this._connectionPoll); this._connectionPoll = setInterval(this._connect.bind(this), CONFIG.WebRTC.connectedUserPollIntervalS * 1000); return true; } /* -------------------------------------------- */ /** * Try to establish a peer connection with each user connected to the server. * @private */ _connect() { const promises = []; for ( let user of game.users ) { if ( user.isSelf || !user.active ) continue; promises.push(this.initializePeerStream(user.id)); } return Promise.all(promises); } /* -------------------------------------------- */ /** @override */ async disconnect() { clearInterval(this._connectionPoll); this._connectionPoll = null; await this.disconnectAll(); return true; } /* -------------------------------------------- */ /** @override */ async initialize() { if ( this._initialized ) return; console.debug(`Initializing SimplePeer client connection`); // Initialize the local stream await this.initializeLocalStream(); // Set up socket listeners this.activateSocketListeners(); // Register callback to close peer connections when the window is closed window.addEventListener("beforeunload", ev => this.disconnectAll()); // Flag the client as initialized this._initialized = true; } /* -------------------------------------------- */ /** @override */ getConnectedUsers() { return [...Array.from(this.peers.keys()), game.userId]; } /* -------------------------------------------- */ /** @override */ getMediaStreamForUser(userId) { return userId === game.user.id ? this.localStream : this.remoteStreams.get(userId); } /* -------------------------------------------- */ /** @override */ getLevelsStreamForUser(userId) { return userId === game.userId ? this.levelsStream : this.getMediaStreamForUser(userId); } /* -------------------------------------------- */ /** @override */ isAudioEnabled() { return !!this.localStream?.getAudioTracks().length; } /* -------------------------------------------- */ /** @override */ isVideoEnabled() { return !!this.localStream?.getVideoTracks().length; } /* -------------------------------------------- */ /** @override */ toggleAudio(enabled) { const stream = this.localStream; if ( !stream ) return; // If "always on" broadcasting is not enabled, don't proceed if ( !this.audioBroadcastEnabled || this.isVoicePTT ) return; // Enable active broadcasting return this.toggleBroadcast(enabled); } /* -------------------------------------------- */ /** @override */ toggleBroadcast(enabled) { const stream = this.localStream; if ( !stream ) return; console.debug(`[SimplePeer] Toggling broadcast of outbound audio: ${enabled}`); this.audioBroadcastEnabled = enabled; for ( let t of stream.getAudioTracks() ) { t.enabled = enabled; } } /* -------------------------------------------- */ /** @override */ toggleVideo(enabled) { const stream = this.localStream; if ( !stream ) return; console.debug(`[SimplePeer] Toggling broadcast of outbound video: ${enabled}`); for (const track of stream.getVideoTracks()) { track.enabled = enabled; } } /* -------------------------------------------- */ /** @override */ async setUserVideo(userId, videoElement) { const stream = this.getMediaStreamForUser(userId); // Set the stream as the video element source if ("srcObject" in videoElement) videoElement.srcObject = stream; else videoElement.src = window.URL.createObjectURL(stream); // for older browsers // Forward volume to the configured audio sink if ( videoElement.sinkId === undefined ) { return console.warn(`[SimplePeer] Your web browser does not support output audio sink selection`); } const requestedSink = this.settings.get("client", "audioSink"); await videoElement.setSinkId(requestedSink).catch(err => { console.warn(`[SimplePeer] An error occurred when requesting the output audio device: ${requestedSink}`); }) } /* -------------------------------------------- */ /* Local Stream Management */ /* -------------------------------------------- */ /** * Initialize a local media stream for the current user * @returns {Promise} */ async initializeLocalStream() { console.debug(`[SimplePeer] Initializing local media stream for current User`); // If there is already an existing local media stream, terminate it if ( this.localStream ) this.localStream.getTracks().forEach(t => t.stop()); this.localStream = null; if ( this.levelsStream ) this.levelsStream.getTracks().forEach(t => t.stop()); this.levelsStream = null; // Determine whether the user can send audio const audioSrc = this.settings.get("client", "audioSrc"); const canBroadcastAudio = this.master.canUserBroadcastAudio(game.user.id); const audioParams = (audioSrc && (audioSrc !== "disabled") && canBroadcastAudio) ? { deviceId: { ideal: audioSrc } } : false; // Configure whether the user can send video const videoSrc = this.settings.get("client", "videoSrc"); const canBroadcastVideo = this.master.canUserBroadcastVideo(game.user.id); const videoParams = (videoSrc && (videoSrc !== "disabled") && canBroadcastVideo) ? { deviceId: { ideal: videoSrc }, width: { ideal: 320 }, height: { ideal: 240 } } : false; // FIXME: Firefox does not allow you to request a specific device, you can only use whatever the browser allows // https://bugzilla.mozilla.org/show_bug.cgi?id=1443294#c7 if ( navigator.userAgent.match(/Firefox/) ) { delete videoParams["deviceId"]; } if ( !videoParams && !audioParams ) return null; let stream = await this._createMediaStream({video: videoParams, audio: audioParams}); if ( (videoParams && audioParams) && (stream instanceof Error) ) { // Even if the game is set to both audio and video, the user may not have one of those devices, or they might have // blocked access to one of them. In those cases we do not want to prevent A/V loading entirely, so we must try // each of them separately to see what is available. if ( audioParams ) stream = await this._createMediaStream({video: false, audio: audioParams}); if ( (stream instanceof Error) && videoParams ) { stream = await this._createMediaStream({video: videoParams, audio: false}); } } if ( stream instanceof Error ) { const error = new Error(`[SimplePeer] Unable to acquire user media stream: ${stream.message}`); error.stack = stream.stack; console.error(error); return null; } this.localStream = stream; this.levelsStream = stream.clone(); this.levelsStream.getVideoTracks().forEach(t => this.levelsStream.removeTrack(t)); return stream; } /* -------------------------------------------- */ /** * Attempt to create local media streams. * @param {{video: object, audio: object}} params Parameters for the getUserMedia request. * @returns {Promise} The created MediaStream or an error. * @private */ async _createMediaStream(params) { try { return await navigator.mediaDevices.getUserMedia(params); } catch(err) { return err; } } /* -------------------------------------------- */ /* Peer Stream Management */ /* -------------------------------------------- */ /** * Listen for Audio/Video updates on the av socket to broker connections between peers */ activateSocketListeners() { game.socket.on("av", (request, userId) => { if ( request.userId !== game.user.id ) return; // The request is not for us, this shouldn't happen switch ( request.action ) { case "peer-signal": if ( request.activity ) this.master.settings.handleUserActivity(userId, request.activity); return this.receiveSignal(userId, request.data); case "peer-close": return this.disconnectPeer(userId); } }); } /* -------------------------------------------- */ /** * Initialize a stream connection with a new peer * @param {string} userId The Foundry user ID for which the peer stream should be established * @returns {Promise} A Promise which resolves once the peer stream is initialized */ async initializePeerStream(userId) { const peer = this.peers.get(userId); if ( peer?.connected || peer?._connecting ) return peer; return this.connectPeer(userId, true); } /* -------------------------------------------- */ /** * Receive a request to establish a peer signal with some other User id * @param {string} userId The Foundry user ID who is requesting to establish a connection * @param {object} data The connection details provided by SimplePeer */ receiveSignal(userId, data) { console.debug(`[SimplePeer] Receiving signal from User [${userId}] to establish initial connection`); let peer = this.peers.get(userId); if ( !peer ) peer = this.connectPeer(userId, false); peer.signal(data); } /* -------------------------------------------- */ /** * Connect to a peer directly, either as the initiator or as the receiver * @param {string} userId The Foundry user ID with whom we are connecting * @param {boolean} isInitiator Is the current user initiating the connection, or responding to it? * @returns {SimplePeer} The constructed and configured SimplePeer instance */ connectPeer(userId, isInitiator=false) { // Create the SimplePeer instance for this connection const peer = this._createPeerConnection(userId, isInitiator); this.peers.set(userId, peer); // Signal to request that a remote user establish a connection with us peer.on("signal", data => { console.debug(`[SimplePeer] Sending signal to User [${userId}] to establish initial connection`); game.socket.emit("av", { action: "peer-signal", userId: userId, data: data, activity: this.master.settings.getUser(game.userId) }, {recipients: [userId]}); }); // Receive a stream provided by a peer peer.on("stream", stream => { console.debug(`[SimplePeer] Received media stream from User [${userId}]`); this.remoteStreams.set(userId, stream); this.master.render(); }); // Close a connection with a current peer peer.on("close", () => { console.debug(`[SimplePeer] Closed connection with remote User [${userId}]`); return this.disconnectPeer(userId); }); // Handle errors peer.on("error", err => { if ( err.code !== "ERR_DATA_CHANNEL" ) { const error = new Error(`[SimplePeer] An unexpected error occurred with User [${userId}]: ${err.message}`); error.stack = err.stack; console.error(error); } if ( peer.connected ) return this.disconnectPeer(userId); }); this.master.render(); return peer; } /* -------------------------------------------- */ /** * Create the SimplePeer instance for the desired peer connection. * Modules may implement more advanced connection strategies by overriding this method. * @param {string} userId The Foundry user ID with whom we are connecting * @param {boolean} isInitiator Is the current user initiating the connection, or responding to it? * @private */ _createPeerConnection(userId, isInitiator) { const options = { initiator: isInitiator, stream: this.localStream }; this._setupCustomTURN(options); return new SimplePeer(options); } /* -------------------------------------------- */ /** * Setup the custom TURN relay to be used in subsequent calls if there is one configured. * TURN credentials are mandatory in WebRTC. * @param {object} options The SimplePeer configuration object. * @private */ _setupCustomTURN(options) { const { url, type, username, password } = this.settings.world.turn; if ( (type !== "custom") || !url || !username || !password ) return; const iceServer = { username, urls: url, credential: password }; options.config = { iceServers: [iceServer] }; } /* -------------------------------------------- */ /** * Disconnect from a peer by stopping current stream tracks and destroying the SimplePeer instance * @param {string} userId The Foundry user ID from whom we are disconnecting * @returns {Promise} A Promise which resolves once the disconnection is complete */ async disconnectPeer(userId) { // Stop audio and video tracks from the remote stream const remoteStream = this.remoteStreams.get(userId); if ( remoteStream ) { this.remoteStreams.delete(userId); for ( let track of remoteStream.getTracks() ) { await track.stop(); } } // Remove the peer const peer = this.peers.get(userId); if ( peer ) { this.peers.delete(userId); await peer.destroy(); } // Re-render the UI on disconnection this.master.render(); } /* -------------------------------------------- */ /** * Disconnect from all current peer streams * @returns {Promise} A Promise which resolves once all peers have been disconnected */ async disconnectAll() { const promises = []; for ( let userId of this.peers.keys() ) { promises.push(this.disconnectPeer(userId)); } return Promise.all(promises); } /* -------------------------------------------- */ /* Settings and Configuration */ /* -------------------------------------------- */ /** @override */ async onSettingsChanged(changed) { const keys = new Set(Object.keys(foundry.utils.flattenObject(changed))); // Change audio or video sources const sourceChange = ["client.videoSrc", "client.audioSrc"].some(k => keys.has(k)); if ( sourceChange ) await this.updateLocalStream(); // Change voice broadcasting mode const modeChange = ["client.voice.mode", `client.users.${game.user.id}.muted`].some(k => keys.has(k)); if ( modeChange ) { const isAlways = this.settings.client.voice.mode === "always"; this.toggleAudio(isAlways && this.master.canUserShareAudio(game.user.id)); this.master.broadcast(isAlways); this.master._initializeUserVoiceDetection(changed.client.voice?.mode); ui.webrtc.setUserIsSpeaking(game.user.id, this.master.broadcasting); } // Re-render the AV camera view const renderChange = ["client.audioSink", "client.muteAll", "client.disableVideo"].some(k => keys.has(k)); if ( sourceChange || renderChange ) this.master.render(); } /* -------------------------------------------- */ /** @inheritdoc */ async updateLocalStream() { const oldStream = this.localStream; await this.initializeLocalStream(); for ( let peer of this.peers.values() ) { if ( oldStream ) peer.removeStream(oldStream); if ( this.localStream ) peer.addStream(this.localStream); } // FIXME: This is a cheat, should be handled elsewhere this.master._initializeUserVoiceDetection(this.settings.client.voice.mode); } }