diff --git a/src/main/java/eu/siacs/conversations/ui/RtpSessionActivity.java b/src/main/java/eu/siacs/conversations/ui/RtpSessionActivity.java index bbd3b2801..2260ec8a7 100644 --- a/src/main/java/eu/siacs/conversations/ui/RtpSessionActivity.java +++ b/src/main/java/eu/siacs/conversations/ui/RtpSessionActivity.java @@ -1,5 +1,8 @@ package eu.siacs.conversations.ui; +import static java.util.Arrays.asList; +import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied; + import android.Manifest; import android.annotation.SuppressLint; import android.app.PictureInPictureParams; @@ -33,6 +36,7 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import org.checkerframework.checker.nullness.compatqual.NullableDecl; +import org.webrtc.RendererCommon; import org.webrtc.SurfaceViewRenderer; import org.webrtc.VideoTrack; @@ -52,6 +56,7 @@ import eu.siacs.conversations.services.AppRTCAudioManager; import eu.siacs.conversations.services.XmppConnectionService; import eu.siacs.conversations.ui.util.AvatarWorkerTask; import eu.siacs.conversations.ui.util.MainThreadExecutor; +import eu.siacs.conversations.ui.util.Rationals; import eu.siacs.conversations.utils.Namespace; import eu.siacs.conversations.utils.PermissionUtils; import eu.siacs.conversations.utils.TimeFrameUtils; @@ -63,10 +68,9 @@ import eu.siacs.conversations.xmpp.jingle.Media; import eu.siacs.conversations.xmpp.jingle.RtpEndUserState; import me.drakeet.support.toast.ToastCompat; -import static eu.siacs.conversations.utils.PermissionUtils.getFirstDenied; -import static java.util.Arrays.asList; - -public class RtpSessionActivity extends XmppActivity implements XmppConnectionService.OnJingleRtpConnectionUpdate { +public class RtpSessionActivity extends XmppActivity + implements XmppConnectionService.OnJingleRtpConnectionUpdate, + eu.siacs.conversations.ui.widget.SurfaceViewRenderer.OnAspectRatioChanged { public static final String EXTRA_WITH = "with"; public static final String EXTRA_SESSION_ID = "session_id"; @@ -78,25 +82,31 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe private static final int CALL_DURATION_UPDATE_INTERVAL = 333; - private boolean shouldAllowBack = false; - - private static final List END_CARD = Arrays.asList( - RtpEndUserState.APPLICATION_ERROR, - RtpEndUserState.SECURITY_ERROR, - RtpEndUserState.DECLINED_OR_BUSY, - RtpEndUserState.CONNECTIVITY_ERROR, - RtpEndUserState.CONNECTIVITY_LOST_ERROR, - RtpEndUserState.RETRACTED - ); - private static final List STATES_SHOWING_HELP_BUTTON = Arrays.asList( - RtpEndUserState.APPLICATION_ERROR, - RtpEndUserState.CONNECTIVITY_ERROR, - RtpEndUserState.SECURITY_ERROR - ); - private static final List STATES_SHOWING_SWITCH_TO_CHAT = Arrays.asList( - RtpEndUserState.CONNECTING, - RtpEndUserState.CONNECTED - ); + private static final List END_CARD = + Arrays.asList( + RtpEndUserState.APPLICATION_ERROR, + RtpEndUserState.SECURITY_ERROR, + RtpEndUserState.DECLINED_OR_BUSY, + RtpEndUserState.CONNECTIVITY_ERROR, + RtpEndUserState.CONNECTIVITY_LOST_ERROR, + RtpEndUserState.RETRACTED); + private static final List STATES_SHOWING_HELP_BUTTON = + Arrays.asList( + RtpEndUserState.APPLICATION_ERROR, + RtpEndUserState.CONNECTIVITY_ERROR, + RtpEndUserState.SECURITY_ERROR); + private static final List STATES_SHOWING_SWITCH_TO_CHAT = + Arrays.asList( + RtpEndUserState.CONNECTING, + RtpEndUserState.CONNECTED, + RtpEndUserState.RECONNECTING); + private static final List STATES_CONSIDERED_CONNECTED = + Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING); + private static final List STATES_SHOWING_PIP_PLACEHOLDER = + Arrays.asList( + RtpEndUserState.ACCEPTING_CALL, + RtpEndUserState.CONNECTING, + RtpEndUserState.RECONNECTING); private static final String PROXIMITY_WAKE_LOCK_TAG = "conversations:in-rtp-session"; private static final int REQUEST_ACCEPT_CALL = 0x1111; private WeakReference rtpConnectionReference; @@ -105,13 +115,14 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe private PowerManager.WakeLock mProximityWakeLock; private final Handler mHandler = new Handler(); - private final Runnable mTickExecutor = new Runnable() { - @Override - public void run() { - updateCallDuration(); - mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL); - } - }; + private final Runnable mTickExecutor = + new Runnable() { + @Override + public void run() { + updateCallDuration(); + mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL); + } + }; private static Set actionToMedia(final String action) { if (ACTION_MAKE_VIDEO_CALL.equals(action)) { @@ -121,21 +132,27 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } } - private static void addSink(final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) { + private static void addSink( + final VideoTrack videoTrack, final SurfaceViewRenderer surfaceViewRenderer) { try { videoTrack.addSink(surfaceViewRenderer); } catch (final IllegalStateException e) { - Log.e(Config.LOGTAG, "possible race condition on trying to display video track. ignoring", e); + Log.e( + Config.LOGTAG, + "possible race condition on trying to display video track. ignoring", + e); } } @Override public void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); - getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON - | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD - | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED - | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON); + getWindow() + .addFlags( + WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON + | WindowManager.LayoutParams.FLAG_DISMISS_KEYGUARD + | WindowManager.LayoutParams.FLAG_SHOW_WHEN_LOCKED + | WindowManager.LayoutParams.FLAG_TURN_SCREEN_ON); this.binding = DataBindingUtil.setContentView(this, R.layout.activity_rtp_session); setSupportActionBar(binding.toolbar); } @@ -155,7 +172,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe return STATES_SHOWING_HELP_BUTTON.contains(requireRtpConnection().getEndUserState()); } catch (IllegalStateException e) { final Intent intent = getIntent(); - final String state = intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null; + final String state = + intent != null ? intent.getStringExtra(EXTRA_LAST_REPORTED_STATE) : null; if (state != null) { return STATES_SHOWING_HELP_BUTTON.contains(RtpEndUserState.valueOf(state)); } else { @@ -165,13 +183,17 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } private boolean isSwitchToConversationVisible() { - final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; - return connection != null && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState()); + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + return connection != null + && STATES_SHOWING_SWITCH_TO_CHAT.contains(connection.getEndUserState()); } private void switchToConversation() { final Contact contact = getWith(); - final Conversation conversation = xmppConnectionService.findOrCreateConversation(contact.getAccount(), contact.getJid(), false, true); + final Conversation conversation = + xmppConnectionService.findOrCreateConversation( + contact.getAccount(), contact.getJid(), false, true); switchToConversation(conversation); } @@ -192,7 +214,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe try { startActivity(intent); } catch (final ActivityNotFoundException e) { - ToastCompat.makeText(this, R.string.no_application_found_to_open_link, ToastCompat.LENGTH_LONG).show(); + ToastCompat.makeText(this, R.string.no_application_found_to_open_link, ToastCompat.LENGTH_LONG) + .show(); } } @@ -215,10 +238,15 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe final Account account = extractAccount(intent); final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH)); final String state = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE); - if (!Intent.ACTION_VIEW.equals(action) || state == null || !END_CARD.contains(RtpEndUserState.valueOf(state))) { - resetIntent(account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction())); + if (!Intent.ACTION_VIEW.equals(action) + || state == null + || !END_CARD.contains(RtpEndUserState.valueOf(state))) { + resetIntent( + account, with, RtpEndUserState.RETRACTED, actionToMedia(intent.getAction())); } - xmppConnectionService.getJingleConnectionManager().retractSessionProposal(account, with.asBareJid()); + xmppConnectionService + .getJingleConnectionManager() + .retractSessionProposal(account, with.asBareJid()); } private void rejectCall(View view) { @@ -233,7 +261,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe private void requestPermissionsAndAcceptCall() { final List permissions; if (getMedia().contains(Media.VIDEO)) { - permissions = ImmutableList.of(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO); + permissions = + ImmutableList.of(Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO); } else { permissions = ImmutableList.of(Manifest.permission.RECORD_AUDIO); } @@ -244,7 +273,7 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } private void checkRecorderAndAcceptCall() { - checkMicrophoneAvailability(); + checkMicrophoneAvailabilityAsync(); try { requireRtpConnection().acceptCall(); } catch (final IllegalStateException e) { @@ -252,6 +281,10 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } } + private void checkMicrophoneAvailabilityAsync() { + new Thread(this::checkMicrophoneAvailability).start(); + } + private void checkMicrophoneAvailability() { new Thread(() -> { final long start = SystemClock.elapsedRealtime(); @@ -262,8 +295,7 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe return; } runOnUiThread(() -> ToastCompat.makeText(this, R.string.microphone_unavailable, ToastCompat.LENGTH_LONG).show()); - } - ).start(); + }).start(); } private void putScreenInCallMode() { @@ -273,9 +305,13 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe private void putScreenInCallMode(final Set media) { getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); if (!media.contains(Media.VIDEO)) { - final JingleRtpConnection rtpConnection = rtpConnectionReference != null ? rtpConnectionReference.get() : null; - final AppRTCAudioManager audioManager = rtpConnection == null ? null : rtpConnection.getAudioManager(); - if (audioManager == null || audioManager.getSelectedAudioDevice() == AppRTCAudioManager.AudioDevice.EARPIECE) { + final JingleRtpConnection rtpConnection = + rtpConnectionReference != null ? rtpConnectionReference.get() : null; + final AppRTCAudioManager audioManager = + rtpConnection == null ? null : rtpConnection.getAudioManager(); + if (audioManager == null + || audioManager.getSelectedAudioDevice() + == AppRTCAudioManager.AudioDevice.EARPIECE) { acquireProximityWakeLock(); } } @@ -288,30 +324,31 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe Log.e(Config.LOGTAG, "power manager not available"); return; } - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - if (this.mProximityWakeLock == null) { - this.mProximityWakeLock = powerManager.newWakeLock(PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG); - } - if (!this.mProximityWakeLock.isHeld()) { - Log.d(Config.LOGTAG, "acquiring proximity wake lock"); - this.mProximityWakeLock.acquire(); - } + if (isFinishing()) { + Log.e(Config.LOGTAG, "do not acquire wakelock. activity is finishing"); + return; + } + if (this.mProximityWakeLock == null) { + this.mProximityWakeLock = + powerManager.newWakeLock( + PowerManager.PROXIMITY_SCREEN_OFF_WAKE_LOCK, PROXIMITY_WAKE_LOCK_TAG); + } + if (!this.mProximityWakeLock.isHeld()) { + Log.d(Config.LOGTAG, "acquiring proximity wake lock"); + this.mProximityWakeLock.acquire(); } } private void releaseProximityWakeLock() { if (this.mProximityWakeLock != null && mProximityWakeLock.isHeld()) { Log.d(Config.LOGTAG, "releasing proximity wake lock"); - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) { - this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY); - } else { - this.mProximityWakeLock.release(); - } + this.mProximityWakeLock.release(PowerManager.RELEASE_FLAG_WAIT_FOR_NO_PROXIMITY); this.mProximityWakeLock = null; } } - private void putProximityWakeLockInProperState(final AppRTCAudioManager.AudioDevice audioDevice) { + private void putProximityWakeLockInProperState( + final AppRTCAudioManager.AudioDevice audioDevice) { if (audioDevice == AppRTCAudioManager.AudioDevice.EARPIECE) { acquireProximityWakeLock(); } else { @@ -320,9 +357,7 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } @Override - protected void refreshUiReal() { - - } + protected void refreshUiReal() {} @Override public void onNewIntent(final Intent intent) { @@ -330,7 +365,9 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe super.onNewIntent(intent); setIntent(intent); if (xmppConnectionService == null) { - Log.d(Config.LOGTAG, "RtpSessionActivity: background service wasn't bound in onNewIntent()"); + Log.d( + Config.LOGTAG, + "RtpSessionActivity: background service wasn't bound in onNewIntent()"); return; } final Account account = extractAccount(intent); @@ -388,7 +425,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe binding.with.setText(account.getRoster().getContact(with).getDisplayName()); } else if (Intent.ACTION_VIEW.equals(action)) { final String extraLastState = intent.getStringExtra(EXTRA_LAST_REPORTED_STATE); - final RtpEndUserState state = extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState); + final RtpEndUserState state = + extraLastState == null ? null : RtpEndUserState.valueOf(extraLastState); if (state != null) { Log.d(Config.LOGTAG, "restored last state from intent extra"); updateButtonConfiguration(state); @@ -398,10 +436,15 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe invalidateOptionsMenu(); } binding.with.setText(account.getRoster().getContact(with).getDisplayName()); - if (xmppConnectionService.getJingleConnectionManager().fireJingleRtpConnectionStateUpdates()) { + if (xmppConnectionService + .getJingleConnectionManager() + .fireJingleRtpConnectionStateUpdates()) { return; } - if (END_CARD.contains(state) || xmppConnectionService.getJingleConnectionManager().hasMatchingProposal(account, with)) { + if (END_CARD.contains(state) + || xmppConnectionService + .getJingleConnectionManager() + .hasMatchingProposal(account, with)) { return; } Log.d(Config.LOGTAG, "restored state (" + state + ") was not an end card. finishing"); @@ -409,12 +452,18 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } } - private void proposeJingleRtpSession(final Account account, final Jid with, final Set media) { - checkMicrophoneAvailability(); + private void proposeJingleRtpSession( + final Account account, final Jid with, final Set media) { + checkMicrophoneAvailabilityAsync(); if (with.isBareJid()) { - xmppConnectionService.getJingleConnectionManager().proposeJingleRtpSession(account, with, media); + xmppConnectionService + .getJingleConnectionManager() + .proposeJingleRtpSession(account, with, media); } else { - final String sessionId = xmppConnectionService.getJingleConnectionManager().initializeRtpSession(account, with, media); + final String sessionId = + xmppConnectionService + .getJingleConnectionManager() + .initializeRtpSession(account, with, media); initializeActivityWithRunningRtpSession(account, with, sessionId); resetIntent(account, with, sessionId); } @@ -422,7 +471,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } @Override - public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { + public void onRequestPermissionsResult( + int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { super.onRequestPermissionsResult(requestCode, permissions, grantResults); if (PermissionUtils.allGranted(grantResults)) { if (requestCode == REQUEST_ACCEPT_CALL) { @@ -438,7 +488,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } else { throw new IllegalStateException("Invalid permission result request"); } - ToastCompat.makeText(this, getString(res, getString(R.string.app_name)), ToastCompat.LENGTH_SHORT).show(); + ToastCompat.makeText(this, getString(res, getString(R.string.app_name)), ToastCompat.LENGTH_SHORT) + .show(); } } @@ -446,15 +497,18 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe public void onStart() { super.onStart(); mHandler.postDelayed(mTickExecutor, CALL_DURATION_UPDATE_INTERVAL); + this.binding.remoteVideo.setOnAspectRatioChanged(this); } @Override public void onStop() { mHandler.removeCallbacks(mTickExecutor); binding.remoteVideo.release(); + binding.remoteVideo.setOnAspectRatioChanged(null); binding.localVideo.release(); final WeakReference weakReference = this.rtpConnectionReference; - final JingleRtpConnection jingleRtpConnection = weakReference == null ? null : weakReference.get(); + final JingleRtpConnection jingleRtpConnection = + weakReference == null ? null : weakReference.get(); if (jingleRtpConnection != null) { releaseVideoTracks(jingleRtpConnection); } @@ -480,10 +534,9 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe return; } } else { - if (shouldAllowBack) { - super.onBackPressed(); - } + endCall(); } + super.onBackPressed(); } @Override @@ -492,15 +545,18 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe if (switchToPictureInPicture()) { return; } - //TODO apparently this method is not getting called on Android 10 when using the task switcher + // TODO apparently this method is not getting called on Android 10 when using the task + // switcher if (emptyReference(rtpConnectionReference) && xmppConnectionService != null) { retractSessionProposal(); } } private boolean isConnected() { - final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; - return connection != null && connection.getEndUserState() == RtpEndUserState.CONNECTED; + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + return connection != null + && STATES_CONSIDERED_CONNECTED.contains(connection.getEndUserState()); } private boolean switchToPictureInPicture() { @@ -516,17 +572,34 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe @RequiresApi(api = Build.VERSION_CODES.O) private void startPictureInPicture() { try { + final Rational rational = this.binding.remoteVideo.getAspectRatio(); + final Rational clippedRational = Rationals.clip(rational); + Log.d( + Config.LOGTAG, + "suggested rational " + rational + ". clipped to " + clippedRational); enterPictureInPictureMode( - new PictureInPictureParams.Builder() - .setAspectRatio(new Rational(10, 16)) - .build() - ); + new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build()); } catch (final IllegalStateException e) { - //this sometimes happens on Samsung phones (possibly when Knox is enabled) + // this sometimes happens on Samsung phones (possibly when Knox is enabled) Log.w(Config.LOGTAG, "unable to enter picture in picture mode", e); } } + @Override + public void onAspectRatioChanged(final Rational rational) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && isPictureInPicture()) { + final Rational clippedRational = Rationals.clip(rational); + Log.d( + Config.LOGTAG, + "suggested rational after aspect ratio change " + + rational + + ". clipped to " + + clippedRational); + setPictureInPictureParams( + new PictureInPictureParams.Builder().setAspectRatio(clippedRational).build()); + } + } + private boolean deviceSupportsPictureInPicture() { if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) { return getPackageManager().hasSystemFeature(PackageManager.FEATURE_PICTURE_IN_PICTURE); @@ -538,24 +611,31 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe private boolean shouldBePictureInPicture() { try { final JingleRtpConnection rtpConnection = requireRtpConnection(); - return rtpConnection.getMedia().contains(Media.VIDEO) && Arrays.asList( + return rtpConnection.getMedia().contains(Media.VIDEO) + && Arrays.asList( RtpEndUserState.ACCEPTING_CALL, RtpEndUserState.CONNECTING, - RtpEndUserState.CONNECTED - ).contains(rtpConnection.getEndUserState()); + RtpEndUserState.CONNECTED) + .contains(rtpConnection.getEndUserState()); } catch (final IllegalStateException e) { return false; } } - private boolean initializeActivityWithRunningRtpSession(final Account account, Jid with, String sessionId) { - final WeakReference reference = xmppConnectionService.getJingleConnectionManager() - .findJingleRtpConnection(account, with, sessionId); + private boolean initializeActivityWithRunningRtpSession( + final Account account, Jid with, String sessionId) { + final WeakReference reference = + xmppConnectionService + .getJingleConnectionManager() + .findJingleRtpConnection(account, with, sessionId); if (reference == null || reference.get() == null) { - final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession = xmppConnectionService - .getJingleConnectionManager().getTerminalSessionState(with, sessionId); + final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession = + xmppConnectionService + .getJingleConnectionManager() + .getTerminalSessionState(with, sessionId); if (terminatedRtpSession == null) { - throw new IllegalStateException("failed to initialize activity with running rtp session. session not found"); + throw new IllegalStateException( + "failed to initialize activity with running rtp session. session not found"); } initializeWithTerminatedSessionState(account, with, terminatedRtpSession); return true; @@ -564,7 +644,6 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe final RtpEndUserState currentState = requireRtpConnection().getEndUserState(); final boolean verified = requireRtpConnection().isVerified(); if (currentState == RtpEndUserState.ENDED) { - reference.get().throwStateTransitionException(); finish(); return true; } @@ -572,7 +651,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe if (currentState == RtpEndUserState.INCOMING_CALL) { getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON); } - if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains(requireRtpConnection().getState())) { + if (JingleRtpConnection.STATES_SHOWING_ONGOING_CALL.contains( + requireRtpConnection().getState())) { putScreenInCallMode(); } binding.with.setText(getWith().getDisplayName()); @@ -585,7 +665,10 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe return false; } - private void initializeWithTerminatedSessionState(final Account account, final Jid with, final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) { + private void initializeWithTerminatedSessionState( + final Account account, + final Jid with, + final JingleConnectionManager.TerminatedRtpSession terminatedRtpSession) { Log.d(Config.LOGTAG, "initializeWithTerminatedSessionState()"); if (terminatedRtpSession.state == RtpEndUserState.ENDED) { finish(); @@ -602,7 +685,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe binding.with.setText(account.getRoster().getContact(with).getDisplayName()); } - private void reInitializeActivityWithRunningRtpSession(final Account account, Jid with, String sessionId) { + private void reInitializeActivityWithRunningRtpSession( + final Account account, Jid with, String sessionId) { runOnUiThread(() -> initializeActivityWithRunningRtpSession(account, with, sessionId)); resetIntent(account, with, sessionId); } @@ -619,8 +703,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe surfaceViewRenderer.setVisibility(View.VISIBLE); try { surfaceViewRenderer.init(requireRtpConnection().getEglBaseContext(), null); - } catch (IllegalStateException e) { - Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized"); + } catch (final IllegalStateException e) { + // Log.d(Config.LOGTAG, "SurfaceViewRenderer was already initialized"); } surfaceViewRenderer.setEnableHardwareScaler(true); } @@ -632,7 +716,6 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe private void updateStateDisplay(final RtpEndUserState state, final Set media) { switch (state) { case INCOMING_CALL: - shouldAllowBack = false; Preconditions.checkArgument(media.size() > 0, "Media must not be empty"); if (media.contains(Media.VIDEO)) { setTitle(R.string.rtp_state_incoming_video_call); @@ -641,56 +724,50 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } break; case CONNECTING: - shouldAllowBack = false; setTitle(R.string.rtp_state_connecting); break; case CONNECTED: - shouldAllowBack = false; setTitle(R.string.rtp_state_connected); break; + case RECONNECTING: + setTitle(R.string.rtp_state_reconnecting); + break; case ACCEPTING_CALL: - shouldAllowBack = false; setTitle(R.string.rtp_state_accepting_call); break; case ENDING_CALL: - shouldAllowBack = false; setTitle(R.string.rtp_state_ending_call); break; case FINDING_DEVICE: - shouldAllowBack = false; setTitle(R.string.rtp_state_finding_device); break; case RINGING: - shouldAllowBack = false; setTitle(R.string.rtp_state_ringing); break; case DECLINED_OR_BUSY: - shouldAllowBack = true; setTitle(R.string.rtp_state_declined_or_busy); break; case CONNECTIVITY_ERROR: - shouldAllowBack = true; setTitle(R.string.rtp_state_connectivity_error); break; case CONNECTIVITY_LOST_ERROR: setTitle(R.string.rtp_state_connectivity_lost_error); break; case RETRACTED: - shouldAllowBack = false; setTitle(R.string.rtp_state_retracted); break; case APPLICATION_ERROR: - shouldAllowBack = true; setTitle(R.string.rtp_state_application_failure); break; case SECURITY_ERROR: setTitle(R.string.rtp_state_security_error); break; case ENDED: - shouldAllowBack = true; - throw new IllegalStateException("Activity should have called finishAndReleaseWakeLock();"); + throw new IllegalStateException( + "Activity should have called finishAndReleaseWakeLock();"); default: - throw new IllegalStateException(String.format("State %s has not been handled in UI", state)); + throw new IllegalStateException( + String.format("State %s has not been handled in UI", state)); } } @@ -712,9 +789,11 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe if (show) { binding.contactPhoto.setVisibility(View.VISIBLE); if (contact == null) { - AvatarWorkerTask.loadAvatar(getWith(), binding.contactPhoto, R.dimen.publish_avatar_size); + AvatarWorkerTask.loadAvatar( + getWith(), binding.contactPhoto, R.dimen.publish_avatar_size); } else { - AvatarWorkerTask.loadAvatar(contact, binding.contactPhoto, R.dimen.publish_avatar_size); + AvatarWorkerTask.loadAvatar( + contact, binding.contactPhoto, R.dimen.publish_avatar_size); } } else { binding.contactPhoto.setVisibility(View.GONE); @@ -763,8 +842,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe RtpEndUserState.CONNECTIVITY_LOST_ERROR, RtpEndUserState.APPLICATION_ERROR, RtpEndUserState.RETRACTED, - RtpEndUserState.SECURITY_ERROR - ).contains(state)) { + RtpEndUserState.SECURITY_ERROR) + .contains(state)) { this.binding.rejectCall.setContentDescription(getString(R.string.exit)); this.binding.rejectCall.setOnClickListener(this::exit); this.binding.rejectCall.setImageResource(R.drawable.ic_clear_white_48dp); @@ -794,26 +873,29 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } private void updateInCallButtonConfiguration() { - updateInCallButtonConfiguration(requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia()); + updateInCallButtonConfiguration( + requireRtpConnection().getEndUserState(), requireRtpConnection().getMedia()); } @SuppressLint("RestrictedApi") - private void updateInCallButtonConfiguration(final RtpEndUserState state, final Set media) { - if (state == RtpEndUserState.CONNECTED && !isPictureInPicture()) { + private void updateInCallButtonConfiguration( + final RtpEndUserState state, final Set media) { + if (STATES_CONSIDERED_CONNECTED.contains(state) && !isPictureInPicture()) { Preconditions.checkArgument(media.size() > 0, "Media must not be empty"); if (media.contains(Media.VIDEO)) { final JingleRtpConnection rtpConnection = requireRtpConnection(); - updateInCallButtonConfigurationVideo(rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable()); + updateInCallButtonConfigurationVideo( + rtpConnection.isVideoEnabled(), rtpConnection.isCameraSwitchable()); } else { final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager(); updateInCallButtonConfigurationSpeaker( audioManager.getSelectedAudioDevice(), - audioManager.getAudioDevices().size() - ); + audioManager.getAudioDevices().size()); this.binding.inCallActionFarRight.setVisibility(View.GONE); } if (media.contains(Media.AUDIO)) { - updateInCallButtonConfigurationMicrophone(requireRtpConnection().isMicrophoneEnabled()); + updateInCallButtonConfigurationMicrophone( + requireRtpConnection().isMicrophoneEnabled()); } else { this.binding.inCallActionLeft.setVisibility(View.GONE); } @@ -825,10 +907,12 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } @SuppressLint("RestrictedApi") - private void updateInCallButtonConfigurationSpeaker(final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) { + private void updateInCallButtonConfigurationSpeaker( + final AppRTCAudioManager.AudioDevice selectedAudioDevice, final int numberOfChoices) { switch (selectedAudioDevice) { case EARPIECE: - this.binding.inCallActionRight.setImageResource(R.drawable.ic_volume_off_black_24dp); + this.binding.inCallActionRight.setImageResource( + R.drawable.ic_volume_off_black_24dp); if (numberOfChoices >= 2) { this.binding.inCallActionRight.setOnClickListener(this::switchToSpeaker); } else { @@ -851,7 +935,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } break; case BLUETOOTH: - this.binding.inCallActionRight.setImageResource(R.drawable.ic_bluetooth_audio_black_24dp); + this.binding.inCallActionRight.setImageResource( + R.drawable.ic_bluetooth_audio_black_24dp); this.binding.inCallActionRight.setOnClickListener(null); this.binding.inCallActionRight.setClickable(false); break; @@ -860,10 +945,12 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } @SuppressLint("RestrictedApi") - private void updateInCallButtonConfigurationVideo(final boolean videoEnabled, final boolean isCameraSwitchable) { + private void updateInCallButtonConfigurationVideo( + final boolean videoEnabled, final boolean isCameraSwitchable) { this.binding.inCallActionRight.setVisibility(View.VISIBLE); if (isCameraSwitchable) { - this.binding.inCallActionFarRight.setImageResource(R.drawable.ic_flip_camera_android_black_24dp); + this.binding.inCallActionFarRight.setImageResource( + R.drawable.ic_flip_camera_android_black_24dp); this.binding.inCallActionFarRight.setVisibility(View.VISIBLE); this.binding.inCallActionFarRight.setOnClickListener(this::switchCamera); } else { @@ -879,18 +966,28 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } private void switchCamera(final View view) { - Futures.addCallback(requireRtpConnection().switchCamera(), new FutureCallback() { - @Override - public void onSuccess(@NullableDecl Boolean isFrontCamera) { - binding.localVideo.setMirror(isFrontCamera); - } + Futures.addCallback( + requireRtpConnection().switchCamera(), + new FutureCallback() { + @Override + public void onSuccess(@NullableDecl Boolean isFrontCamera) { + binding.localVideo.setMirror(isFrontCamera); + } - @Override - public void onFailure(@NonNull final Throwable throwable) { - Log.d(Config.LOGTAG, "could not switch camera", Throwables.getRootCause(throwable)); - ToastCompat.makeText(RtpSessionActivity.this, R.string.could_not_switch_camera, ToastCompat.LENGTH_LONG).show(); - } - }, MainThreadExecutor.getInstance()); + @Override + public void onFailure(@NonNull final Throwable throwable) { + Log.d( + Config.LOGTAG, + "could not switch camera", + Throwables.getRootCause(throwable)); + ToastCompat.makeText( + RtpSessionActivity.this, + R.string.could_not_switch_camera, + ToastCompat.LENGTH_LONG) + .show(); + } + }, + MainThreadExecutor.getInstance()); } private void enableVideo(View view) { @@ -906,7 +1003,6 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe private void disableVideo(View view) { requireRtpConnection().setVideoEnabled(false); updateInCallButtonConfigurationVideo(false, requireRtpConnection().isCameraSwitchable()); - } @SuppressLint("RestrictedApi") @@ -922,19 +1018,18 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } private void updateCallDuration() { - final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; if (connection == null || connection.getMedia().contains(Media.VIDEO)) { this.binding.duration.setVisibility(View.GONE); return; } - final long rtpConnectionStarted = connection.getRtpConnectionStarted(); - final long rtpConnectionEnded = connection.getRtpConnectionEnded(); - if (rtpConnectionStarted != 0) { - final long ended = rtpConnectionEnded == 0 ? SystemClock.elapsedRealtime() : rtpConnectionEnded; - this.binding.duration.setText(TimeFrameUtils.formatTimePassed(rtpConnectionStarted, ended, false)); - this.binding.duration.setVisibility(View.VISIBLE); - } else { + if (connection.zeroDuration()) { this.binding.duration.setVisibility(View.GONE); + } else { + this.binding.duration.setText( + TimeFrameUtils.formatElapsedTime(connection.getCallDuration(), false)); + this.binding.duration.setVisibility(View.VISIBLE); } } @@ -942,17 +1037,17 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe if (END_CARD.contains(state) || state == RtpEndUserState.ENDING_CALL) { binding.localVideo.setVisibility(View.GONE); binding.localVideo.release(); - binding.remoteVideo.setVisibility(View.GONE); + binding.remoteVideoWrapper.setVisibility(View.GONE); binding.remoteVideo.release(); binding.pipLocalMicOffIndicator.setVisibility(View.GONE); if (isPictureInPicture()) { + binding.appBarLayout.setVisibility(View.GONE); + binding.pipPlaceholder.setVisibility(View.VISIBLE); if (Arrays.asList( RtpEndUserState.APPLICATION_ERROR, RtpEndUserState.CONNECTIVITY_ERROR, RtpEndUserState.SECURITY_ERROR) .contains(state)) { - binding.appBarLayout.setVisibility(View.GONE); - binding.pipPlaceholder.setVisibility(View.VISIBLE); binding.pipWarning.setVisibility(View.VISIBLE); binding.pipWaiting.setVisibility(View.GONE); } else { @@ -966,9 +1061,9 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN); return; } - if (isPictureInPicture() && (state == RtpEndUserState.CONNECTING || state == RtpEndUserState.ACCEPTING_CALL)) { + if (isPictureInPicture() && STATES_SHOWING_PIP_PLACEHOLDER.contains(state)) { binding.localVideo.setVisibility(View.GONE); - binding.remoteVideo.setVisibility(View.GONE); + binding.remoteVideoWrapper.setVisibility(View.GONE); binding.appBarLayout.setVisibility(View.GONE); binding.pipPlaceholder.setVisibility(View.VISIBLE); binding.pipWarning.setVisibility(View.GONE); @@ -979,7 +1074,7 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe final Optional localVideoTrack = getLocalVideoTrack(); if (localVideoTrack.isPresent() && !isPictureInPicture()) { ensureSurfaceViewRendererIsSetup(binding.localVideo); - //paint local view over remote view + // paint local view over remote view binding.localVideo.setZOrderMediaOverlay(true); binding.localVideo.setMirror(requireRtpConnection().isFrontCamera()); addSink(localVideoTrack.get(), binding.localVideo); @@ -990,12 +1085,17 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe if (remoteVideoTrack.isPresent()) { ensureSurfaceViewRendererIsSetup(binding.remoteVideo); addSink(remoteVideoTrack.get(), binding.remoteVideo); + binding.remoteVideo.setScalingType( + RendererCommon.ScalingType.SCALE_ASPECT_FILL, + RendererCommon.ScalingType.SCALE_ASPECT_FIT); if (state == RtpEndUserState.CONNECTED) { binding.appBarLayout.setVisibility(View.GONE); getWindow().addFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN); + binding.remoteVideoWrapper.setVisibility(View.VISIBLE); } else { + binding.appBarLayout.setVisibility(View.VISIBLE); getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN); - binding.remoteVideo.setVisibility(View.GONE); + binding.remoteVideoWrapper.setVisibility(View.GONE); } if (isPictureInPicture() && !requireRtpConnection().isMicrophoneEnabled()) { binding.pipLocalMicOffIndicator.setVisibility(View.VISIBLE); @@ -1004,13 +1104,14 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } } else { getWindow().clearFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN); - binding.remoteVideo.setVisibility(View.GONE); + binding.remoteVideoWrapper.setVisibility(View.GONE); binding.pipLocalMicOffIndicator.setVisibility(View.GONE); } } private Optional getLocalVideoTrack() { - final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; if (connection == null) { return Optional.absent(); } @@ -1018,7 +1119,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } private Optional getRemoteVideoTrack() { - final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; if (connection == null) { return Optional.absent(); } @@ -1040,12 +1142,16 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } private void switchToEarpiece(View view) { - requireRtpConnection().getAudioManager().setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE); + requireRtpConnection() + .getAudioManager() + .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.EARPIECE); acquireProximityWakeLock(); } private void switchToSpeaker(View view) { - requireRtpConnection().getAudioManager().setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE); + requireRtpConnection() + .getAudioManager() + .setDefaultAudioDevice(AppRTCAudioManager.AudioDevice.SPEAKER_PHONE); releaseProximityWakeLock(); } @@ -1069,12 +1175,15 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe final Intent intent = getIntent(); final Account account = extractAccount(intent); final Jid with = Jid.ofEscaped(intent.getStringExtra(EXTRA_WITH)); - final Conversation conversation = xmppConnectionService.findOrCreateConversation(account, with, false, true); + final Conversation conversation = + xmppConnectionService.findOrCreateConversation(account, with, false, true); final Intent launchIntent = new Intent(this, ConversationsActivity.class); launchIntent.setAction(ConversationsActivity.ACTION_VIEW_CONVERSATION); launchIntent.putExtra(ConversationsActivity.EXTRA_CONVERSATION, conversation.getUuid()); launchIntent.setFlags(intent.getFlags() | Intent.FLAG_ACTIVITY_CLEAR_TOP); - launchIntent.putExtra(ConversationsActivity.EXTRA_POST_INIT_ACTION, ConversationsActivity.POST_ACTION_RECORD_VOICE); + launchIntent.putExtra( + ConversationsActivity.EXTRA_POST_INIT_ACTION, + ConversationsActivity.POST_ACTION_RECORD_VOICE); startActivity(launchIntent); finish(); } @@ -1086,7 +1195,8 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } private JingleRtpConnection requireRtpConnection() { - final JingleRtpConnection connection = this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; + final JingleRtpConnection connection = + this.rtpConnectionReference != null ? this.rtpConnectionReference.get() : null; if (connection == null) { throw new IllegalStateException("No RTP connection found"); } @@ -1094,12 +1204,14 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } @Override - public void onJingleRtpConnectionUpdate(Account account, Jid with, final String sessionId, RtpEndUserState state) { + public void onJingleRtpConnectionUpdate( + Account account, Jid with, final String sessionId, RtpEndUserState state) { Log.d(Config.LOGTAG, "onJingleRtpConnectionUpdate(" + state + ")"); if (END_CARD.contains(state)) { Log.d(Config.LOGTAG, "end card reached"); releaseProximityWakeLock(); - runOnUiThread(() -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)); + runOnUiThread( + () -> getWindow().clearFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON)); } if (with.isBareJid()) { updateRtpSessionProposalState(account, with, state); @@ -1110,7 +1222,7 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe Log.d(Config.LOGTAG, "not reinitializing session"); return; } - //this happens when going from proposed session to actual session + // this happens when going from proposed session to actual session reInitializeActivityWithRunningRtpSession(account, with, sessionId); return; } @@ -1123,14 +1235,16 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe finish(); return; } - runOnUiThread(() -> { - updateStateDisplay(state, media); - updateVerifiedShield(verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state)); - updateButtonConfiguration(state, media); - updateVideoViews(state); - updateProfilePicture(state, contact); - invalidateOptionsMenu(); - }); + runOnUiThread( + () -> { + updateStateDisplay(state, media); + updateVerifiedShield( + verified && STATES_SHOWING_SWITCH_TO_CHAT.contains(state)); + updateButtonConfiguration(state, media); + updateVideoViews(state); + updateProfilePicture(state, contact); + invalidateOptionsMenu(); + }); if (END_CARD.contains(state)) { final JingleRtpConnection rtpConnection = requireRtpConnection(); resetIntent(account, with, state, rtpConnection.getMedia()); @@ -1143,8 +1257,15 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } @Override - public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set availableAudioDevices) { - Log.d(Config.LOGTAG, "onAudioDeviceChanged in activity: selected:" + selectedAudioDevice + ", available:" + availableAudioDevices); + public void onAudioDeviceChanged( + AppRTCAudioManager.AudioDevice selectedAudioDevice, + Set availableAudioDevices) { + Log.d( + Config.LOGTAG, + "onAudioDeviceChanged in activity: selected:" + + selectedAudioDevice + + ", available:" + + availableAudioDevices); try { if (getMedia().contains(Media.VIDEO)) { Log.d(Config.LOGTAG, "nothing to do; in video mode"); @@ -1155,10 +1276,11 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe final AppRTCAudioManager audioManager = requireRtpConnection().getAudioManager(); updateInCallButtonConfigurationSpeaker( audioManager.getSelectedAudioDevice(), - audioManager.getAudioDevices().size() - ); + audioManager.getAudioDevices().size()); } else if (END_CARD.contains(endUserState)) { - Log.d(Config.LOGTAG, "onAudioDeviceChanged() nothing to do because end card has been reached"); + Log.d( + Config.LOGTAG, + "onAudioDeviceChanged() nothing to do because end card has been reached"); } else { putProximityWakeLockInProperState(selectedAudioDevice); } @@ -1167,20 +1289,23 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe } } - private void updateRtpSessionProposalState(final Account account, final Jid with, final RtpEndUserState state) { + private void updateRtpSessionProposalState( + final Account account, final Jid with, final RtpEndUserState state) { final Intent currentIntent = getIntent(); - final String withExtra = currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH); + final String withExtra = + currentIntent == null ? null : currentIntent.getStringExtra(EXTRA_WITH); if (withExtra == null) { return; } if (Jid.ofEscaped(withExtra).asBareJid().equals(with)) { - runOnUiThread(() -> { - updateVerifiedShield(false); - updateStateDisplay(state); - updateButtonConfiguration(state); - updateProfilePicture(state); - invalidateOptionsMenu(); - }); + runOnUiThread( + () -> { + updateVerifiedShield(false); + updateStateDisplay(state); + updateButtonConfiguration(state); + updateProfilePicture(state); + invalidateOptionsMenu(); + }); resetIntent(account, with, state, actionToMedia(currentIntent.getAction())); } } @@ -1191,16 +1316,22 @@ public class RtpSessionActivity extends XmppActivity implements XmppConnectionSe setIntent(intent); } - private void resetIntent(final Account account, Jid with, final RtpEndUserState state, final Set media) { + private void resetIntent( + final Account account, Jid with, final RtpEndUserState state, final Set media) { final Intent intent = new Intent(Intent.ACTION_VIEW); intent.putExtra(EXTRA_ACCOUNT, account.getJid().toEscapedString()); - if (account.getRoster().getContact(with).getPresences().anySupport(Namespace.JINGLE_MESSAGE)) { + if (account.getRoster() + .getContact(with) + .getPresences() + .anySupport(Namespace.JINGLE_MESSAGE)) { intent.putExtra(EXTRA_WITH, with.asBareJid().toEscapedString()); } else { intent.putExtra(EXTRA_WITH, with.toEscapedString()); } intent.putExtra(EXTRA_LAST_REPORTED_STATE, state.toString()); - intent.putExtra(EXTRA_LAST_ACTION, media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL); + intent.putExtra( + EXTRA_LAST_ACTION, + media.contains(Media.VIDEO) ? ACTION_MAKE_VIDEO_CALL : ACTION_MAKE_VOICE_CALL); setIntent(intent); } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java index 91523c62b..605da3079 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -1,10 +1,13 @@ package eu.siacs.conversations.xmpp.jingle; -import android.os.SystemClock; import android.util.Log; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import com.google.common.base.Stopwatch; import com.google.common.base.Strings; import com.google.common.base.Throwables; import com.google.common.collect.Collections2; @@ -12,20 +15,25 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; +import com.google.common.util.concurrent.FutureCallback; +import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; +import com.google.common.util.concurrent.MoreExecutors; import org.webrtc.EglBase; import org.webrtc.IceCandidate; import org.webrtc.PeerConnection; import org.webrtc.VideoTrack; -import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.Queue; import java.util.Set; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ScheduledFuture; import java.util.concurrent.TimeUnit; @@ -53,112 +61,123 @@ import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription; import eu.siacs.conversations.xmpp.stanzas.IqPacket; import eu.siacs.conversations.xmpp.stanzas.MessagePacket; -public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback { +public class JingleRtpConnection extends AbstractJingleConnection + implements WebRTCWrapper.EventCallback { - public static final List STATES_SHOWING_ONGOING_CALL = Arrays.asList( - State.PROCEED, - State.SESSION_INITIALIZED_PRE_APPROVED, - State.SESSION_ACCEPTED - ); + public static final List STATES_SHOWING_ONGOING_CALL = + Arrays.asList( + State.PROCEED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED); private static final long BUSY_TIME_OUT = 30; - private static final List TERMINATED = Arrays.asList( - State.ACCEPTED, - State.REJECTED, - State.REJECTED_RACED, - State.RETRACTED, - State.RETRACTED_RACED, - State.TERMINATED_SUCCESS, - State.TERMINATED_DECLINED_OR_BUSY, - State.TERMINATED_CONNECTIVITY_ERROR, - State.TERMINATED_CANCEL_OR_TIMEOUT, - State.TERMINATED_APPLICATION_FAILURE, - State.TERMINATED_SECURITY_ERROR - ); + private static final List TERMINATED = + Arrays.asList( + State.ACCEPTED, + State.REJECTED, + State.REJECTED_RACED, + State.RETRACTED, + State.RETRACTED_RACED, + State.TERMINATED_SUCCESS, + State.TERMINATED_DECLINED_OR_BUSY, + State.TERMINATED_CONNECTIVITY_ERROR, + State.TERMINATED_CANCEL_OR_TIMEOUT, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR); private static final Map> VALID_TRANSITIONS; static { - final ImmutableMap.Builder> transitionBuilder = new ImmutableMap.Builder<>(); - transitionBuilder.put(State.NULL, ImmutableList.of( + final ImmutableMap.Builder> transitionBuilder = + new ImmutableMap.Builder<>(); + transitionBuilder.put( + State.NULL, + ImmutableList.of( + State.PROPOSED, + State.SESSION_INITIALIZED, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR)); + transitionBuilder.put( State.PROPOSED, - State.SESSION_INITIALIZED, - State.TERMINATED_APPLICATION_FAILURE, - State.TERMINATED_SECURITY_ERROR - )); - transitionBuilder.put(State.PROPOSED, ImmutableList.of( - State.ACCEPTED, + ImmutableList.of( + State.ACCEPTED, + State.PROCEED, + State.REJECTED, + State.RETRACTED, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR, + State.TERMINATED_CONNECTIVITY_ERROR // only used when the xmpp connection + // rebinds + )); + transitionBuilder.put( State.PROCEED, - State.REJECTED, - State.RETRACTED, - State.TERMINATED_APPLICATION_FAILURE, - State.TERMINATED_SECURITY_ERROR, - State.TERMINATED_CONNECTIVITY_ERROR //only used when the xmpp connection rebinds - )); - transitionBuilder.put(State.PROCEED, ImmutableList.of( - State.REJECTED_RACED, - State.RETRACTED_RACED, + ImmutableList.of( + State.REJECTED_RACED, + State.RETRACTED_RACED, + State.SESSION_INITIALIZED_PRE_APPROVED, + State.TERMINATED_SUCCESS, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR, + State.TERMINATED_CONNECTIVITY_ERROR // at this state used for error + // bounces of the proceed message + )); + transitionBuilder.put( + State.SESSION_INITIALIZED, + ImmutableList.of( + State.SESSION_ACCEPTED, + State.TERMINATED_SUCCESS, + State.TERMINATED_DECLINED_OR_BUSY, + State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors + // and IQ timeouts + State.TERMINATED_CANCEL_OR_TIMEOUT, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR)); + transitionBuilder.put( State.SESSION_INITIALIZED_PRE_APPROVED, - State.TERMINATED_SUCCESS, - State.TERMINATED_APPLICATION_FAILURE, - State.TERMINATED_SECURITY_ERROR, - State.TERMINATED_CONNECTIVITY_ERROR //at this state used for error bounces of the proceed message - )); - transitionBuilder.put(State.SESSION_INITIALIZED, ImmutableList.of( + ImmutableList.of( + State.SESSION_ACCEPTED, + State.TERMINATED_SUCCESS, + State.TERMINATED_DECLINED_OR_BUSY, + State.TERMINATED_CONNECTIVITY_ERROR, // at this state used for IQ errors + // and IQ timeouts + State.TERMINATED_CANCEL_OR_TIMEOUT, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR)); + transitionBuilder.put( State.SESSION_ACCEPTED, - State.TERMINATED_SUCCESS, - State.TERMINATED_DECLINED_OR_BUSY, - State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts - State.TERMINATED_CANCEL_OR_TIMEOUT, - State.TERMINATED_APPLICATION_FAILURE, - State.TERMINATED_SECURITY_ERROR - )); - transitionBuilder.put(State.SESSION_INITIALIZED_PRE_APPROVED, ImmutableList.of( - State.SESSION_ACCEPTED, - State.TERMINATED_SUCCESS, - State.TERMINATED_DECLINED_OR_BUSY, - State.TERMINATED_CONNECTIVITY_ERROR, //at this state used for IQ errors and IQ timeouts - State.TERMINATED_CANCEL_OR_TIMEOUT, - State.TERMINATED_APPLICATION_FAILURE, - State.TERMINATED_SECURITY_ERROR - )); - transitionBuilder.put(State.SESSION_ACCEPTED, ImmutableList.of( - State.TERMINATED_SUCCESS, - State.TERMINATED_DECLINED_OR_BUSY, - State.TERMINATED_CONNECTIVITY_ERROR, - State.TERMINATED_CANCEL_OR_TIMEOUT, - State.TERMINATED_APPLICATION_FAILURE, - State.TERMINATED_SECURITY_ERROR - )); + ImmutableList.of( + State.TERMINATED_SUCCESS, + State.TERMINATED_DECLINED_OR_BUSY, + State.TERMINATED_CONNECTIVITY_ERROR, + State.TERMINATED_CANCEL_OR_TIMEOUT, + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_SECURITY_ERROR)); VALID_TRANSITIONS = transitionBuilder.build(); } private final WebRTCWrapper webRTCWrapper = new WebRTCWrapper(this); - private final ArrayDeque>> pendingIceCandidates = new ArrayDeque<>(); + private final Queue> + pendingIceCandidates = new LinkedList<>(); private final OmemoVerification omemoVerification = new OmemoVerification(); private final Message message; private State state = State.NULL; - private StateTransitionException stateTransitionException; private Set proposedMedia; private RtpContentMap initiatorRtpContentMap; private RtpContentMap responderRtpContentMap; - private long rtpConnectionStarted = 0; //time of 'connected' - private long rtpConnectionEnded = 0; + private IceUdpTransportInfo.Setup peerDtlsSetup; + private final Stopwatch sessionDuration = Stopwatch.createUnstarted(); + private final Queue stateHistory = new LinkedList<>(); private ScheduledFuture ringingTimeoutFuture; JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) { super(jingleConnectionManager, id, initiator); - final Conversation conversation = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation( - id.account, - id.with.asBareJid(), - false, - false - ); - this.message = new Message( - conversation, - isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED, - Message.TYPE_RTP_SESSION, - id.sessionId - ); + final Conversation conversation = + jingleConnectionManager + .getXmppConnectionService() + .findOrCreateConversation(id.account, id.with.asBareJid(), false, false); + this.message = + new Message( + conversation, + isInitiator() ? Message.STATUS_SEND : Message.STATUS_RECEIVED, + Message.TYPE_RTP_SESSION, + id.sessionId); } private static State reasonToState(Reason reason) { @@ -184,7 +203,6 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web @Override synchronized void deliverPacket(final JinglePacket jinglePacket) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": packet delivered to JingleRtpConnection"); switch (jinglePacket.getAction()) { case SESSION_INITIATE: receiveSessionInitiate(jinglePacket); @@ -200,7 +218,11 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web break; default: respondOk(jinglePacket); - Log.d(Config.LOGTAG, String.format("%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction())); + Log.d( + Config.LOGTAG, + String.format( + "%s: received unhandled jingle action %s", + id.account.getJid().asBareJid(), jinglePacket.getAction())); break; } } @@ -214,8 +236,12 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web if (!isInitiator() && isInState(State.PROPOSED, State.SESSION_INITIALIZED)) { xmppConnectionService.getNotificationService().cancelIncomingCallNotification(); } - if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) { - //we might have already changed resources (full jid) at this point; so this might not even reach the other party + if (isInState( + State.SESSION_INITIALIZED, + State.SESSION_INITIALIZED_PRE_APPROVED, + State.SESSION_ACCEPTED)) { + // we might have already changed resources (full jid) at this point; so this might not + // even reach the other party sendSessionTerminate(Reason.CONNECTIVITY_ERROR); } else { transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR); @@ -227,9 +253,21 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web respondOk(jinglePacket); final JinglePacket.ReasonWrapper wrapper = jinglePacket.getReason(); final State previous = this.state; - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session terminate reason=" + wrapper.reason + "(" + Strings.nullToEmpty(wrapper.text) + ") while in state " + previous); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": received session terminate reason=" + + wrapper.reason + + "(" + + Strings.nullToEmpty(wrapper.text) + + ") while in state " + + previous); if (TERMINATED.contains(previous)) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring session terminate because already in " + previous); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": ignoring session terminate because already in " + + previous); return; } webRTCWrapper.close(); @@ -243,127 +281,338 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } private void receiveTransportInfo(final JinglePacket jinglePacket) { - if (isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED, State.SESSION_ACCEPTED)) { - respondOk(jinglePacket); + // Due to the asynchronicity of processing session-init we might move from NULL|PROCEED to + // INITIALIZED only after transport-info has been received + if (isInState( + State.NULL, + State.PROCEED, + State.SESSION_INITIALIZED, + State.SESSION_INITIALIZED_PRE_APPROVED, + State.SESSION_ACCEPTED)) { final RtpContentMap contentMap; try { contentMap = RtpContentMap.of(jinglePacket); - } catch (IllegalArgumentException | NullPointerException e) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents; ignoring", e); + } catch (final IllegalArgumentException | NullPointerException e) { + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": improperly formatted contents; ignoring", + e); + respondOk(jinglePacket); return; } - final Set> candidates = contentMap.contents.entrySet(); - if (this.state == State.SESSION_ACCEPTED) { - try { - processCandidates(candidates); - } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) { - Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored"); - } - } else { - pendingIceCandidates.push(candidates); - } + receiveTransportInfo(jinglePacket, contentMap); } else { if (isTerminated()) { respondOk(jinglePacket); - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring out-of-order transport info; we where already terminated"); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": ignoring out-of-order transport info; we where already terminated"); } else { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received transport info while in state=" + this.state); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": received transport info while in state=" + + this.state); terminateWithOutOfOrder(jinglePacket); } } } - private void processCandidates(final Set> contents) { - final RtpContentMap rtpContentMap = isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap; - final Group originalGroup = rtpContentMap.group; - final List identificationTags = originalGroup == null ? rtpContentMap.getNames() : originalGroup.getIdentificationTags(); - if (identificationTags.size() == 0) { - Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices"); - } - processCandidates(identificationTags, contents); - } - - private void processCandidates(final List indices, final Set> contents) { - for (final Map.Entry content : contents) { - final String ufrag = content.getValue().transport.getAttribute("ufrag"); - for (final IceUdpTransportInfo.Candidate candidate : content.getValue().transport.getCandidates()) { - final String sdp; - try { - sdp = candidate.toSdpAttribute(ufrag); - } catch (IllegalArgumentException e) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring invalid ICE candidate " + e.getMessage()); - continue; - } - final String sdpMid = content.getKey(); - final int mLineIndex = indices.indexOf(sdpMid); - if (mLineIndex < 0) { - Log.w(Config.LOGTAG, "mLineIndex not found for " + sdpMid + ". available indices " + indices); - } - final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp); - Log.d(Config.LOGTAG, "received candidate: " + iceCandidate); - this.webRTCWrapper.addIceCandidate(iceCandidate); + private void receiveTransportInfo( + final JinglePacket jinglePacket, final RtpContentMap contentMap) { + final Set> candidates = + contentMap.contents.entrySet(); + if (this.state == State.SESSION_ACCEPTED) { + // zero candidates + modified credentials are an ICE restart offer + if (checkForIceRestart(jinglePacket, contentMap)) { + return; } - } - } - - private RtpContentMap receiveRtpContentMap(final JinglePacket jinglePacket, final boolean expectVerification) { - final RtpContentMap receivedContentMap = RtpContentMap.of(jinglePacket); - if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) { - final AxolotlService.OmemoVerifiedPayload omemoVerifiedPayload; + respondOk(jinglePacket); try { - omemoVerifiedPayload = id.account.getAxolotlService().decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with); - } catch (final CryptoFailedException e) { - throw new SecurityException("Unable to verify DTLS Fingerprint with OMEMO", e); + processCandidates(candidates); + } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) { + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": PeerConnection was not initialized when processing transport info. this usually indicates a race condition that can be ignored"); } - this.omemoVerification.setOrEnsureEqual(omemoVerifiedPayload); - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received verifiable DTLS fingerprint via " + this.omemoVerification); - return omemoVerifiedPayload.getPayload(); - } else if (expectVerification) { - throw new SecurityException("DTLS fingerprint was unexpectedly not verifiable"); } else { - return receivedContentMap; + respondOk(jinglePacket); + pendingIceCandidates.addAll(candidates); + } + } + + private boolean checkForIceRestart( + final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) { + final RtpContentMap existing = getRemoteContentMap(); + final Set existingCredentials; + final IceUdpTransportInfo.Credentials newCredentials; + try { + existingCredentials = existing.getCredentials(); + newCredentials = rtpContentMap.getDistinctCredentials(); + } catch (final IllegalStateException e) { + Log.d(Config.LOGTAG, "unable to gather credentials for comparison", e); + return false; + } + if (existingCredentials.contains(newCredentials)) { + return false; + } + // TODO an alternative approach is to check if we already got an iq result to our + // ICE-restart + // and if that's the case we are seeing an answer. + // This might be more spec compliant but also more error prone potentially + final boolean isOffer = rtpContentMap.emptyCandidates(); + final RtpContentMap restartContentMap; + try { + if (isOffer) { + Log.d(Config.LOGTAG, "received offer to restart ICE " + newCredentials); + restartContentMap = + existing.modifiedCredentials( + newCredentials, IceUdpTransportInfo.Setup.ACTPASS); + } else { + final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup(); + Log.d( + Config.LOGTAG, + "received confirmation of ICE restart" + + newCredentials + + " peer_setup=" + + setup); + // DTLS setup attribute needs to be rewritten to reflect current peer state + // https://groups.google.com/g/discuss-webrtc/c/DfpIMwvUfeM + restartContentMap = existing.modifiedCredentials(newCredentials, setup); + } + if (applyIceRestart(jinglePacket, restartContentMap, isOffer)) { + return isOffer; + } else { + Log.d(Config.LOGTAG, "ignoring ICE restart. sending tie-break"); + respondWithTieBreak(jinglePacket); + return true; + } + } catch (final Exception exception) { + respondOk(jinglePacket); + final Throwable rootCause = Throwables.getRootCause(exception); + if (rootCause instanceof WebRTCWrapper.PeerConnectionNotInitialized) { + // If this happens a termination is already in progress + Log.d(Config.LOGTAG, "ignoring PeerConnectionNotInitialized on ICE restart"); + return true; + } + Log.d(Config.LOGTAG, "failure to apply ICE restart", rootCause); + webRTCWrapper.close(); + sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage()); + return true; + } + } + + private IceUdpTransportInfo.Setup getPeerDtlsSetup() { + final IceUdpTransportInfo.Setup peerSetup = this.peerDtlsSetup; + if (peerSetup == null || peerSetup == IceUdpTransportInfo.Setup.ACTPASS) { + throw new IllegalStateException("Invalid peer setup"); + } + return peerSetup; + } + + private void storePeerDtlsSetup(final IceUdpTransportInfo.Setup setup) { + if (setup == null || setup == IceUdpTransportInfo.Setup.ACTPASS) { + throw new IllegalArgumentException("Trying to store invalid peer dtls setup"); + } + this.peerDtlsSetup = setup; + } + + private boolean applyIceRestart( + final JinglePacket jinglePacket, + final RtpContentMap restartContentMap, + final boolean isOffer) + throws ExecutionException, InterruptedException { + final SessionDescription sessionDescription = SessionDescription.of(restartContentMap); + final org.webrtc.SessionDescription.Type type = + isOffer + ? org.webrtc.SessionDescription.Type.OFFER + : org.webrtc.SessionDescription.Type.ANSWER; + org.webrtc.SessionDescription sdp = + new org.webrtc.SessionDescription(type, sessionDescription.toString()); + if (isOffer && webRTCWrapper.getSignalingState() != PeerConnection.SignalingState.STABLE) { + if (isInitiator()) { + // We ignore the offer and respond with tie-break. This will clause the responder + // not to apply the content map + return false; + } + } + webRTCWrapper.setRemoteDescription(sdp).get(); + setRemoteContentMap(restartContentMap); + if (isOffer) { + webRTCWrapper.setIsReadyToReceiveIceCandidates(false); + final SessionDescription localSessionDescription = setLocalSessionDescription(); + setLocalContentMap(RtpContentMap.of(localSessionDescription)); + // We need to respond OK before sending any candidates + respondOk(jinglePacket); + webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + } else { + storePeerDtlsSetup(restartContentMap.getDtlsSetup()); + } + return true; + } + + private void processCandidates( + final Set> contents) { + for (final Map.Entry content : contents) { + processCandidate(content); + } + } + + private void processCandidate( + final Map.Entry content) { + final RtpContentMap rtpContentMap = getRemoteContentMap(); + final List indices = toIdentificationTags(rtpContentMap); + final String sdpMid = content.getKey(); // aka content name + final IceUdpTransportInfo transport = content.getValue().transport; + final IceUdpTransportInfo.Credentials credentials = transport.getCredentials(); + + // TODO check that credentials remained the same + + for (final IceUdpTransportInfo.Candidate candidate : transport.getCandidates()) { + final String sdp; + try { + sdp = candidate.toSdpAttribute(credentials.ufrag); + } catch (final IllegalArgumentException e) { + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": ignoring invalid ICE candidate " + + e.getMessage()); + continue; + } + final int mLineIndex = indices.indexOf(sdpMid); + if (mLineIndex < 0) { + Log.w( + Config.LOGTAG, + "mLineIndex not found for " + sdpMid + ". available indices " + indices); + } + final IceCandidate iceCandidate = new IceCandidate(sdpMid, mLineIndex, sdp); + Log.d(Config.LOGTAG, "received candidate: " + iceCandidate); + this.webRTCWrapper.addIceCandidate(iceCandidate); + } + } + + private RtpContentMap getRemoteContentMap() { + return isInitiator() ? this.responderRtpContentMap : this.initiatorRtpContentMap; + } + + private List toIdentificationTags(final RtpContentMap rtpContentMap) { + final Group originalGroup = rtpContentMap.group; + final List identificationTags = + originalGroup == null + ? rtpContentMap.getNames() + : originalGroup.getIdentificationTags(); + if (identificationTags.size() == 0) { + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": no identification tags found in initial offer. we won't be able to calculate mLineIndices"); + } + return identificationTags; + } + + private ListenableFuture receiveRtpContentMap( + final JinglePacket jinglePacket, final boolean expectVerification) { + final RtpContentMap receivedContentMap; + try { + receivedContentMap = RtpContentMap.of(jinglePacket); + } catch (final Exception e) { + return Futures.immediateFailedFuture(e); + } + if (receivedContentMap instanceof OmemoVerifiedRtpContentMap) { + final ListenableFuture> future = + id.account + .getAxolotlService() + .decrypt((OmemoVerifiedRtpContentMap) receivedContentMap, id.with); + return Futures.transform( + future, + omemoVerifiedPayload -> { + // TODO test if an exception here triggers a correct abort + omemoVerification.setOrEnsureEqual(omemoVerifiedPayload); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": received verifiable DTLS fingerprint via " + + omemoVerification); + return omemoVerifiedPayload.getPayload(); + }, + MoreExecutors.directExecutor()); + } else if (Config.REQUIRE_RTP_VERIFICATION || expectVerification) { + return Futures.immediateFailedFuture( + new SecurityException("DTLS fingerprint was unexpectedly not verifiable")); + } else { + return Futures.immediateFuture(receivedContentMap); } } private void receiveSessionInitiate(final JinglePacket jinglePacket) { if (isInitiator()) { - Log.d(Config.LOGTAG, String.format("%s: received session-initiate even though we were initiating", id.account.getJid().asBareJid())); + Log.d( + Config.LOGTAG, + String.format( + "%s: received session-initiate even though we were initiating", + id.account.getJid().asBareJid())); if (isTerminated()) { - Log.d(Config.LOGTAG, String.format( - "%s: got a reason to terminate with out-of-order. but already in state %s", - id.account.getJid().asBareJid(), - getState() - )); + Log.d( + Config.LOGTAG, + String.format( + "%s: got a reason to terminate with out-of-order. but already in state %s", + id.account.getJid().asBareJid(), getState())); respondWithOutOfOrder(jinglePacket); } else { terminateWithOutOfOrder(jinglePacket); } return; } - final RtpContentMap contentMap; + final ListenableFuture future = receiveRtpContentMap(jinglePacket, false); + Futures.addCallback( + future, + new FutureCallback() { + @Override + public void onSuccess(@Nullable RtpContentMap rtpContentMap) { + receiveSessionInitiate(jinglePacket, rtpContentMap); + } + + @Override + public void onFailure(@NonNull final Throwable throwable) { + respondOk(jinglePacket); + sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage()); + } + }, + MoreExecutors.directExecutor()); + } + + private void receiveSessionInitiate( + final JinglePacket jinglePacket, final RtpContentMap contentMap) { try { - contentMap = receiveRtpContentMap(jinglePacket, false); contentMap.requireContentDescriptions(); - contentMap.requireDTLSFingerprint(); + contentMap.requireDTLSFingerprint(true); } catch (final RuntimeException e) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents", Throwables.getRootCause(e)); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + ": improperly formatted contents", + Throwables.getRootCause(e)); respondOk(jinglePacket); sendSessionTerminate(Reason.of(e), e.getMessage()); return; } - Log.d(Config.LOGTAG, "processing session-init with " + contentMap.contents.size() + " contents"); + Log.d( + Config.LOGTAG, + "processing session-init with " + contentMap.contents.size() + " contents"); final State target; if (this.state == State.PROCEED) { Preconditions.checkState( proposedMedia != null && proposedMedia.size() > 0, - "proposed media must be set when processing pre-approved session-initiate" - ); + "proposed media must be set when processing pre-approved session-initiate"); if (!this.proposedMedia.equals(contentMap.getMedia())) { - sendSessionTerminate(Reason.SECURITY_ERROR, String.format( - "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s", - this.proposedMedia, - contentMap.getMedia() - )); + sendSessionTerminate( + Reason.SECURITY_ERROR, + String.format( + "Your session proposal (Jingle Message Initiation) included media %s but your session-initiate was %s", + this.proposedMedia, contentMap.getMedia())); return; } target = State.SESSION_INITIALIZED_PRE_APPROVED; @@ -372,86 +621,139 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } if (transition(target, () -> this.initiatorRtpContentMap = contentMap)) { respondOk(jinglePacket); - - final Set> candidates = contentMap.contents.entrySet(); - if (candidates.size() > 0) { - pendingIceCandidates.push(candidates); - } + pendingIceCandidates.addAll(contentMap.contents.entrySet()); if (target == State.SESSION_INITIALIZED_PRE_APPROVED) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": automatically accepting session-initiate"); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": automatically accepting session-initiate"); sendSessionAccept(); } else { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received not pre-approved session-initiate. start ringing"); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": received not pre-approved session-initiate. start ringing"); startRinging(); } } else { - Log.d(Config.LOGTAG, String.format("%s: received session-initiate while in state %s", id.account.getJid().asBareJid(), state)); + Log.d( + Config.LOGTAG, + String.format( + "%s: received session-initiate while in state %s", + id.account.getJid().asBareJid(), state)); terminateWithOutOfOrder(jinglePacket); } } private void receiveSessionAccept(final JinglePacket jinglePacket) { if (!isInitiator()) { - Log.d(Config.LOGTAG, String.format("%s: received session-accept even though we were responding", id.account.getJid().asBareJid())); + Log.d( + Config.LOGTAG, + String.format( + "%s: received session-accept even though we were responding", + id.account.getJid().asBareJid())); terminateWithOutOfOrder(jinglePacket); return; } - final RtpContentMap contentMap; + final ListenableFuture future = + receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint()); + Futures.addCallback( + future, + new FutureCallback() { + @Override + public void onSuccess(@Nullable RtpContentMap rtpContentMap) { + receiveSessionAccept(jinglePacket, rtpContentMap); + } + + @Override + public void onFailure(@NonNull final Throwable throwable) { + respondOk(jinglePacket); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": improperly formatted contents in session-accept", + throwable); + webRTCWrapper.close(); + sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage()); + } + }, + MoreExecutors.directExecutor()); + } + + private void receiveSessionAccept( + final JinglePacket jinglePacket, final RtpContentMap contentMap) { try { - contentMap = receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint()); contentMap.requireContentDescriptions(); contentMap.requireDTLSFingerprint(); } catch (final RuntimeException e) { respondOk(jinglePacket); - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": improperly formatted contents in session-accept", e); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": improperly formatted contents in session-accept", + e); webRTCWrapper.close(); sendSessionTerminate(Reason.of(e), e.getMessage()); return; } final Set initiatorMedia = this.initiatorRtpContentMap.getMedia(); if (!initiatorMedia.equals(contentMap.getMedia())) { - sendSessionTerminate(Reason.SECURITY_ERROR, String.format( - "Your session-included included media %s but our session-initiate was %s", - this.proposedMedia, - contentMap.getMedia() - )); + sendSessionTerminate( + Reason.SECURITY_ERROR, + String.format( + "Your session-included included media %s but our session-initiate was %s", + this.proposedMedia, contentMap.getMedia())); return; } - Log.d(Config.LOGTAG, "processing session-accept with " + contentMap.contents.size() + " contents"); + Log.d( + Config.LOGTAG, + "processing session-accept with " + contentMap.contents.size() + " contents"); if (transition(State.SESSION_ACCEPTED)) { respondOk(jinglePacket); receiveSessionAccept(contentMap); } else { - Log.d(Config.LOGTAG, String.format("%s: received session-accept while in state %s", id.account.getJid().asBareJid(), state)); + Log.d( + Config.LOGTAG, + String.format( + "%s: received session-accept while in state %s", + id.account.getJid().asBareJid(), state)); respondOk(jinglePacket); } } private void receiveSessionAccept(final RtpContentMap contentMap) { this.responderRtpContentMap = contentMap; + this.storePeerDtlsSetup(contentMap.getDtlsSetup()); final SessionDescription sessionDescription; try { sessionDescription = SessionDescription.of(contentMap); } catch (final IllegalArgumentException | NullPointerException e) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-accept to SDP", e); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": unable convert offer from session-accept to SDP", + e); webRTCWrapper.close(); sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); return; } - final org.webrtc.SessionDescription answer = new org.webrtc.SessionDescription( - org.webrtc.SessionDescription.Type.ANSWER, - sessionDescription.toString() - ); + final org.webrtc.SessionDescription answer = + new org.webrtc.SessionDescription( + org.webrtc.SessionDescription.Type.ANSWER, sessionDescription.toString()); try { this.webRTCWrapper.setRemoteDescription(answer).get(); } catch (final Exception e) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", Throwables.getRootCause(e)); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": unable to set remote description after receiving session-accept", + Throwables.getRootCause(e)); webRTCWrapper.close(); - sendSessionTerminate(Reason.FAILED_APPLICATION); + sendSessionTerminate( + Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage()); return; } - final List identificationTags = contentMap.group == null ? contentMap.getNames() : contentMap.group.getIdentificationTags(); - processCandidates(identificationTags, contentMap.contents.entrySet()); + processCandidates(contentMap.contents.entrySet()); } private void sendSessionAccept() { @@ -463,7 +765,11 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web try { offer = SessionDescription.of(rtpContentMap); } catch (final IllegalArgumentException | NullPointerException e) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable convert offer from session-initiate to SDP", e); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": unable convert offer from session-initiate to SDP", + e); webRTCWrapper.close(); sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); return; @@ -475,9 +781,15 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web discoverIceServers(iceServers -> sendSessionAccept(media, offer, iceServers)); } - private synchronized void sendSessionAccept(final Set media, final SessionDescription offer, final List iceServers) { + private synchronized void sendSessionAccept( + final Set media, + final SessionDescription offer, + final List iceServers) { if (isTerminated()) { - Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do."); + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": ICE servers got discovered when session was already terminated. nothing to do."); return; } try { @@ -488,54 +800,113 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web sendSessionTerminate(Reason.FAILED_APPLICATION); return; } - final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription( - org.webrtc.SessionDescription.Type.OFFER, - offer.toString() - ); + final org.webrtc.SessionDescription sdp = + new org.webrtc.SessionDescription( + org.webrtc.SessionDescription.Type.OFFER, offer.toString()); try { this.webRTCWrapper.setRemoteDescription(sdp).get(); addIceCandidatesFromBlackLog(); - org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createAnswer().get(); - final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description); - final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription); - sendSessionAccept(respondingRtpContentMap); - this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get(); + org.webrtc.SessionDescription webRTCSessionDescription = + this.webRTCWrapper.setLocalDescription().get(); + prepareSessionAccept(webRTCSessionDescription); } catch (final Exception e) { - Log.d(Config.LOGTAG, "unable to send session accept", Throwables.getRootCause(e)); - webRTCWrapper.close(); - sendSessionTerminate(Reason.FAILED_APPLICATION); + failureToAcceptSession(e); } } + private void failureToAcceptSession(final Throwable throwable) { + if (isTerminated()) { + return; + } + final Throwable rootCause = Throwables.getRootCause(throwable); + Log.d(Config.LOGTAG, "unable to send session accept", rootCause); + webRTCWrapper.close(); + sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage()); + } + private void addIceCandidatesFromBlackLog() { - while (!this.pendingIceCandidates.isEmpty()) { - processCandidates(this.pendingIceCandidates.poll()); - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": added candidates from back log"); + Map.Entry foo; + while ((foo = this.pendingIceCandidates.poll()) != null) { + processCandidate(foo); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + ": added candidate from back log"); } } + private void prepareSessionAccept( + final org.webrtc.SessionDescription webRTCSessionDescription) { + final SessionDescription sessionDescription = + SessionDescription.parse(webRTCSessionDescription.description); + final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription); + this.responderRtpContentMap = respondingRtpContentMap; + storePeerDtlsSetup(respondingRtpContentMap.getDtlsSetup().flip()); + webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + final ListenableFuture outgoingContentMapFuture = + prepareOutgoingContentMap(respondingRtpContentMap); + Futures.addCallback( + outgoingContentMapFuture, + new FutureCallback() { + @Override + public void onSuccess(final RtpContentMap outgoingContentMap) { + sendSessionAccept(outgoingContentMap); + } + + @Override + public void onFailure(@NonNull Throwable throwable) { + failureToAcceptSession(throwable); + } + }, + MoreExecutors.directExecutor()); + } + private void sendSessionAccept(final RtpContentMap rtpContentMap) { - this.responderRtpContentMap = rtpContentMap; - this.transitionOrThrow(State.SESSION_ACCEPTED); - final RtpContentMap outgoingContentMap; - if (this.omemoVerification.hasDeviceId()) { - final AxolotlService.OmemoVerifiedPayload verifiedPayload; - try { - verifiedPayload = id.account.getAxolotlService().encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId()); - outgoingContentMap = verifiedPayload.getPayload(); - this.omemoVerification.setOrEnsureEqual(verifiedPayload); - } catch (final Exception e) { - throw new SecurityException("Unable to verify DTLS Fingerprint with OMEMO", e); - } - } else { - outgoingContentMap = rtpContentMap; + if (isTerminated()) { + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": preparing session accept was too slow. already terminated. nothing to do."); + return; } - final JinglePacket sessionAccept = outgoingContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId); + transitionOrThrow(State.SESSION_ACCEPTED); + final JinglePacket sessionAccept = + rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_ACCEPT, id.sessionId); send(sessionAccept); } - synchronized void deliveryMessage(final Jid from, final Element message, final String serverMessageId, final long timestamp) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": delivered message to JingleRtpConnection " + message); + private ListenableFuture prepareOutgoingContentMap( + final RtpContentMap rtpContentMap) { + if (this.omemoVerification.hasDeviceId()) { + ListenableFuture> + verifiedPayloadFuture = + id.account + .getAxolotlService() + .encrypt( + rtpContentMap, + id.with, + omemoVerification.getDeviceId()); + return Futures.transform( + verifiedPayloadFuture, + verifiedPayload -> { + omemoVerification.setOrEnsureEqual(verifiedPayload); + return verifiedPayload.getPayload(); + }, + MoreExecutors.directExecutor()); + } else { + return Futures.immediateFuture(rtpContentMap); + } + } + + synchronized void deliveryMessage( + final Jid from, + final Element message, + final String serverMessageId, + final long timestamp) { + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": delivered message to JingleRtpConnection " + + message); switch (message.getName()) { case "propose": receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp); @@ -558,47 +929,73 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } void deliverFailedProceed() { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": receive message error for proceed message"); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + ": receive message error for proceed message"); if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) { webRTCWrapper.close(); - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": transitioned into connectivity error"); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + ": transitioned into connectivity error"); this.finish(); } } private void receiveAccept(final Jid from, final String serverMsgId, final long timestamp) { - final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid()); + final boolean originatedFromMyself = + from.asBareJid().equals(id.account.getJid().asBareJid()); if (originatedFromMyself) { if (transition(State.ACCEPTED)) { if (serverMsgId != null) { this.message.setServerMsgId(serverMsgId); } this.message.setTime(timestamp); - this.message.setCarbon(true); //indicate that call was accepted on other device + this.message.setCarbon(true); // indicate that call was accepted on other device this.writeLogMessageSuccess(0); - this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification(); + this.xmppConnectionService + .getNotificationService() + .cancelIncomingCallNotification(); this.finish(); } else { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to transition to accept because already in state=" + this.state); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": unable to transition to accept because already in state=" + + this.state); } } else { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + ": ignoring 'accept' from " + from); } } private void receiveReject(final Jid from, final String serverMsgId, final long timestamp) { - final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid()); - //reject from another one of my clients + final boolean originatedFromMyself = + from.asBareJid().equals(id.account.getJid().asBareJid()); + // reject from another one of my clients if (originatedFromMyself) { receiveRejectFromMyself(serverMsgId, timestamp); } else if (isInitiator()) { if (from.equals(id.with)) { receiveRejectFromResponder(); } else { - Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with); + Log.d( + Config.LOGTAG, + id.account.getJid() + + ": ignoring reject from " + + from + + " for session with " + + id.with); } } else { - Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from " + from + " for session with " + id.with); + Log.d( + Config.LOGTAG, + id.account.getJid() + + ": ignoring reject from " + + from + + " for session with " + + id.with); } } @@ -610,54 +1007,94 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web this.message.setServerMsgId(serverMsgId); } this.message.setTime(timestamp); - this.message.setCarbon(true); //indicate that call was rejected on other device + this.message.setCarbon(true); // indicate that call was rejected on other device writeLogMessageMissed(); } else { - Log.d(Config.LOGTAG, "not able to transition into REJECTED because already in " + this.state); + Log.d( + Config.LOGTAG, + "not able to transition into REJECTED because already in " + this.state); } } private void receiveRejectFromResponder() { if (isInState(State.PROCEED)) { - Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while still in proceed. callee reconsidered"); + Log.d( + Config.LOGTAG, + id.account.getJid() + + ": received reject while still in proceed. callee reconsidered"); closeTransitionLogFinish(State.REJECTED_RACED); return; } if (isInState(State.SESSION_INITIALIZED_PRE_APPROVED)) { - Log.d(Config.LOGTAG, id.account.getJid() + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init"); + Log.d( + Config.LOGTAG, + id.account.getJid() + + ": received reject while in SESSION_INITIATED_PRE_APPROVED. callee reconsidered before receiving session-init"); closeTransitionLogFinish(State.TERMINATED_DECLINED_OR_BUSY); return; } - Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring reject from responder because already in state " + this.state); + Log.d( + Config.LOGTAG, + id.account.getJid() + + ": ignoring reject from responder because already in state " + + this.state); } - private void receivePropose(final Jid from, final Propose propose, final String serverMsgId, final long timestamp) { - final boolean originatedFromMyself = from.asBareJid().equals(id.account.getJid().asBareJid()); + private void receivePropose( + final Jid from, final Propose propose, final String serverMsgId, final long timestamp) { + final boolean originatedFromMyself = + from.asBareJid().equals(id.account.getJid().asBareJid()); if (originatedFromMyself) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring"); - } else if (transition(State.PROPOSED, () -> { - final Collection descriptions = Collections2.transform( - Collections2.filter(propose.getDescriptions(), d -> d instanceof RtpDescription), - input -> (RtpDescription) input - ); - final Collection media = Collections2.transform(descriptions, RtpDescription::getMedia); - Preconditions.checkState(!media.contains(Media.UNKNOWN), "RTP descriptions contain unknown media"); - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received session proposal from " + from + " for " + media); - this.proposedMedia = Sets.newHashSet(media); - })) { + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + ": saw proposal from myself. ignoring"); + } else if (transition( + State.PROPOSED, + () -> { + final Collection descriptions = + Collections2.transform( + Collections2.filter( + propose.getDescriptions(), + d -> d instanceof RtpDescription), + input -> (RtpDescription) input); + final Collection media = + Collections2.transform(descriptions, RtpDescription::getMedia); + Preconditions.checkState( + !media.contains(Media.UNKNOWN), + "RTP descriptions contain unknown media"); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": received session proposal from " + + from + + " for " + + media); + this.proposedMedia = Sets.newHashSet(media); + })) { if (serverMsgId != null) { this.message.setServerMsgId(serverMsgId); } this.message.setTime(timestamp); startRinging(); } else { - Log.d(Config.LOGTAG, id.account.getJid() + ": ignoring session proposal because already in " + state); + Log.d( + Config.LOGTAG, + id.account.getJid() + + ": ignoring session proposal because already in " + + state); } } private void startRinging() { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received call from " + id.with + ". start ringing"); - ringingTimeoutFuture = jingleConnectionManager.schedule(this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": received call from " + + id.with + + ". start ringing"); + ringingTimeoutFuture = + jingleConnectionManager.schedule( + this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS); final String uuid = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(id.account, id.with.asBareJid(), false, false).getUuid(); xmppConnectionService.getNotificationService().showIncomingCallNotification(id, getMedia(), uuid); } @@ -683,8 +1120,11 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } } - private void receiveProceed(final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) { - final Set media = Preconditions.checkNotNull(this.proposedMedia, "Proposed media has to be set before handling proceed"); + private void receiveProceed( + final Jid from, final Proceed proceed, final String serverMsgId, final long timestamp) { + final Set media = + Preconditions.checkNotNull( + this.proposedMedia, "Proposed media has to be set before handling proceed"); Preconditions.checkState(media.size() > 0, "Proposed media should not be empty"); if (from.equals(id.with)) { if (isInitiator()) { @@ -698,34 +1138,64 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web this.omemoVerification.setDeviceId(remoteDeviceId); } else { if (remoteDeviceId != null) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": remote party signaled support for OMEMO verification but we have OMEMO disabled"); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": remote party signaled support for OMEMO verification but we have OMEMO disabled"); } this.omemoVerification.setDeviceId(null); } this.sendSessionInitiate(media, State.SESSION_INITIALIZED_PRE_APPROVED); } else { - Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because already in %s", id.account.getJid().asBareJid(), this.state)); + Log.d( + Config.LOGTAG, + String.format( + "%s: ignoring proceed because already in %s", + id.account.getJid().asBareJid(), this.state)); } } else { - Log.d(Config.LOGTAG, String.format("%s: ignoring proceed because we were not initializing", id.account.getJid().asBareJid())); + Log.d( + Config.LOGTAG, + String.format( + "%s: ignoring proceed because we were not initializing", + id.account.getJid().asBareJid())); } } else if (from.asBareJid().equals(id.account.getJid().asBareJid())) { if (transition(State.ACCEPTED)) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": moved session with " + id.with + " into state accepted after received carbon copied procced"); - this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification(); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": moved session with " + + id.with + + " into state accepted after received carbon copied procced"); + this.xmppConnectionService + .getNotificationService() + .cancelIncomingCallNotification(); this.finish(); } } else { - Log.d(Config.LOGTAG, String.format("%s: ignoring proceed from %s. was expected from %s", id.account.getJid().asBareJid(), from, id.with)); + Log.d( + Config.LOGTAG, + String.format( + "%s: ignoring proceed from %s. was expected from %s", + id.account.getJid().asBareJid(), from, id.with)); } } private void receiveRetract(final Jid from, final String serverMsgId, final long timestamp) { if (from.equals(id.with)) { - final State target = this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED; + final State target = + this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED; if (transition(target)) { xmppConnectionService.getNotificationService().cancelIncomingCallNotification(); - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": session with " + id.with + " has been retracted (serverMsgId=" + serverMsgId + ")"); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": session with " + + id.with + + " has been retracted (serverMsgId=" + + serverMsgId + + ")"); if (serverMsgId != null) { this.message.setServerMsgId(serverMsgId); } @@ -740,8 +1210,15 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state); } } else { - //TODO parse retract from self - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received retract from " + from + ". expected retract from" + id.with + ". ignoring"); + // TODO parse retract from self + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": received retract from " + + from + + ". expected retract from" + + id.with + + ". ignoring"); } } @@ -754,9 +1231,15 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web discoverIceServers(iceServers -> sendSessionInitiate(media, targetState, iceServers)); } - private synchronized void sendSessionInitiate(final Set media, final State targetState, final List iceServers) { + private synchronized void sendSessionInitiate( + final Set media, + final State targetState, + final List iceServers) { if (isTerminated()) { - Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": ICE servers got discovered when session was already terminated. nothing to do."); + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": ICE servers got discovered when session was already terminated. nothing to do."); return; } try { @@ -764,52 +1247,120 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } catch (final WebRTCWrapper.InitializationException e) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC"); webRTCWrapper.close(); - sendJingleMessage("retract", id.with.asBareJid()); - transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE); - this.finish(); + sendRetract(Reason.ofThrowable(e)); return; } try { - org.webrtc.SessionDescription webRTCSessionDescription = this.webRTCWrapper.createOffer().get(); - final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description); - final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription); - sendSessionInitiate(rtpContentMap, targetState); - this.webRTCWrapper.setLocalDescription(webRTCSessionDescription).get(); + org.webrtc.SessionDescription webRTCSessionDescription = + this.webRTCWrapper.setLocalDescription().get(); + prepareSessionInitiate(webRTCSessionDescription, targetState); } catch (final Exception e) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", Throwables.getRootCause(e)); - webRTCWrapper.close(); - if (isInState(targetState)) { - sendSessionTerminate(Reason.FAILED_APPLICATION); - } else { - sendJingleMessage("retract", id.with.asBareJid()); - transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE); - this.finish(); - } + // TODO sending the error text is worthwhile as well. Especially for FailureToSet + // exceptions + failureToInitiateSession(e, targetState); } } - private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) { + private void failureToInitiateSession(final Throwable throwable, final State targetState) { + if (isTerminated()) { + return; + } + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + ": unable to sendSessionInitiate", + Throwables.getRootCause(throwable)); + webRTCWrapper.close(); + final Reason reason = Reason.ofThrowable(throwable); + if (isInState(targetState)) { + sendSessionTerminate(reason); + } else { + sendRetract(reason); + } + } + + private void sendRetract(final Reason reason) { + // TODO embed reason into retract + sendJingleMessage("retract", id.with.asBareJid()); + transitionOrThrow(reasonToState(reason)); + this.finish(); + } + + private void prepareSessionInitiate( + final org.webrtc.SessionDescription webRTCSessionDescription, final State targetState) { + final SessionDescription sessionDescription = + SessionDescription.parse(webRTCSessionDescription.description); + final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription); this.initiatorRtpContentMap = rtpContentMap; + this.webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + final ListenableFuture outgoingContentMapFuture = + encryptSessionInitiate(rtpContentMap); + Futures.addCallback( + outgoingContentMapFuture, + new FutureCallback() { + @Override + public void onSuccess(final RtpContentMap outgoingContentMap) { + sendSessionInitiate(outgoingContentMap, targetState); + } + + @Override + public void onFailure(@NonNull final Throwable throwable) { + failureToInitiateSession(throwable, targetState); + } + }, + MoreExecutors.directExecutor()); + } + + private void sendSessionInitiate(final RtpContentMap rtpContentMap, final State targetState) { + if (isTerminated()) { + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": preparing session was too slow. already terminated. nothing to do."); + return; + } this.transitionOrThrow(targetState); - //TODO do on background thread? - final RtpContentMap outgoingContentMap = encryptSessionInitiate(rtpContentMap); - final JinglePacket sessionInitiate = outgoingContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId); + final JinglePacket sessionInitiate = + rtpContentMap.toJinglePacket(JinglePacket.Action.SESSION_INITIATE, id.sessionId); send(sessionInitiate); } - private RtpContentMap encryptSessionInitiate(final RtpContentMap rtpContentMap) { + private ListenableFuture encryptSessionInitiate( + final RtpContentMap rtpContentMap) { if (this.omemoVerification.hasDeviceId()) { - final AxolotlService.OmemoVerifiedPayload verifiedPayload; - try { - verifiedPayload = id.account.getAxolotlService().encrypt(rtpContentMap, id.with, omemoVerification.getDeviceId()); - } catch (final CryptoFailedException e) { - Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", e); - return rtpContentMap; + final ListenableFuture> + verifiedPayloadFuture = + id.account + .getAxolotlService() + .encrypt( + rtpContentMap, + id.with, + omemoVerification.getDeviceId()); + final ListenableFuture future = + Futures.transform( + verifiedPayloadFuture, + verifiedPayload -> { + omemoVerification.setSessionFingerprint( + verifiedPayload.getFingerprint()); + return verifiedPayload.getPayload(); + }, + MoreExecutors.directExecutor()); + if (Config.REQUIRE_RTP_VERIFICATION) { + return future; } - this.omemoVerification.setSessionFingerprint(verifiedPayload.getFingerprint()); - return verifiedPayload.getPayload(); + return Futures.catching( + future, + CryptoFailedException.class, + e -> { + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": unable to use OMEMO DTLS verification on outgoing session initiate. falling back", + e); + return rtpContentMap; + }, + MoreExecutors.directExecutor()); } else { - return rtpContentMap; + return Futures.immediateFuture(rtpContentMap); } } @@ -824,23 +1375,31 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web if (previous != State.NULL) { writeLogMessage(target); } - final JinglePacket jinglePacket = new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId); + final JinglePacket jinglePacket = + new JinglePacket(JinglePacket.Action.SESSION_TERMINATE, id.sessionId); jinglePacket.setReason(reason, text); Log.d(Config.LOGTAG, jinglePacket.toString()); send(jinglePacket); finish(); } - private void sendTransportInfo(final String contentName, IceUdpTransportInfo.Candidate candidate) { + private void sendTransportInfo( + final String contentName, IceUdpTransportInfo.Candidate candidate) { final RtpContentMap transportInfo; try { - final RtpContentMap rtpContentMap = isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap; + final RtpContentMap rtpContentMap = + isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap; transportInfo = rtpContentMap.transportInfo(contentName, candidate); } catch (final Exception e) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to prepare transport-info from candidate for content=" + contentName); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": unable to prepare transport-info from candidate for content=" + + contentName); return; } - final JinglePacket jinglePacket = transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId); + final JinglePacket jinglePacket = + transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId); send(jinglePacket); } @@ -851,59 +1410,97 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web private synchronized void handleIqResponse(final Account account, final IqPacket response) { if (response.getType() == IqPacket.TYPE.ERROR) { - final String errorCondition = response.getErrorCondition(); - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ-error from " + response.getFrom() + " in RTP session. " + errorCondition); - if (isTerminated()) { - Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated"); - return; - } - this.webRTCWrapper.close(); - final State target; - if (Arrays.asList( - "service-unavailable", - "recipient-unavailable", - "remote-server-not-found", - "remote-server-timeout" - ).contains(errorCondition)) { - target = State.TERMINATED_CONNECTIVITY_ERROR; - } else { - target = State.TERMINATED_APPLICATION_FAILURE; - } - transitionOrThrow(target); - this.finish(); - } else if (response.getType() == IqPacket.TYPE.TIMEOUT) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": received IQ timeout in RTP session with " + id.with + ". terminating with connectivity error"); - if (isTerminated()) { - Log.i(Config.LOGTAG, id.account.getJid().asBareJid() + ": ignoring error because session was already terminated"); - return; - } - this.webRTCWrapper.close(); - transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR); - this.finish(); + handleIqErrorResponse(response); + return; + } + if (response.getType() == IqPacket.TYPE.TIMEOUT) { + handleIqTimeoutResponse(response); } } + private void handleIqErrorResponse(final IqPacket response) { + Preconditions.checkArgument(response.getType() == IqPacket.TYPE.ERROR); + final String errorCondition = response.getErrorCondition(); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": received IQ-error from " + + response.getFrom() + + " in RTP session. " + + errorCondition); + if (isTerminated()) { + Log.i( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": ignoring error because session was already terminated"); + return; + } + this.webRTCWrapper.close(); + final State target; + if (Arrays.asList( + "service-unavailable", + "recipient-unavailable", + "remote-server-not-found", + "remote-server-timeout") + .contains(errorCondition)) { + target = State.TERMINATED_CONNECTIVITY_ERROR; + } else { + target = State.TERMINATED_APPLICATION_FAILURE; + } + transitionOrThrow(target); + this.finish(); + } + + private void handleIqTimeoutResponse(final IqPacket response) { + Preconditions.checkArgument(response.getType() == IqPacket.TYPE.TIMEOUT); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": received IQ timeout in RTP session with " + + id.with + + ". terminating with connectivity error"); + if (isTerminated()) { + Log.i( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": ignoring error because session was already terminated"); + return; + } + this.webRTCWrapper.close(); + transitionOrThrow(State.TERMINATED_CONNECTIVITY_ERROR); + this.finish(); + } + private void terminateWithOutOfOrder(final JinglePacket jinglePacket) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": terminating session with out-of-order"); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + ": terminating session with out-of-order"); this.webRTCWrapper.close(); transitionOrThrow(State.TERMINATED_APPLICATION_FAILURE); respondWithOutOfOrder(jinglePacket); this.finish(); } + private void respondWithTieBreak(final JinglePacket jinglePacket) { + respondWithJingleError(jinglePacket, "tie-break", "conflict", "cancel"); + } + private void respondWithOutOfOrder(final JinglePacket jinglePacket) { - jingleConnectionManager.respondWithJingleError(id.account, jinglePacket, "out-of-order", "unexpected-request", "wait"); + respondWithJingleError(jinglePacket, "out-of-order", "unexpected-request", "wait"); + } + + void respondWithJingleError( + final IqPacket original, + String jingleCondition, + String condition, + String conditionType) { + jingleConnectionManager.respondWithJingleError( + id.account, original, jingleCondition, condition, conditionType); } private void respondOk(final JinglePacket jinglePacket) { - xmppConnectionService.sendIqPacket(id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null); - } - - public void throwStateTransitionException() { - final StateTransitionException exception = this.stateTransitionException; - if (exception != null) { - throw new IllegalStateException(String.format("Transition to %s did not call finish", exception.state), exception); - } + xmppConnectionService.sendIqPacket( + id.account, jinglePacket.generateResponse(IqPacket.TYPE.RESULT), null); } public RtpEndUserState getEndUserState() { @@ -929,23 +1526,7 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web return RtpEndUserState.CONNECTING; } case SESSION_ACCEPTED: - final PeerConnection.PeerConnectionState state; - try { - state = webRTCWrapper.getState(); - } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) { - //We usually close the WebRTCWrapper *before* transitioning so we might still - //be in SESSION_ACCEPTED even though the peerConnection has been torn down - return RtpEndUserState.ENDING_CALL; - } - if (state == PeerConnection.PeerConnectionState.CONNECTED) { - return RtpEndUserState.CONNECTED; - } else if (state == PeerConnection.PeerConnectionState.NEW || state == PeerConnection.PeerConnectionState.CONNECTING) { - return RtpEndUserState.CONNECTING; - } else if (state == PeerConnection.PeerConnectionState.CLOSED) { - return RtpEndUserState.ENDING_CALL; - } else { - return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR; - } + return getPeerConnectionStateAsEndUserState(); case REJECTED: case REJECTED_RACED: case TERMINATED_DECLINED_OR_BUSY: @@ -966,13 +1547,40 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web return RtpEndUserState.RETRACTED; } case TERMINATED_CONNECTIVITY_ERROR: - return rtpConnectionStarted == 0 ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR; + return zeroDuration() + ? RtpEndUserState.CONNECTIVITY_ERROR + : RtpEndUserState.CONNECTIVITY_LOST_ERROR; case TERMINATED_APPLICATION_FAILURE: return RtpEndUserState.APPLICATION_ERROR; case TERMINATED_SECURITY_ERROR: return RtpEndUserState.SECURITY_ERROR; } - throw new IllegalStateException(String.format("%s has no equivalent EndUserState", this.state)); + throw new IllegalStateException( + String.format("%s has no equivalent EndUserState", this.state)); + } + + private RtpEndUserState getPeerConnectionStateAsEndUserState() { + final PeerConnection.PeerConnectionState state; + try { + state = webRTCWrapper.getState(); + } catch (final WebRTCWrapper.PeerConnectionNotInitialized e) { + // We usually close the WebRTCWrapper *before* transitioning so we might still + // be in SESSION_ACCEPTED even though the peerConnection has been torn down + return RtpEndUserState.ENDING_CALL; + } + switch (state) { + case CONNECTED: + return RtpEndUserState.CONNECTED; + case NEW: + case CONNECTING: + return RtpEndUserState.CONNECTING; + case CLOSED: + return RtpEndUserState.ENDING_CALL; + default: + return zeroDuration() + ? RtpEndUserState.CONNECTIVITY_ERROR + : RtpEndUserState.RECONNECTING; + } } public Set getMedia() { @@ -980,36 +1588,33 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web if (current == State.NULL) { if (isInitiator()) { return Preconditions.checkNotNull( - this.proposedMedia, - "RTP connection has not been initialized properly" - ); + this.proposedMedia, "RTP connection has not been initialized properly"); } throw new IllegalStateException("RTP connection has not been initialized yet"); } if (Arrays.asList(State.PROPOSED, State.PROCEED).contains(current)) { return Preconditions.checkNotNull( - this.proposedMedia, - "RTP connection has not been initialized properly" - ); + this.proposedMedia, "RTP connection has not been initialized properly"); } final RtpContentMap initiatorContentMap = initiatorRtpContentMap; if (initiatorContentMap != null) { return initiatorContentMap.getMedia(); } else if (isTerminated()) { - return Collections.emptySet(); //we might fail before we ever got a chance to set media + return Collections.emptySet(); // we might fail before we ever got a chance to set media } else { - return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly"); + return Preconditions.checkNotNull( + this.proposedMedia, "RTP connection has not been initialized properly"); } } - public boolean isVerified() { final String fingerprint = this.omemoVerification.getFingerprint(); if (fingerprint == null) { return false; } - final FingerprintStatus status = id.account.getAxolotlService().getFingerprintTrust(fingerprint); - return status != null && status.getTrust() == FingerprintStatus.Trust.VERIFIED; + final FingerprintStatus status = + id.account.getAxolotlService().getFingerprintTrust(fingerprint); + return status != null && status.isVerified(); } public synchronized void acceptCall() { @@ -1023,18 +1628,23 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web acceptCallFromSessionInitialized(); break; case ACCEPTED: - Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted with another client. UI was just lagging behind"); + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": the call has already been accepted with another client. UI was just lagging behind"); break; case PROCEED: case SESSION_ACCEPTED: - Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": the call has already been accepted. user probably double tapped the UI"); + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": the call has already been accepted. user probably double tapped the UI"); break; default: throw new IllegalStateException("Can not accept call from " + this.state); } } - public void notifyPhoneCall() { Log.d(Config.LOGTAG, "a phone call has just been started. killing jingle rtp connections"); if (Arrays.asList(State.PROPOSED, State.SESSION_INITIALIZED).contains(this.state)) { @@ -1046,7 +1656,10 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web public synchronized void rejectCall() { if (isTerminated()) { - Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received rejectCall() when session has already been terminated. nothing to do"); + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": received rejectCall() when session has already been terminated. nothing to do"); return; } switch (this.state) { @@ -1063,7 +1676,10 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web public synchronized void endCall() { if (isTerminated()) { - Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": received endCall() when session has already been terminated. nothing to do"); + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": received endCall() when session has already been terminated. nothing to do"); return; } if (isInState(State.PROPOSED) && !isInitiator()) { @@ -1078,7 +1694,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } return; } - if (isInitiator() && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) { + if (isInitiator() + && isInState(State.SESSION_INITIALIZED, State.SESSION_INITIALIZED_PRE_APPROVED)) { this.webRTCWrapper.close(); sendSessionTerminate(Reason.CANCEL); return; @@ -1092,11 +1709,17 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web sendSessionTerminate(Reason.SUCCESS); return; } - if (isInState(State.TERMINATED_APPLICATION_FAILURE, State.TERMINATED_CONNECTIVITY_ERROR, State.TERMINATED_DECLINED_OR_BUSY)) { - Log.d(Config.LOGTAG, "ignoring request to end call because already in state " + this.state); + if (isInState( + State.TERMINATED_APPLICATION_FAILURE, + State.TERMINATED_CONNECTIVITY_ERROR, + State.TERMINATED_DECLINED_OR_BUSY)) { + Log.d( + Config.LOGTAG, + "ignoring request to end call because already in state " + this.state); return; } - throw new IllegalStateException("called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator()); + throw new IllegalStateException( + "called 'endCall' while in state " + this.state + ". isInitiator=" + isInitiator()); } private void retractFromProceed() { @@ -1112,7 +1735,9 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web finish(); } - private void setupWebRTC(final Set media, final List iceServers) throws WebRTCWrapper.InitializationException { + private void setupWebRTC( + final Set media, final List iceServers) + throws WebRTCWrapper.InitializationException { this.jingleConnectionManager.ensureConnectionIsRegistered(this); final AppRTCAudioManager.SpeakerPhonePreference speakerPhonePreference; if (media.contains(Media.VIDEO)) { @@ -1156,14 +1781,18 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web private void sendJingleMessage(final String action, final Jid to) { final MessagePacket messagePacket = new MessagePacket(); - messagePacket.setType(MessagePacket.TYPE_CHAT); //we want to carbon copy those + messagePacket.setType(MessagePacket.TYPE_CHAT); // we want to carbon copy those messagePacket.setTo(to); - final Element intent = messagePacket.addChild(action, Namespace.JINGLE_MESSAGE).setAttribute("id", id.sessionId); + final Element intent = + messagePacket + .addChild(action, Namespace.JINGLE_MESSAGE) + .setAttribute("id", id.sessionId); if ("proceed".equals(action)) { messagePacket.setId(JINGLE_MESSAGE_PROCEED_ID_PREFIX + id.sessionId); if (isOmemoEnabled()) { final int deviceId = id.account.getAxolotlService().getOwnDeviceId(); - final Element device = intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION); + final Element device = + intent.addChild("device", Namespace.OMEMO_DTLS_SRTP_VERIFICATION); device.setAttribute("id", deviceId); } } @@ -1174,7 +1803,8 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web private boolean isOmemoEnabled() { final Conversational conversational = message.getConversation(); if (conversational instanceof Conversation) { - return ((Conversation) conversational).getNextEncryption() == Message.ENCRYPTION_AXOLOTL; + return ((Conversation) conversational).getNextEncryption() + == Message.ENCRYPTION_AXOLOTL; } return false; } @@ -1196,7 +1826,6 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web final Collection validTransitions = VALID_TRANSITIONS.get(this.state); if (validTransitions != null && validTransitions.contains(target)) { this.state = target; - this.stateTransitionException = new StateTransitionException(target); if (runnable != null) { runnable.run(); } @@ -1211,57 +1840,160 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web void transitionOrThrow(final State target) { if (!transition(target)) { - throw new IllegalStateException(String.format("Unable to transition from %s to %s", this.state, target)); + throw new IllegalStateException( + String.format("Unable to transition from %s to %s", this.state, target)); } } @Override public void onIceCandidate(final IceCandidate iceCandidate) { - final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp); - Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate.toString()); + final RtpContentMap rtpContentMap = + isInitiator() ? this.initiatorRtpContentMap : this.responderRtpContentMap; + final IceUdpTransportInfo.Credentials credentials; + try { + credentials = rtpContentMap.getCredentials(iceCandidate.sdpMid); + } catch (final IllegalArgumentException e) { + Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate, e); + return; + } + final String uFrag = credentials.ufrag; + final IceUdpTransportInfo.Candidate candidate = + IceUdpTransportInfo.Candidate.fromSdpAttribute(iceCandidate.sdp, uFrag); + if (candidate == null) { + Log.d(Config.LOGTAG, "ignoring (not sending) candidate: " + iceCandidate); + return; + } + Log.d(Config.LOGTAG, "sending candidate: " + iceCandidate); sendTransportInfo(iceCandidate.sdpMid, candidate); } @Override public void onConnectionChange(final PeerConnection.PeerConnectionState newState) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState); - if (newState == PeerConnection.PeerConnectionState.CONNECTED && this.rtpConnectionStarted == 0) { - this.rtpConnectionStarted = SystemClock.elapsedRealtime(); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + ": PeerConnectionState changed to " + newState); + this.stateHistory.add(newState); + if (newState == PeerConnection.PeerConnectionState.CONNECTED) { + this.sessionDuration.start(); + updateOngoingCallNotification(); + } else if (this.sessionDuration.isRunning()) { + this.sessionDuration.stop(); + updateOngoingCallNotification(); } - if (newState == PeerConnection.PeerConnectionState.CLOSED && this.rtpConnectionEnded == 0) { - this.rtpConnectionEnded = SystemClock.elapsedRealtime(); - } - //TODO 'DISCONNECTED' might be an opportunity to renew the offer and send a transport-replace - //TODO exact syntax is yet to be determined but transport-replace sounds like the most reasonable - //as there is no content-replace - if (Arrays.asList(PeerConnection.PeerConnectionState.FAILED, PeerConnection.PeerConnectionState.DISCONNECTED).contains(newState)) { - if (isTerminated()) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": not sending session-terminate after connectivity error because session is already in state " + this.state); + + final boolean neverConnected = + !this.stateHistory.contains(PeerConnection.PeerConnectionState.CONNECTED); + + if (newState == PeerConnection.PeerConnectionState.FAILED) { + if (neverConnected) { + if (isTerminated()) { + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": not sending session-terminate after connectivity error because session is already in state " + + this.state); + return; + } + webRTCWrapper.execute(this::closeWebRTCSessionAfterFailedConnection); return; + } else { + webRTCWrapper.restartIce(); } - new Thread(this::closeWebRTCSessionAfterFailedConnection).start(); - } else { - updateEndUserState(); } + updateEndUserState(); + } + + @Override + public void onRenegotiationNeeded() { + this.webRTCWrapper.execute(this::initiateIceRestart); + } + + private void initiateIceRestart() { + // TODO discover new TURN/STUN credentials + this.stateHistory.clear(); + this.webRTCWrapper.setIsReadyToReceiveIceCandidates(false); + final SessionDescription sessionDescription; + try { + sessionDescription = setLocalSessionDescription(); + } catch (final Exception e) { + final Throwable cause = Throwables.getRootCause(e); + Log.d(Config.LOGTAG, "failed to renegotiate", cause); + sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage()); + return; + } + final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription); + final RtpContentMap transportInfo = rtpContentMap.transportInfo(); + final JinglePacket jinglePacket = + transportInfo.toJinglePacket(JinglePacket.Action.TRANSPORT_INFO, id.sessionId); + Log.d(Config.LOGTAG, "initiating ice restart: " + jinglePacket); + jinglePacket.setTo(id.with); + xmppConnectionService.sendIqPacket( + id.account, + jinglePacket, + (account, response) -> { + if (response.getType() == IqPacket.TYPE.RESULT) { + Log.d(Config.LOGTAG, "received success to our ice restart"); + setLocalContentMap(rtpContentMap); + webRTCWrapper.setIsReadyToReceiveIceCandidates(true); + return; + } + if (response.getType() == IqPacket.TYPE.ERROR) { + final Element error = response.findChild("error"); + if (error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS)) { + Log.d(Config.LOGTAG, "received tie-break as result of ice restart"); + return; + } + handleIqErrorResponse(response); + } + if (response.getType() == IqPacket.TYPE.TIMEOUT) { + handleIqTimeoutResponse(response); + } + }); + } + + private void setLocalContentMap(final RtpContentMap rtpContentMap) { + if (isInitiator()) { + this.initiatorRtpContentMap = rtpContentMap; + } else { + this.responderRtpContentMap = rtpContentMap; + } + } + + private void setRemoteContentMap(final RtpContentMap rtpContentMap) { + if (isInitiator()) { + this.responderRtpContentMap = rtpContentMap; + } else { + this.initiatorRtpContentMap = rtpContentMap; + } + } + + private SessionDescription setLocalSessionDescription() + throws ExecutionException, InterruptedException { + final org.webrtc.SessionDescription sessionDescription = + this.webRTCWrapper.setLocalDescription().get(); + return SessionDescription.parse(sessionDescription.description); } private void closeWebRTCSessionAfterFailedConnection() { this.webRTCWrapper.close(); synchronized (this) { if (isTerminated()) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": no need to send session-terminate after failed connection. Other party already did"); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": no need to send session-terminate after failed connection. Other party already did"); return; } sendSessionTerminate(Reason.CONNECTIVITY_ERROR); } } - public long getRtpConnectionStarted() { - return this.rtpConnectionStarted; + public boolean zeroDuration() { + return this.sessionDuration.elapsed(TimeUnit.NANOSECONDS) <= 0; } - public long getRtpConnectionEnded() { - return this.rtpConnectionEnded; + public long getCallDuration() { + return this.sessionDuration.elapsed(TimeUnit.MILLISECONDS); } public AppRTCAudioManager getAudioManager() { @@ -1297,19 +2029,31 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web } @Override - public void onAudioDeviceChanged(AppRTCAudioManager.AudioDevice selectedAudioDevice, Set availableAudioDevices) { - xmppConnectionService.notifyJingleRtpConnectionUpdate(selectedAudioDevice, availableAudioDevices); + public void onAudioDeviceChanged( + AppRTCAudioManager.AudioDevice selectedAudioDevice, + Set availableAudioDevices) { + xmppConnectionService.notifyJingleRtpConnectionUpdate( + selectedAudioDevice, availableAudioDevices); } private void updateEndUserState() { final RtpEndUserState endUserState = getEndUserState(); jingleConnectionManager.toneManager.transition(isInitiator(), endUserState, getMedia()); - xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState); + xmppConnectionService.notifyJingleRtpConnectionUpdate( + id.account, id.with, id.sessionId, endUserState); } private void updateOngoingCallNotification() { - if (STATES_SHOWING_ONGOING_CALL.contains(this.state)) { - xmppConnectionService.setOngoingCall(id, getMedia()); + final State state = this.state; + if (STATES_SHOWING_ONGOING_CALL.contains(state)) { + final boolean reconnecting; + if (state == State.SESSION_ACCEPTED) { + reconnecting = + getPeerConnectionStateAsEndUserState() == RtpEndUserState.RECONNECTING; + } else { + reconnecting = false; + } + xmppConnectionService.setOngoingCall(id, getMedia(), reconnecting); } else { xmppConnectionService.removeOngoingCall(); } @@ -1320,58 +2064,102 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web final IqPacket request = new IqPacket(IqPacket.TYPE.GET); request.setTo(id.account.getDomain()); request.addChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY); - xmppConnectionService.sendIqPacket(id.account, request, (account, response) -> { - ImmutableList.Builder listBuilder = new ImmutableList.Builder<>(); - if (response.getType() == IqPacket.TYPE.RESULT) { - final Element services = response.findChild("services", Namespace.EXTERNAL_SERVICE_DISCOVERY); - final List children = services == null ? Collections.emptyList() : services.getChildren(); - for (final Element child : children) { - if ("service".equals(child.getName())) { - final String type = child.getAttribute("type"); - final String host = child.getAttribute("host"); - final String sport = child.getAttribute("port"); - final Integer port = sport == null ? null : Ints.tryParse(sport); - final String transport = child.getAttribute("transport"); - final String username = child.getAttribute("username"); - final String password = child.getAttribute("password"); - if (Strings.isNullOrEmpty(host) || port == null) { - continue; - } - if (port < 0 || port > 65535) { - continue; - } - if (Arrays.asList("stun", "stuns", "turn", "turns").contains(type) && Arrays.asList("udp", "tcp").contains(transport)) { - if (Arrays.asList("stuns", "turns").contains(type) && "udp".equals(transport)) { - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping invalid combination of udp/tls in external services"); - continue; + xmppConnectionService.sendIqPacket( + id.account, + request, + (account, response) -> { + ImmutableList.Builder listBuilder = + new ImmutableList.Builder<>(); + if (response.getType() == IqPacket.TYPE.RESULT) { + final Element services = + response.findChild( + "services", Namespace.EXTERNAL_SERVICE_DISCOVERY); + final List children = + services == null + ? Collections.emptyList() + : services.getChildren(); + for (final Element child : children) { + if ("service".equals(child.getName())) { + final String type = child.getAttribute("type"); + final String host = child.getAttribute("host"); + final String sport = child.getAttribute("port"); + final Integer port = + sport == null ? null : Ints.tryParse(sport); + final String transport = child.getAttribute("transport"); + final String username = child.getAttribute("username"); + final String password = child.getAttribute("password"); + if (Strings.isNullOrEmpty(host) || port == null) { + continue; + } + if (port < 0 || port > 65535) { + continue; + } + if (Arrays.asList("stun", "stuns", "turn", "turns") + .contains(type) + && Arrays.asList("udp", "tcp").contains(transport)) { + if (Arrays.asList("stuns", "turns").contains(type) + && "udp".equals(transport)) { + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": skipping invalid combination of udp/tls in external services"); + continue; + } + final PeerConnection.IceServer.Builder iceServerBuilder = + PeerConnection.IceServer.builder( + String.format( + "%s:%s:%s?transport=%s", + type, + IP.wrapIPv6(host), + port, + transport)); + iceServerBuilder.setTlsCertPolicy( + PeerConnection.TlsCertPolicy + .TLS_CERT_POLICY_INSECURE_NO_CHECK); + if (username != null && password != null) { + iceServerBuilder.setUsername(username); + iceServerBuilder.setPassword(password); + } else if (Arrays.asList("turn", "turns").contains(type)) { + // The WebRTC spec requires throwing an + // InvalidAccessError when username (from libwebrtc + // source coder) + // https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": skipping " + + type + + "/" + + transport + + " without username and password"); + continue; + } + final PeerConnection.IceServer iceServer = + iceServerBuilder.createIceServer(); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": discovered ICE Server: " + + iceServer); + listBuilder.add(iceServer); + } } - final PeerConnection.IceServer.Builder iceServerBuilder = PeerConnection.IceServer - .builder(String.format("%s:%s:%s?transport=%s", type, IP.wrapIPv6(host), port, transport)); - iceServerBuilder.setTlsCertPolicy(PeerConnection.TlsCertPolicy.TLS_CERT_POLICY_INSECURE_NO_CHECK); - if (username != null && password != null) { - iceServerBuilder.setUsername(username); - iceServerBuilder.setPassword(password); - } else if (Arrays.asList("turn", "turns").contains(type)) { - //The WebRTC spec requires throwing an InvalidAccessError when username (from libwebrtc source coder) - //https://chromium.googlesource.com/external/webrtc/+/master/pc/ice_server_parsing.cc - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": skipping " + type + "/" + transport + " without username and password"); - continue; - } - final PeerConnection.IceServer iceServer = iceServerBuilder.createIceServer(); - Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": discovered ICE Server: " + iceServer); - listBuilder.add(iceServer); } } - } - } - final List iceServers = listBuilder.build(); - if (iceServers.size() == 0) { - Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": no ICE server found " + response); - } - onIceServersDiscovered.onIceServersDiscovered(iceServers); - }); + final List iceServers = listBuilder.build(); + if (iceServers.size() == 0) { + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": no ICE server found " + + response); + } + onIceServersDiscovered.onIceServersDiscovered(iceServers); + }); } else { - Log.w(Config.LOGTAG, id.account.getJid().asBareJid() + ": has no external service discovery"); + Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + ": has no external service discovery"); onIceServersDiscovered.onIceServersDiscovered(Collections.emptyList()); } } @@ -1383,14 +2171,15 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia()); this.jingleConnectionManager.finishConnectionOrThrow(this); } else { - throw new IllegalStateException(String.format("Unable to call finish from %s", this.state)); + throw new IllegalStateException( + String.format("Unable to call finish from %s", this.state)); } } private void writeLogMessage(final State state) { - final long started = this.rtpConnectionStarted; - long duration = started <= 0 ? 0 : SystemClock.elapsedRealtime() - started; - if (state == State.TERMINATED_SUCCESS || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) { + final long duration = getCallDuration(); + if (state == State.TERMINATED_SUCCESS + || (state == State.TERMINATED_CONNECTIVITY_ERROR && duration > 0)) { writeLogMessageSuccess(duration); } else { writeLogMessageMissed(); @@ -1435,7 +2224,6 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web return webRTCWrapper.getRemoteVideoTrack(); } - public EglBase.Context getEglBaseContext() { return webRTCWrapper.getEglBaseContext(); } @@ -1446,18 +2234,11 @@ public class JingleRtpConnection extends AbstractJingleConnection implements Web public void fireStateUpdate() { final RtpEndUserState endUserState = getEndUserState(); - xmppConnectionService.notifyJingleRtpConnectionUpdate(id.account, id.with, id.sessionId, endUserState); + xmppConnectionService.notifyJingleRtpConnectionUpdate( + id.account, id.with, id.sessionId, endUserState); } private interface OnIceServersDiscovered { void onIceServersDiscovered(List iceServers); } - - private static class StateTransitionException extends Exception { - private final State state; - - private StateTransitionException(final State state) { - this.state = state; - } - } } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java index 974ad4511..9a431bc01 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpEndUserState.java @@ -4,6 +4,7 @@ public enum RtpEndUserState { INCOMING_CALL, //received a 'propose' message CONNECTING, //session-initiate or session-accepted but no webrtc peer connection yet CONNECTED, //session-accepted and webrtc peer connection is connected + RECONNECTING, //session-accepted and webrtc peer connection was connected once but is currently disconnected or failed FINDING_DEVICE, //'propose' has been sent out; no 184 ack yet RINGING, //'propose' has been sent out and it has been 184 acked ACCEPTING_CALL, //'proceed' message has been sent; but no session-initiate has been received @@ -15,4 +16,4 @@ public enum RtpEndUserState { RETRACTED, //user pressed home or power button during 'ringing' - shows retry button APPLICATION_ERROR, //something rather bad happened; libwebrtc failed or we got in IQ-error SECURITY_ERROR //problem with DTLS (missing) or verification - } +} diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java index 6bc8df4fb..8a2ad80ca 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java @@ -311,7 +311,7 @@ public class WebRTCWrapper { rtcConfig.continualGatheringPolicy = PeerConnection.ContinualGatheringPolicy.GATHER_CONTINUALLY; rtcConfig.sdpSemantics = PeerConnection.SdpSemantics.UNIFIED_PLAN; rtcConfig.rtcpMuxPolicy = PeerConnection.RtcpMuxPolicy.NEGOTIATE; - //rtcConfig.enableImplicitRollback = true; + rtcConfig.enableImplicitRollback = true; return rtcConfig; } @@ -320,7 +320,7 @@ public class WebRTCWrapper { } void restartIce() { - executorService.execute(() -> requirePeerConnection()); + executorService.execute(() -> requirePeerConnection().restartIce()); } public void setIsReadyToReceiveIceCandidates(final boolean ready) { @@ -447,7 +447,20 @@ public class WebRTCWrapper { synchronized ListenableFuture setLocalDescription() { return Futures.transformAsync(getPeerConnectionFuture(), peerConnection -> { final SettableFuture future = SettableFuture.create(); + peerConnection.setLocalDescription(new SetSdpObserver() { + @Override + public void onSetSuccess() { + final SessionDescription description = peerConnection.getLocalDescription(); + Log.d(EXTENDED_LOGGING_TAG, "set local description:"); + logDescription(description); + future.set(description); + } + @Override + public void onSetFailure(final String message) { + future.setException(new FailureToSetDescriptionException(message)); + } + }); return future; }, MoreExecutors.directExecutor()); }