From 6b60529dd9209552ef842f6244f4bed8d13aa837 Mon Sep 17 00:00:00 2001 From: Arne Date: Sun, 3 Dec 2023 14:44:38 +0100 Subject: [PATCH] Rework RTP + upgrade to Java 17 fixes crash when switch from audio to videocall --- build.gradle | 4 +- .../de/monocles/chat/ConnectionService.java | 2 +- .../conversations/parser/MessageParser.java | 2 +- .../ui/ConversationFragment.java | 5 +- .../conversations/ui/util/CallManager.java | 2 +- .../xmpp/jingle/JingleConnectionManager.java | 40 +- .../xmpp/jingle/JingleRtpConnection.java | 626 +++++++++--------- .../xmpp/jingle/RtpContentMap.java | 1 + .../xmpp/jingle/SessionDescription.java | 1 - .../xmpp/jingle/WebRTCWrapper.java | 126 ++-- .../jingle/stanzas/IceUdpTransportInfo.java | 7 +- 11 files changed, 400 insertions(+), 416 deletions(-) diff --git a/build.gradle b/build.gradle index 0d5a97cd9..7eec8e97e 100644 --- a/build.gradle +++ b/build.gradle @@ -188,8 +188,8 @@ android { compileOptions { coreLibraryDesugaringEnabled true - sourceCompatibility JavaVersion.VERSION_1_9 - targetCompatibility JavaVersion.VERSION_1_9 + sourceCompatibility JavaVersion.VERSION_17 + targetCompatibility JavaVersion.VERSION_17 } flavorDimensions("distribution") diff --git a/src/main/java/de/monocles/chat/ConnectionService.java b/src/main/java/de/monocles/chat/ConnectionService.java index 9b7743889..169585f15 100644 --- a/src/main/java/de/monocles/chat/ConnectionService.java +++ b/src/main/java/de/monocles/chat/ConnectionService.java @@ -121,7 +121,7 @@ public class ConnectionService extends android.telecom.ConnectionService { ); } - if (xmppConnectionService.getJingleConnectionManager().isBusy()) { + if (xmppConnectionService.getJingleConnectionManager().isBusy() != null) { return Connection.createFailedConnection( new DisconnectCause(DisconnectCause.BUSY) ); diff --git a/src/main/java/eu/siacs/conversations/parser/MessageParser.java b/src/main/java/eu/siacs/conversations/parser/MessageParser.java index 3cb623b27..3429ed431 100644 --- a/src/main/java/eu/siacs/conversations/parser/MessageParser.java +++ b/src/main/java/eu/siacs/conversations/parser/MessageParser.java @@ -448,7 +448,7 @@ public class MessageParser extends AbstractParser implements OnMessagePacketRece } if (id.startsWith(JingleRtpConnection.JINGLE_MESSAGE_PROCEED_ID_PREFIX)) { final String sessionId = id.substring(JingleRtpConnection.JINGLE_MESSAGE_PROCEED_ID_PREFIX.length()); - mXmppConnectionService.getJingleConnectionManager().failProceed(account, from, sessionId); + mXmppConnectionService.getJingleConnectionManager().failProceed(account, from, sessionId, extractErrorMessage(packet)); //TODO: Test it and check this changes again! return true; } mXmppConnectionService.markMessage(account, diff --git a/src/main/java/eu/siacs/conversations/ui/ConversationFragment.java b/src/main/java/eu/siacs/conversations/ui/ConversationFragment.java index 7113ababf..a1151ce8a 100644 --- a/src/main/java/eu/siacs/conversations/ui/ConversationFragment.java +++ b/src/main/java/eu/siacs/conversations/ui/ConversationFragment.java @@ -2351,8 +2351,9 @@ public class ConversationFragment extends XmppFragment } private void triggerRtpSession(final String action) { - if (activity.xmppConnectionService.getJingleConnectionManager().isBusy()) { - Toast.makeText(getActivity(), R.string.only_one_call_at_a_time, Toast.LENGTH_LONG).show(); + if (activity.xmppConnectionService.getJingleConnectionManager().isBusy() != null) { + Toast.makeText(getActivity(), R.string.only_one_call_at_a_time, Toast.LENGTH_LONG) + .show(); return; } final Account account = conversation.getAccount(); diff --git a/src/main/java/eu/siacs/conversations/ui/util/CallManager.java b/src/main/java/eu/siacs/conversations/ui/util/CallManager.java index 6ffe3aaa3..439ad9301 100644 --- a/src/main/java/eu/siacs/conversations/ui/util/CallManager.java +++ b/src/main/java/eu/siacs/conversations/ui/util/CallManager.java @@ -53,7 +53,7 @@ public class CallManager { } public static void triggerRtpSession(final String action, XmppActivity activity, Conversation conversation) { - if (activity.xmppConnectionService.getJingleConnectionManager().isBusy()) { + if (activity.xmppConnectionService.getJingleConnectionManager().isBusy() != null) { ToastCompat.makeText(activity, R.string.only_one_call_at_a_time, ToastCompat.LENGTH_LONG).show(); return; } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java index d4f110608..448f1f9bd 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java @@ -36,8 +36,8 @@ import eu.siacs.conversations.entities.RtpSessionStatus; import eu.siacs.conversations.entities.Transferable; import eu.siacs.conversations.services.AbstractConnectionManager; import eu.siacs.conversations.services.XmppConnectionService; -import eu.siacs.conversations.xml.Namespace; import eu.siacs.conversations.xml.Element; +import eu.siacs.conversations.xml.Namespace; import eu.siacs.conversations.xmpp.Jid; import eu.siacs.conversations.xmpp.OnIqPacketReceived; import eu.siacs.conversations.xmpp.XmppConnection; @@ -100,7 +100,7 @@ public class JingleConnectionManager extends AbstractConnectionManager { this.terminatedSessions.asMap().containsKey(PersistableSessionId.of(id)); final boolean stranger = isWithStrangerAndStrangerNotificationsAreOff(account, id.with); - final boolean busy = isBusy(); + final boolean busy = isBusy() != null; if (busy || sessionEnded || stranger) { Log.d( Config.LOGTAG, @@ -145,26 +145,26 @@ public class JingleConnectionManager extends AbstractConnectionManager { } private boolean isUsingClearNet(final Account account) { - return !account.isOnion() && !mXmppConnectionService.useTorToConnect() && !account.isI2P() && !mXmppConnectionService.useI2PToConnect(); + return !account.isOnion() && !mXmppConnectionService.useTorToConnect(); } - public boolean isBusy() { + public String isBusy() { if (mXmppConnectionService.isPhoneInCall()) { - return true; + return "isPhoneInCall"; } for (AbstractJingleConnection connection : this.connections.values()) { if (connection instanceof JingleRtpConnection) { if (((JingleRtpConnection) connection).isTerminated()) { continue; } - return true; + return "connection !isTerminated"; } } synchronized (this.rtpSessionProposals) { - return this.rtpSessionProposals.containsValue(DeviceDiscoveryState.DISCOVERED) - || this.rtpSessionProposals.containsValue(DeviceDiscoveryState.SEARCHING) - || this.rtpSessionProposals.containsValue( - DeviceDiscoveryState.SEARCHING_ACKNOWLEDGED); + if (this.rtpSessionProposals.containsValue(DeviceDiscoveryState.DISCOVERED)) return "discovered"; + if (this.rtpSessionProposals.containsValue(DeviceDiscoveryState.SEARCHING)) return "searching"; + if (this.rtpSessionProposals.containsValue(DeviceDiscoveryState.SEARCHING_ACKNOWLEDGED)) return "searching_acknolwedged"; + return null; } } @@ -395,18 +395,20 @@ public class JingleConnectionManager extends AbstractConnectionManager { this.connections.put(id, rtpConnection); rtpConnection.setProposedMedia(ImmutableSet.copyOf(media)); rtpConnection.deliveryMessage(from, message, serverMsgId, timestamp); + // TODO actually do the automatic accept?! } else { Log.d( Config.LOGTAG, account.getJid().asBareJid() + ": our session won tie break. waiting for other party to accept. winningSession=" + ourSessionId); + // TODO reject their session with ? } return; } final boolean stranger = isWithStrangerAndStrangerNotificationsAreOff(account, id.with); - if (isBusy() || stranger) { + if (isBusy() != null || stranger) { writeLogMissedIncoming( account, id.with.asBareJid(), @@ -786,19 +788,23 @@ public class JingleConnectionManager extends AbstractConnectionManager { final RtpEndUserState endUserState = preexistingState.toEndUserState(); toneManager.transition(endUserState, media); mXmppConnectionService.notifyJingleRtpConnectionUpdate( - account, with, proposal.sessionId, endUserState); + account, + with, + proposal.sessionId, + endUserState + ); return proposal.sessionId; } } } - if (isBusy()) { + String busyCode = isBusy(); + if (busyCode != null) { String sessionId = hasMatchingRtpSession(account, with, media); if (sessionId != null) { Log.d(Config.LOGTAG, "ignoring request to propose jingle session because the other party already created one for us: " + sessionId); return sessionId; } - throw new IllegalStateException( - "There is already a running RTP session. This should have been caught by the UI"); + throw new IllegalStateException("There is already a running RTP session: " + busyCode); } final RtpSessionProposal proposal = RtpSessionProposal.of(account, with.asBareJid(), media); @@ -964,12 +970,12 @@ public class JingleConnectionManager extends AbstractConnectionManager { } } - public void failProceed(Account account, final Jid with, String sessionId) { + public void failProceed(Account account, final Jid with, final String sessionId, final String message) { final AbstractJingleConnection.Id id = AbstractJingleConnection.Id.of(account, with, sessionId); final AbstractJingleConnection existingJingleConnection = connections.get(id); if (existingJingleConnection instanceof JingleRtpConnection) { - ((JingleRtpConnection) existingJingleConnection).deliverFailedProceed(); + ((JingleRtpConnection) existingJingleConnection).deliverFailedProceed(message); } } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java index b686cbfbd..46a8bd892 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/JingleRtpConnection.java @@ -1,6 +1,7 @@ package eu.siacs.conversations.xmpp.jingle; import android.util.Log; +import android.os.Environment; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -14,7 +15,9 @@ import com.google.common.base.Throwables; import com.google.common.collect.Collections2; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableMultimap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Iterables; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; @@ -22,28 +25,6 @@ import com.google.common.util.concurrent.FutureCallback; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; -import com.google.common.collect.ImmutableMultimap; -import com.google.common.collect.Iterables; -import com.google.common.collect.Multimap; - - -import org.webrtc.EglBase; -import org.webrtc.IceCandidate; -import org.webrtc.PeerConnection; -import org.webrtc.VideoTrack; -import org.webrtc.DtmfSender; - -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; import eu.siacs.conversations.BuildConfig; import eu.siacs.conversations.Config; @@ -60,10 +41,9 @@ import eu.siacs.conversations.entities.RtpSessionStatus; import eu.siacs.conversations.entities.ServiceDiscoveryResult; import eu.siacs.conversations.services.AppRTCAudioManager; import eu.siacs.conversations.utils.IP; -import eu.siacs.conversations.xml.Namespace; import eu.siacs.conversations.xml.Element; +import eu.siacs.conversations.xml.Namespace; import eu.siacs.conversations.xmpp.Jid; -import eu.siacs.conversations.xmpp.OnIqPacketReceived; import eu.siacs.conversations.xmpp.jingle.stanzas.Content; import eu.siacs.conversations.xmpp.jingle.stanzas.Group; import eu.siacs.conversations.xmpp.jingle.stanzas.IceUdpTransportInfo; @@ -75,6 +55,26 @@ import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription; import eu.siacs.conversations.xmpp.stanzas.IqPacket; import eu.siacs.conversations.xmpp.stanzas.MessagePacket; +import org.webrtc.DtmfSender; +import org.webrtc.EglBase; +import org.webrtc.IceCandidate; +import org.webrtc.PeerConnection; +import org.webrtc.VideoTrack; + +import java.io.File; +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ScheduledFuture; +import java.util.concurrent.TimeUnit; + public class JingleRtpConnection extends AbstractJingleConnection implements WebRTCWrapper.EventCallback { @@ -181,6 +181,7 @@ public class JingleRtpConnection extends AbstractJingleConnection private final Stopwatch sessionDuration = Stopwatch.createUnstarted(); private final Queue stateHistory = new LinkedList<>(); private ScheduledFuture ringingTimeoutFuture; + private final long created = System.currentTimeMillis() / 1000L; JingleRtpConnection(JingleConnectionManager jingleConnectionManager, Id id, Jid initiator) { super(jingleConnectionManager, id, initiator); @@ -197,64 +198,37 @@ public class JingleRtpConnection extends AbstractJingleConnection } private static State reasonToState(Reason reason) { - switch (reason) { - case SUCCESS: - return State.TERMINATED_SUCCESS; - case DECLINE: - case BUSY: - return State.TERMINATED_DECLINED_OR_BUSY; - case CANCEL: - case TIMEOUT: - return State.TERMINATED_CANCEL_OR_TIMEOUT; - case SECURITY_ERROR: - return State.TERMINATED_SECURITY_ERROR; - case FAILED_APPLICATION: - case UNSUPPORTED_TRANSPORTS: - case UNSUPPORTED_APPLICATIONS: - return State.TERMINATED_APPLICATION_FAILURE; - default: - return State.TERMINATED_CONNECTIVITY_ERROR; - } + return switch (reason) { + case SUCCESS -> State.TERMINATED_SUCCESS; + case DECLINE, BUSY -> State.TERMINATED_DECLINED_OR_BUSY; + case CANCEL, TIMEOUT -> State.TERMINATED_CANCEL_OR_TIMEOUT; + case SECURITY_ERROR -> State.TERMINATED_SECURITY_ERROR; + case FAILED_APPLICATION, UNSUPPORTED_TRANSPORTS, UNSUPPORTED_APPLICATIONS -> State + .TERMINATED_APPLICATION_FAILURE; + default -> State.TERMINATED_CONNECTIVITY_ERROR; + }; } @Override synchronized void deliverPacket(final JinglePacket jinglePacket) { switch (jinglePacket.getAction()) { - case SESSION_INITIATE: - receiveSessionInitiate(jinglePacket); - break; - case TRANSPORT_INFO: - receiveTransportInfo(jinglePacket); - break; - case SESSION_ACCEPT: - receiveSessionAccept(jinglePacket); - break; - case SESSION_TERMINATE: - receiveSessionTerminate(jinglePacket); - break; - case CONTENT_ADD: - receiveContentAdd(jinglePacket); - break; - case CONTENT_ACCEPT: - receiveContentAccept(jinglePacket); - break; - case CONTENT_REJECT: - receiveContentReject(jinglePacket); - break; - case CONTENT_REMOVE: - receiveContentRemove(jinglePacket); - break; - case CONTENT_MODIFY: - receiveContentModify(jinglePacket); - break; - default: + case SESSION_INITIATE -> receiveSessionInitiate(jinglePacket); + case TRANSPORT_INFO -> receiveTransportInfo(jinglePacket); + case SESSION_ACCEPT -> receiveSessionAccept(jinglePacket); + case SESSION_TERMINATE -> receiveSessionTerminate(jinglePacket); + case CONTENT_ADD -> receiveContentAdd(jinglePacket); + case CONTENT_ACCEPT -> receiveContentAccept(jinglePacket); + case CONTENT_REJECT -> receiveContentReject(jinglePacket); + case CONTENT_REMOVE -> receiveContentRemove(jinglePacket); + case CONTENT_MODIFY -> receiveContentModify(jinglePacket); + default -> { respondOk(jinglePacket); Log.d( Config.LOGTAG, String.format( "%s: received unhandled jingle action %s", id.account.getJid().asBareJid(), jinglePacket.getAction())); - break; + } } } @@ -360,15 +334,22 @@ public class JingleRtpConnection extends AbstractJingleConnection final Set> candidates = contentMap.contents.entrySet(); final RtpContentMap remote = getRemoteContentMap(); - final Set remoteContentIds = remote == null ? Collections.emptySet() : remote.contents.keySet(); + final Set remoteContentIds = + remote == null ? Collections.emptySet() : remote.contents.keySet(); if (Collections.disjoint(remoteContentIds, contentMap.contents.keySet())) { - Log.d(Config.LOGTAG,"received transport-info for unknown contents "+contentMap.contents.keySet()+" (known: "+remoteContentIds+")"); + Log.d( + Config.LOGTAG, + "received transport-info for unknown contents " + + contentMap.contents.keySet() + + " (known: " + + remoteContentIds + + ")"); respondOk(jinglePacket); pendingIceCandidates.addAll(candidates); return; } if (this.state != State.SESSION_ACCEPTED) { - Log.d(Config.LOGTAG,"received transport-info prematurely. adding to backlog"); + Log.d(Config.LOGTAG, "received transport-info prematurely. adding to backlog"); respondOk(jinglePacket); pendingIceCandidates.addAll(candidates); return; @@ -388,19 +369,6 @@ public class JingleRtpConnection extends AbstractJingleConnection } } - private void rejectContent(final RtpContentMap contentMap) { - final JinglePacket jinglePacket = - contentMap - .toStub() - .toJinglePacket(JinglePacket.Action.CONTENT_REJECT, id.sessionId); - Log.d( - Config.LOGTAG, - id.getAccount().getJid().asBareJid() - + ": rejecting content " - + ContentAddition.summary(contentMap)); - send(jinglePacket); - } - private void receiveContentAdd(final JinglePacket jinglePacket) { final RtpContentMap modification; try { @@ -420,26 +388,31 @@ public class JingleRtpConnection extends AbstractJingleConnection final boolean hasFullTransportInfo = modification.hasFullTransportInfo(); final ListenableFuture future = receiveRtpContentMap( - modification, this.omemoVerification.hasFingerprint() && hasFullTransportInfo); - Futures.addCallback(future, new FutureCallback() { - @Override - public void onSuccess(final RtpContentMap rtpContentMap) { - receiveContentAdd(jinglePacket, rtpContentMap); - } + modification, + this.omemoVerification.hasFingerprint() && hasFullTransportInfo); + Futures.addCallback( + future, + new FutureCallback<>() { + @Override + public void onSuccess(final RtpContentMap rtpContentMap) { + receiveContentAdd(jinglePacket, rtpContentMap); + } - @Override - public void onFailure(@NonNull Throwable throwable) { - respondOk(jinglePacket); - final Throwable rootCause = Throwables.getRootCause(throwable); - Log.d( - Config.LOGTAG, - id.account.getJid().asBareJid() - + ": improperly formatted contents in content-add", - throwable); - webRTCWrapper.close(); - sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage()); - } - }, MoreExecutors.directExecutor()); + @Override + public void onFailure(@NonNull Throwable throwable) { + respondOk(jinglePacket); + final Throwable rootCause = Throwables.getRootCause(throwable); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": improperly formatted contents in content-add", + throwable); + webRTCWrapper.close(); + sendSessionTerminate( + Reason.ofThrowable(rootCause), rootCause.getMessage()); + } + }, + MoreExecutors.directExecutor()); } else { terminateWithOutOfOrder(jinglePacket); } @@ -527,19 +500,24 @@ public class JingleRtpConnection extends AbstractJingleConnection final boolean hasFullTransportInfo = receivedContentAccept.hasFullTransportInfo(); final ListenableFuture future = receiveRtpContentMap( - receivedContentAccept, this.omemoVerification.hasFingerprint() && hasFullTransportInfo); - Futures.addCallback(future, new FutureCallback() { - @Override - public void onSuccess(final RtpContentMap result) { - receiveContentAccept(result); - } + receivedContentAccept, + this.omemoVerification.hasFingerprint() && hasFullTransportInfo); + Futures.addCallback( + future, + new FutureCallback<>() { + @Override + public void onSuccess(final RtpContentMap result) { + receiveContentAccept(result); + } - @Override - public void onFailure(@NonNull final Throwable throwable) { - webRTCWrapper.close(); - sendSessionTerminate(Reason.ofThrowable(throwable), throwable.getMessage()); - } - }, MoreExecutors.directExecutor()); + @Override + public void onFailure(@NonNull final Throwable throwable) { + webRTCWrapper.close(); + sendSessionTerminate( + Reason.ofThrowable(throwable), throwable.getMessage()); + } + }, + MoreExecutors.directExecutor()); } else { Log.d(Config.LOGTAG, "received content-accept did not match our outgoing content-add"); terminateWithOutOfOrder(jinglePacket); @@ -592,25 +570,34 @@ public class JingleRtpConnection extends AbstractJingleConnection final boolean isInitiator = isInitiator(); final RtpContentMap currentOutgoing = this.outgoingContentAdd; final RtpContentMap remoteContentMap = this.getRemoteContentMap(); - final Set currentOutgoingMediaIds = currentOutgoing == null ? Collections.emptySet() : currentOutgoing.contents.keySet(); + final Set currentOutgoingMediaIds = + currentOutgoing == null + ? Collections.emptySet() + : currentOutgoing.contents.keySet(); Log.d(Config.LOGTAG, "receiveContentModification(" + modification + ")"); if (currentOutgoing != null && currentOutgoingMediaIds.containsAll(modification.keySet())) { respondOk(jinglePacket); final RtpContentMap modifiedContentMap; try { - modifiedContentMap = currentOutgoing.modifiedSendersChecked(isInitiator, modification); + modifiedContentMap = + currentOutgoing.modifiedSendersChecked(isInitiator, modification); } catch (final IllegalArgumentException e) { webRTCWrapper.close(); sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); return; } this.outgoingContentAdd = modifiedContentMap; - Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": processed content-modification for pending content-add"); - } else if (remoteContentMap != null && remoteContentMap.contents.keySet().containsAll(modification.keySet())) { + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": processed content-modification for pending content-add"); + } else if (remoteContentMap != null + && remoteContentMap.contents.keySet().containsAll(modification.keySet())) { respondOk(jinglePacket); final RtpContentMap modifiedRemoteContentMap; try { - modifiedRemoteContentMap = remoteContentMap.modifiedSendersChecked(isInitiator, modification); + modifiedRemoteContentMap = + remoteContentMap.modifiedSendersChecked(isInitiator, modification); } catch (final IllegalArgumentException e) { webRTCWrapper.close(); sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); @@ -620,20 +607,27 @@ public class JingleRtpConnection extends AbstractJingleConnection try { offer = SessionDescription.of(modifiedRemoteContentMap, !isInitiator()); } catch (final IllegalArgumentException | NullPointerException e) { - Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-modify to SDP", e); + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + + ": unable convert offer from content-modify to SDP", + e); webRTCWrapper.close(); sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); return; } - Log.d(Config.LOGTAG, id.account.getJid().asBareJid()+": auto accepting content-modification"); + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + ": auto accepting content-modification"); this.autoAcceptContentModify(modifiedRemoteContentMap, offer); } else { - Log.d(Config.LOGTAG,"received unsupported content modification "+modification); + Log.d(Config.LOGTAG, "received unsupported content modification " + modification); respondWithItemNotFound(jinglePacket); } } - private void autoAcceptContentModify(final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) { + private void autoAcceptContentModify( + final RtpContentMap modifiedRemoteContentMap, final SessionDescription offer) { this.setRemoteContentMap(modifiedRemoteContentMap); final org.webrtc.SessionDescription sdp = new org.webrtc.SessionDescription( @@ -644,8 +638,9 @@ public class JingleRtpConnection extends AbstractJingleConnection final SessionDescription answer = setLocalSessionDescription(); final RtpContentMap rtpContentMap = RtpContentMap.of(answer, isInitiator()); modifyLocalContentMap(rtpContentMap); - // we do not need to send an answer but do we have to resend the candidates currently in SDP? - //resendCandidatesFromSdp(answer); + // we do not need to send an answer but do we have to resend the candidates currently in + // SDP? + // resendCandidatesFromSdp(answer); webRTCWrapper.setIsReadyToReceiveIceCandidates(true); } catch (final Exception e) { Log.d(Config.LOGTAG, "unable to accept content add", Throwables.getRootCause(e)); @@ -654,19 +649,20 @@ public class JingleRtpConnection extends AbstractJingleConnection } } - - - private static ImmutableMultimap parseCandidates(final SessionDescription answer) { - final ImmutableMultimap.Builder candidateBuilder = new ImmutableMultimap.Builder<>(); - for(final SessionDescription.Media media : answer.media) { + private static ImmutableMultimap parseCandidates( + final SessionDescription answer) { + final ImmutableMultimap.Builder candidateBuilder = + new ImmutableMultimap.Builder<>(); + for (final SessionDescription.Media media : answer.media) { final String mid = Iterables.getFirst(media.attributes.get("mid"), null); if (Strings.isNullOrEmpty(mid)) { continue; } - for(final String sdpCandidate : media.attributes.get("candidate")) { - final IceUdpTransportInfo.Candidate candidate = IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null); + for (final String sdpCandidate : media.attributes.get("candidate")) { + final IceUdpTransportInfo.Candidate candidate = + IceUdpTransportInfo.Candidate.fromSdpAttributeValue(sdpCandidate, null); if (candidate != null) { - candidateBuilder.put(mid,candidate); + candidateBuilder.put(mid, candidate); } } } @@ -698,7 +694,7 @@ public class JingleRtpConnection extends AbstractJingleConnection if (ourSummary.equals(ContentAddition.summary(receivedContentReject))) { this.outgoingContentAdd = null; respondOk(jinglePacket); - Log.d(Config.LOGTAG,jinglePacket.toString()); + Log.d(Config.LOGTAG, jinglePacket.toString()); receiveContentReject(ourSummary); } else { Log.d(Config.LOGTAG, "received content-reject did not match our outgoing content-add"); @@ -765,7 +761,7 @@ public class JingleRtpConnection extends AbstractJingleConnection Reason.FAILED_APPLICATION, String.format( "%s only supports %s as a means to retract a not yet accepted %s", - BuildConfig.LOGTAG, + "monocles chat", JinglePacket.Action.CONTENT_REMOVE, JinglePacket.Action.CONTENT_ADD)); } @@ -834,7 +830,8 @@ public class JingleRtpConnection extends AbstractJingleConnection "Unexpected rollback condition. Senders were not uniformly none"); } - public synchronized void acceptContentAdd(@NonNull final Set contentAddition) { + public synchronized void acceptContentAdd( + @NonNull final Set contentAddition) { final RtpContentMap incomingContentAdd = this.incomingContentAdd; if (incomingContentAdd == null) { throw new IllegalStateException("No incoming content add"); @@ -843,37 +840,63 @@ public class JingleRtpConnection extends AbstractJingleConnection if (contentAddition.equals(ContentAddition.summary(incomingContentAdd))) { this.incomingContentAdd = null; final Set senders = incomingContentAdd.getSenders(); - Log.d(Config.LOGTAG,"senders of incoming content-add: "+senders); + Log.d(Config.LOGTAG, "senders of incoming content-add: " + senders); if (senders.equals(Content.Senders.receiveOnly(isInitiator()))) { - Log.d(Config.LOGTAG,"content addition is receive only. we want to upgrade to 'both'"); - final RtpContentMap modifiedSenders = incomingContentAdd.modifiedSenders(Content.Senders.BOTH); - final JinglePacket proposedContentModification = modifiedSenders.toStub().toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId); + Log.d( + Config.LOGTAG, + "content addition is receive only. we want to upgrade to 'both'"); + final RtpContentMap modifiedSenders = + incomingContentAdd.modifiedSenders(Content.Senders.BOTH); + final JinglePacket proposedContentModification = + modifiedSenders + .toStub() + .toJinglePacket(JinglePacket.Action.CONTENT_MODIFY, id.sessionId); proposedContentModification.setTo(id.with); - xmppConnectionService.sendIqPacket(id.account, proposedContentModification, (account, response) -> { - if (response.getType() == IqPacket.TYPE.RESULT) { - Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has accepted our upgrade to senders=both"); - acceptContentAdd(ContentAddition.summary(modifiedSenders), modifiedSenders); - } else { - Log.d(Config.LOGTAG,id.account.getJid().asBareJid()+": remote has rejected our upgrade to senders=both"); - acceptContentAdd(contentAddition, incomingContentAdd); - } - }); + xmppConnectionService.sendIqPacket( + id.account, + proposedContentModification, + (account, response) -> { + if (response.getType() == IqPacket.TYPE.RESULT) { + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": remote has accepted our upgrade to senders=both"); + acceptContentAdd( + ContentAddition.summary(modifiedSenders), modifiedSenders); + } else { + Log.d( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": remote has rejected our upgrade to senders=both"); + acceptContentAdd(contentAddition, incomingContentAdd); + } + }); + } else { + acceptContentAdd(contentAddition, incomingContentAdd); } } else { - throw new IllegalStateException("Accepted content add does not match pending content-add"); + throw new IllegalStateException( + "Accepted content add does not match pending content-add"); } } - private void acceptContentAdd(@NonNull final Set contentAddition, final RtpContentMap incomingContentAdd) { + private void acceptContentAdd( + @NonNull final Set contentAddition, + final RtpContentMap incomingContentAdd) { final IceUdpTransportInfo.Setup setup = getPeerDtlsSetup(); - final RtpContentMap modifiedContentMap = getRemoteContentMap().addContent(incomingContentAdd, setup); + final RtpContentMap modifiedContentMap = + getRemoteContentMap().addContent(incomingContentAdd, setup); this.setRemoteContentMap(modifiedContentMap); final SessionDescription offer; try { offer = SessionDescription.of(modifiedContentMap, !isInitiator()); } catch (final IllegalArgumentException | NullPointerException e) { - Log.d(Config.LOGTAG, id.getAccount().getJid().asBareJid() + ": unable convert offer from content-add to SDP", e); + Log.d( + Config.LOGTAG, + id.getAccount().getJid().asBareJid() + + ": unable convert offer from content-add to SDP", + e); webRTCWrapper.close(); sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); return; @@ -914,10 +937,11 @@ public class JingleRtpConnection extends AbstractJingleConnection addIceCandidatesFromBlackLog(); modifyLocalContentMap(rtpContentMap); - final ListenableFuture future = prepareOutgoingContentMap(contentAcceptMap); + final ListenableFuture future = + prepareOutgoingContentMap(contentAcceptMap); Futures.addCallback( future, - new FutureCallback() { + new FutureCallback<>() { @Override public void onSuccess(final RtpContentMap rtpContentMap) { sendContentAccept(rtpContentMap); @@ -938,7 +962,8 @@ public class JingleRtpConnection extends AbstractJingleConnection } private void sendContentAccept(final RtpContentMap contentAcceptMap) { - final JinglePacket jinglePacket = contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId); + final JinglePacket jinglePacket = + contentAcceptMap.toJinglePacket(JinglePacket.Action.CONTENT_ACCEPT, id.sessionId); send(jinglePacket); } @@ -965,7 +990,6 @@ public class JingleRtpConnection extends AbstractJingleConnection send(jinglePacket); } - private boolean checkForIceRestart( final JinglePacket jinglePacket, final RtpContentMap rtpContentMap) { final RtpContentMap existing = getRemoteContentMap(); @@ -985,7 +1009,8 @@ public class JingleRtpConnection extends AbstractJingleConnection // ICE-restart // and if that's the case we are seeing an answer. // This might be more spec compliant but also more error prone potentially - final boolean isSignalStateStable = this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE; + final boolean isSignalStateStable = + this.webRTCWrapper.getSignalingState() == PeerConnection.SignalingState.STABLE; // TODO a stable signal state can be another indicator that we have an offer to restart ICE final boolean isOffer = rtpContentMap.emptyCandidates(); final RtpContentMap restartContentMap; @@ -1049,7 +1074,8 @@ public class JingleRtpConnection extends AbstractJingleConnection final RtpContentMap restartContentMap, final boolean isOffer) throws ExecutionException, InterruptedException { - final SessionDescription sessionDescription = SessionDescription.of(restartContentMap, !isInitiator()); + final SessionDescription sessionDescription = + SessionDescription.of(restartContentMap, !isInitiator()); final org.webrtc.SessionDescription.Type type = isOffer ? org.webrtc.SessionDescription.Type.OFFER @@ -1148,8 +1174,10 @@ public class JingleRtpConnection extends AbstractJingleConnection } catch (final Exception e) { return Futures.immediateFailedFuture(e); } - } - private ListenableFuture receiveRtpContentMap(final RtpContentMap receivedContentMap, final boolean expectVerification) { + } + + private ListenableFuture receiveRtpContentMap( + final RtpContentMap receivedContentMap, final boolean expectVerification) { Log.d( Config.LOGTAG, "receiveRtpContentMap(" @@ -1205,7 +1233,7 @@ public class JingleRtpConnection extends AbstractJingleConnection final ListenableFuture future = receiveRtpContentMap(jinglePacket, false); Futures.addCallback( future, - new FutureCallback() { + new FutureCallback<>() { @Override public void onSuccess(@Nullable RtpContentMap rtpContentMap) { receiveSessionInitiate(jinglePacket, rtpContentMap); @@ -1294,7 +1322,7 @@ public class JingleRtpConnection extends AbstractJingleConnection receiveRtpContentMap(jinglePacket, this.omemoVerification.hasFingerprint()); Futures.addCallback( future, - new FutureCallback() { + new FutureCallback<>() { @Override public void onSuccess(@Nullable RtpContentMap rtpContentMap) { receiveSessionAccept(jinglePacket, rtpContentMap); @@ -1378,15 +1406,14 @@ public class JingleRtpConnection extends AbstractJingleConnection try { this.webRTCWrapper.setRemoteDescription(answer).get(); } catch (final Exception e) { - final Throwable cause = Throwables.getRootCause(e); Log.d( Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to set remote description after receiving session-accept", - cause); + Throwables.getRootCause(e)); webRTCWrapper.close(); sendSessionTerminate( - Reason.FAILED_APPLICATION, cause.getMessage()); + Reason.FAILED_APPLICATION, Throwables.getRootCause(e).getMessage()); return; } processCandidates(contentMap.contents.entrySet()); @@ -1434,7 +1461,7 @@ public class JingleRtpConnection extends AbstractJingleConnection } catch (final WebRTCWrapper.InitializationException e) { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": unable to initialize WebRTC"); webRTCWrapper.close(); - sendSessionTerminate(Reason.FAILED_APPLICATION); + sendSessionTerminate(Reason.FAILED_APPLICATION, e.getMessage()); return; } final org.webrtc.SessionDescription sdp = @@ -1461,7 +1488,8 @@ public class JingleRtpConnection extends AbstractJingleConnection sendSessionTerminate(Reason.ofThrowable(rootCause), rootCause.getMessage()); } - private void failureToPerformAction(final JinglePacket.Action action, final Throwable throwable) { + private void failureToPerformAction( + final JinglePacket.Action action, final Throwable throwable) { if (isTerminated()) { return; } @@ -1482,7 +1510,8 @@ public class JingleRtpConnection extends AbstractJingleConnection } private void prepareSessionAccept( - final org.webrtc.SessionDescription webRTCSessionDescription, final boolean includeCandidates) { + final org.webrtc.SessionDescription webRTCSessionDescription, + final boolean includeCandidates) { final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description); final RtpContentMap respondingRtpContentMap = RtpContentMap.of(sessionDescription, false); @@ -1498,7 +1527,7 @@ public class JingleRtpConnection extends AbstractJingleConnection prepareOutgoingContentMap(respondingRtpContentMap); Futures.addCallback( outgoingContentMapFuture, - new FutureCallback() { + new FutureCallback<>() { @Override public void onSuccess(final RtpContentMap outgoingContentMap) { if (includeCandidates) { @@ -1571,30 +1600,23 @@ public class JingleRtpConnection extends AbstractJingleConnection + ": delivered message to JingleRtpConnection " + message); switch (message.getName()) { - case "propose": - receivePropose(from, Propose.upgrade(message), serverMessageId, timestamp); - break; - case "proceed": - receiveProceed(from, Proceed.upgrade(message), serverMessageId, timestamp); - break; - case "retract": - receiveRetract(from, serverMessageId, timestamp); - break; - case "reject": - receiveReject(from, serverMessageId, timestamp); - break; - case "accept": - receiveAccept(from, serverMessageId, timestamp); - break; - default: - break; + case "propose" -> receivePropose( + from, Propose.upgrade(message), serverMessageId, timestamp); + case "proceed" -> receiveProceed( + from, Proceed.upgrade(message), serverMessageId, timestamp); + case "retract" -> receiveRetract(from, serverMessageId, timestamp); + case "reject" -> receiveReject(from, serverMessageId, timestamp); + case "accept" -> receiveAccept(from, serverMessageId, timestamp); } } - void deliverFailedProceed() { + void deliverFailedProceed(final String message) { Log.d( Config.LOGTAG, - id.account.getJid().asBareJid() + ": receive message error for proceed message"); + id.account.getJid().asBareJid() + + ": receive message error for proceed message (" + + Strings.nullToEmpty(message) + + ")"); if (transition(State.TERMINATED_CONNECTIVITY_ERROR)) { webRTCWrapper.close(); Log.d( @@ -1632,9 +1654,7 @@ public class JingleRtpConnection extends AbstractJingleConnection this.message.setTime(timestamp); this.message.setCarbon(true); // indicate that call was accepted on other device this.writeLogMessageSuccess(0); - this.xmppConnectionService - .getNotificationService() - .cancelIncomingCallNotification(); + this.xmppConnectionService.getNotificationService().cancelIncomingCallNotification(); this.finish(); } @@ -1766,22 +1786,22 @@ public class JingleRtpConnection extends AbstractJingleConnection ringingTimeoutFuture = jingleConnectionManager.schedule( this::ringingTimeout, BUSY_TIME_OUT, TimeUnit.SECONDS); - final String uuid = jingleConnectionManager.getXmppConnectionService().findOrCreateConversation(id.account, id.with.asBareJid(), false, false).getUuid(); - xmppConnectionService.getNotificationService().showIncomingCallNotification(id, getMedia(), uuid); + xmppConnectionService.getNotificationService().startRinging(id, getMedia()); } private synchronized void ringingTimeout() { Log.d(Config.LOGTAG, id.account.getJid().asBareJid() + ": timeout reached for ringing"); switch (this.state) { - case PROPOSED: + case PROPOSED -> { message.markUnread(); rejectCallFromProposed(); - break; - case SESSION_INITIALIZED: + } + case SESSION_INITIALIZED -> { message.markUnread(); rejectCallFromSessionInitiate(); - break; + } } + xmppConnectionService.getNotificationService().pushMissedCallNow(message); } private void cancelRingingTimeout() { @@ -1856,6 +1876,7 @@ public class JingleRtpConnection extends AbstractJingleConnection this.state == State.PROCEED ? State.RETRACTED_RACED : State.RETRACTED; if (transition(target)) { xmppConnectionService.getNotificationService().cancelIncomingCallNotification(); + xmppConnectionService.getNotificationService().pushMissedCallNow(message); Log.d( Config.LOGTAG, id.account.getJid().asBareJid() @@ -1872,7 +1893,6 @@ public class JingleRtpConnection extends AbstractJingleConnection this.message.markUnread(); } writeLogMessageMissed(); - xmppConnectionService.getNotificationService().pushMissedCallNow(message); finish(); } else { Log.d(Config.LOGTAG, "ignoring retract because already in " + this.state); @@ -1941,7 +1961,7 @@ public class JingleRtpConnection extends AbstractJingleConnection webRTCWrapper.close(); final Reason reason = Reason.ofThrowable(throwable); if (isInState(targetState)) { - sendSessionTerminate(reason); + sendSessionTerminate(reason, throwable.getMessage()); } else { sendRetract(reason); } @@ -1955,7 +1975,9 @@ public class JingleRtpConnection extends AbstractJingleConnection } private void prepareSessionInitiate( - final org.webrtc.SessionDescription webRTCSessionDescription, final boolean includeCandidates, final State targetState) { + final org.webrtc.SessionDescription webRTCSessionDescription, + final boolean includeCandidates, + final State targetState) { final SessionDescription sessionDescription = SessionDescription.parse(webRTCSessionDescription.description); final RtpContentMap rtpContentMap = RtpContentMap.of(sessionDescription, true); @@ -1970,7 +1992,7 @@ public class JingleRtpConnection extends AbstractJingleConnection encryptSessionInitiate(rtpContentMap); Futures.addCallback( outgoingContentMapFuture, - new FutureCallback() { + new FutureCallback<>() { @Override public void onSuccess(final RtpContentMap outgoingContentMap) { if (includeCandidates) { @@ -1979,7 +2001,8 @@ public class JingleRtpConnection extends AbstractJingleConnection "including " + candidates.size() + " candidates in session initiate"); - sendSessionInitiate(outgoingContentMap.withCandidates(candidates), targetState); + sendSessionInitiate( + outgoingContentMap.withCandidates(candidates), targetState); webRTCWrapper.resetPendingCandidates(); } else { sendSessionInitiate(outgoingContentMap, targetState); @@ -2122,10 +2145,10 @@ public class JingleRtpConnection extends AbstractJingleConnection this.webRTCWrapper.close(); final State target; if (Arrays.asList( - "service-unavailable", - "recipient-unavailable", - "remote-server-not-found", - "remote-server-timeout") + "service-unavailable", + "recipient-unavailable", + "remote-server-not-found", + "remote-server-timeout") .contains(errorCondition)) { target = State.TERMINATED_CONNECTIVITY_ERROR; } else { @@ -2193,59 +2216,62 @@ public class JingleRtpConnection extends AbstractJingleConnection public RtpEndUserState getEndUserState() { switch (this.state) { - case NULL: - case PROPOSED: - case SESSION_INITIALIZED: + case NULL, PROPOSED, SESSION_INITIALIZED -> { if (isInitiator()) { return RtpEndUserState.RINGING; } else { return RtpEndUserState.INCOMING_CALL; } - case PROCEED: + } + case PROCEED -> { if (isInitiator()) { return RtpEndUserState.RINGING; } else { return RtpEndUserState.ACCEPTING_CALL; } - case SESSION_INITIALIZED_PRE_APPROVED: + } + case SESSION_INITIALIZED_PRE_APPROVED -> { if (isInitiator()) { return RtpEndUserState.RINGING; } else { return RtpEndUserState.CONNECTING; } - case SESSION_ACCEPTED: + } + case SESSION_ACCEPTED -> { final ContentAddition ca = getPendingContentAddition(); if (ca != null && ca.direction == ContentAddition.Direction.INCOMING) { return RtpEndUserState.INCOMING_CONTENT_ADD; } return getPeerConnectionStateAsEndUserState(); - case REJECTED: - case REJECTED_RACED: - case TERMINATED_DECLINED_OR_BUSY: + } + case REJECTED, REJECTED_RACED, TERMINATED_DECLINED_OR_BUSY -> { if (isInitiator()) { return RtpEndUserState.DECLINED_OR_BUSY; } else { return RtpEndUserState.ENDED; } - case TERMINATED_SUCCESS: - case ACCEPTED: - case RETRACTED: - case TERMINATED_CANCEL_OR_TIMEOUT: + } + case TERMINATED_SUCCESS, ACCEPTED, RETRACTED, TERMINATED_CANCEL_OR_TIMEOUT -> { return RtpEndUserState.ENDED; - case RETRACTED_RACED: + } + case RETRACTED_RACED -> { if (isInitiator()) { return RtpEndUserState.ENDED; } else { return RtpEndUserState.RETRACTED; } - case TERMINATED_CONNECTIVITY_ERROR: + } + case TERMINATED_CONNECTIVITY_ERROR -> { return zeroDuration() ? RtpEndUserState.CONNECTIVITY_ERROR : RtpEndUserState.CONNECTIVITY_LOST_ERROR; - case TERMINATED_APPLICATION_FAILURE: + } + case TERMINATED_APPLICATION_FAILURE -> { return RtpEndUserState.APPLICATION_ERROR; - case TERMINATED_SECURITY_ERROR: + } + case TERMINATED_SECURITY_ERROR -> { return RtpEndUserState.SECURITY_ERROR; + } } throw new IllegalStateException( String.format("%s has no equivalent EndUserState", this.state)); @@ -2260,19 +2286,14 @@ public class JingleRtpConnection extends AbstractJingleConnection // be in SESSION_ACCEPTED even though the peerConnection has been torn down return RtpEndUserState.ENDING_CALL; } - switch (state) { - case CONNECTED: - return RtpEndUserState.CONNECTED; - case NEW: - case CONNECTING: - return RtpEndUserState.CONNECTING; - case CLOSED: - return RtpEndUserState.ENDING_CALL; - default: - return zeroDuration() - ? RtpEndUserState.CONNECTIVITY_ERROR - : RtpEndUserState.RECONNECTING; - } + return switch (state) { + case CONNECTED -> RtpEndUserState.CONNECTED; + case NEW, CONNECTING -> RtpEndUserState.CONNECTING; + case CLOSED -> RtpEndUserState.ENDING_CALL; + default -> zeroDuration() + ? RtpEndUserState.CONNECTIVITY_ERROR + : RtpEndUserState.RECONNECTING; + }; } public ContentAddition getPendingContentAddition() { @@ -2287,7 +2308,6 @@ public class JingleRtpConnection extends AbstractJingleConnection } } - public Set getMedia() { final State current = getState(); if (current == State.NULL) { @@ -2308,9 +2328,10 @@ public class JingleRtpConnection extends AbstractJingleConnection } else if (initiatorContentMap != null) { return initiatorContentMap.getMedia(); } else if (isTerminated()) { - return Collections.emptySet(); //we might fail before we ever got a chance to set media + return Collections.emptySet(); // we might fail before we ever got a chance to set media } else { - return Preconditions.checkNotNull(this.proposedMedia, "RTP connection has not been initialized properly"); + return Preconditions.checkNotNull( + this.proposedMedia, "RTP connection has not been initialized properly"); } } @@ -2330,37 +2351,29 @@ public class JingleRtpConnection extends AbstractJingleConnection throw new IllegalStateException(String.format("%s has already been proposed", media)); } // TODO add state protection - can only add while ACCEPTED or so - Log.d(Config.LOGTAG,"adding media: "+media); + Log.d(Config.LOGTAG, "adding media: " + media); return webRTCWrapper.addTrack(media); } - - public synchronized void acceptCall() { switch (this.state) { - case PROPOSED: + case PROPOSED -> { cancelRingingTimeout(); acceptCallFromProposed(); - break; - case SESSION_INITIALIZED: + } + case SESSION_INITIALIZED -> { cancelRingingTimeout(); acceptCallFromSessionInitialized(); - break; - case ACCEPTED: - Log.w( - Config.LOGTAG, - id.account.getJid().asBareJid() - + ": the call has already been accepted with another client. UI was just lagging behind"); - break; - case PROCEED: - case SESSION_ACCEPTED: - Log.w( - Config.LOGTAG, - id.account.getJid().asBareJid() - + ": the call has already been accepted. user probably double tapped the UI"); - break; - default: - throw new IllegalStateException("Can not accept call from " + this.state); + } + case ACCEPTED -> Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": the call has already been accepted with another client. UI was just lagging behind"); + case PROCEED, SESSION_ACCEPTED -> Log.w( + Config.LOGTAG, + id.account.getJid().asBareJid() + + ": the call has already been accepted. user probably double tapped the UI"); + default -> throw new IllegalStateException("Can not accept call from " + this.state); } } @@ -2382,14 +2395,9 @@ public class JingleRtpConnection extends AbstractJingleConnection return; } switch (this.state) { - case PROPOSED: - rejectCallFromProposed(); - break; - case SESSION_INITIALIZED: - rejectCallFromSessionInitiate(); - break; - default: - throw new IllegalStateException("Can not reject call from " + this.state); + case PROPOSED -> rejectCallFromProposed(); + case SESSION_INITIALIZED -> rejectCallFromSessionInitiate(); + default -> throw new IllegalStateException("Can not reject call from " + this.state); } } @@ -2454,9 +2462,14 @@ public class JingleRtpConnection extends AbstractJingleConnection finish(); } - private void setupWebRTC(final Set media, final List iceServers, final boolean trickle) throws WebRTCWrapper.InitializationException { + private void setupWebRTC( + final Set media, + final List iceServers, + final boolean trickle) + throws WebRTCWrapper.InitializationException { this.jingleConnectionManager.ensureConnectionIsRegistered(this); - this.webRTCWrapper.setup(this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media)); + this.webRTCWrapper.setup( + this.xmppConnectionService, AppRTCAudioManager.SpeakerPhonePreference.of(media)); this.webRTCWrapper.initializePeerConnection(media, iceServers, trickle); } @@ -2613,30 +2626,32 @@ public class JingleRtpConnection extends AbstractJingleConnection } updateEndUserState(); } + private void restartIce() { this.stateHistory.clear(); this.webRTCWrapper.restartIceAsync(); } - @Override public void onRenegotiationNeeded() { this.webRTCWrapper.execute(this::renegotiate); } - @Override - public void onTrackModification() { - this.updateEndUserState(); - } - private void renegotiate() { - Log.d(Config.LOGTAG,"method JingleRtpConnection.renegotiate()"); final SessionDescription sessionDescription; try { sessionDescription = setLocalSessionDescription(); } catch (final Exception e) { final Throwable cause = Throwables.getRootCause(e); - Log.d(Config.LOGTAG, "failed to renegotiate", cause); + webRTCWrapper.close(); + if (isTerminated()) { + Log.d( + Config.LOGTAG, + "failed to renegotiate. session was already terminated", + cause); + return; + } + Log.d(Config.LOGTAG, "failed to renegotiate. sending session-terminate", cause); sendSessionTerminate(Reason.FAILED_APPLICATION, cause.getMessage()); return; } @@ -2654,6 +2669,7 @@ public class JingleRtpConnection extends AbstractJingleConnection + diff); if (diff.hasModifications() && iceRestart) { + webRTCWrapper.close(); sendSessionTerminate( Reason.FAILED_APPLICATION, "WebRTC unexpectedly tried to modify content and transport at once"); @@ -2674,7 +2690,6 @@ public class JingleRtpConnection extends AbstractJingleConnection modifyLocalContentMap(rtpContentMap); sendContentAdd(rtpContentMap, diff.added); } - } private void initiateIceRestart(final RtpContentMap rtpContentMap) { @@ -2710,6 +2725,7 @@ public class JingleRtpConnection extends AbstractJingleConnection final Element error = response.findChild("error"); return error != null && error.hasChild("tie-break", Namespace.JINGLE_ERRORS); } + private void sendContentAdd(final RtpContentMap rtpContentMap, final Collection added) { final RtpContentMap contentAdd = rtpContentMap.toContentModification(added); this.outgoingContentAdd = contentAdd; @@ -2717,7 +2733,7 @@ public class JingleRtpConnection extends AbstractJingleConnection prepareOutgoingContentMap(contentAdd); Futures.addCallback( outgoingContentMapFuture, - new FutureCallback() { + new FutureCallback<>() { @Override public void onSuccess(final RtpContentMap outgoingContentMap) { sendContentAdd(outgoingContentMap); @@ -2762,7 +2778,6 @@ public class JingleRtpConnection extends AbstractJingleConnection }); } - private void setLocalContentMap(final RtpContentMap rtpContentMap) { if (isInitiator()) { this.initiatorRtpContentMap = rtpContentMap; @@ -2778,6 +2793,7 @@ public class JingleRtpConnection extends AbstractJingleConnection this.initiatorRtpContentMap = rtpContentMap; } } + // this method is to be used for content map modifications that modify media private void modifyLocalContentMap(final RtpContentMap rtpContentMap) { final RtpContentMap activeContents = rtpContentMap.activeContents(); @@ -2787,7 +2803,6 @@ public class JingleRtpConnection extends AbstractJingleConnection updateEndUserState(); } - private SessionDescription setLocalSessionDescription() throws ExecutionException, InterruptedException { final org.webrtc.SessionDescription sessionDescription = @@ -2795,12 +2810,6 @@ public class JingleRtpConnection extends AbstractJingleConnection return SessionDescription.parse(sessionDescription.description); } - private SessionDescription rollbackLocalSessionDescription() throws ExecutionException, InterruptedException { - final org.webrtc.SessionDescription sessionDescription = - this.webRTCWrapper.rollback().get(); - return SessionDescription.parse(sessionDescription.description); - } - private void closeWebRTCSessionAfterFailedConnection() { this.webRTCWrapper.close(); synchronized (this) { @@ -2921,6 +2930,7 @@ public class JingleRtpConnection extends AbstractJingleConnection if (port < 0 || port > 65535) { continue; } + if (Arrays.asList("stun", "stuns", "turn", "turns") .contains(type) && Arrays.asList("udp", "tcp").contains(transport)) { @@ -2935,15 +2945,19 @@ public class JingleRtpConnection extends AbstractJingleConnection // STUN URLs do not support a query section since M110 final String uri; - if (Arrays.asList("stun","stuns").contains(type)) { - uri = String.format("%s:%s:%s", type, IP.wrapIPv6(host),port); + if (Arrays.asList("stun", "stuns").contains(type)) { + uri = + String.format( + "%s:%s:%s", + type, IP.wrapIPv6(host), port); } else { - uri = String.format( - "%s:%s:%s?transport=%s", - type, - IP.wrapIPv6(host), - port, - transport); + uri = + String.format( + "%s:%s:%s?transport=%s", + type, + IP.wrapIPv6(host), + port, + transport); } final PeerConnection.IceServer.Builder iceServerBuilder = @@ -3005,6 +3019,11 @@ public class JingleRtpConnection extends AbstractJingleConnection this.webRTCWrapper.verifyClosed(); this.jingleConnectionManager.setTerminalSessionState(id, getEndUserState(), getMedia()); this.jingleConnectionManager.finishConnectionOrThrow(this); + try { + File log = new File(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOWNLOADS), "Cheogram/calls/" + id.getWith().asBareJid() + "." + id.getSessionId() + "." + created + ".log"); + log.getParentFile().mkdirs(); + Runtime.getRuntime().exec(new String[]{"logcat", "-dT", "" + created + ".0", "-f", log.getAbsolutePath()}); + } catch (final IOException e) { } } else { throw new IllegalStateException( String.format("Unable to call finish from %s", this.state)); @@ -3037,7 +3056,6 @@ public class JingleRtpConnection extends AbstractJingleConnection ((Conversation) conversational).add(this.message); xmppConnectionService.createMessageAsync(message); xmppConnectionService.updateConversationUi(); - } else { throw new IllegalStateException("Somehow the conversation in a message was a stub"); } @@ -3077,7 +3095,7 @@ public class JingleRtpConnection extends AbstractJingleConnection final boolean prerequisite = Media.audioOnly(getMedia()) && Arrays.asList(RtpEndUserState.CONNECTED, RtpEndUserState.RECONNECTING) - .contains(getEndUserState()); + .contains(getEndUserState()); return prerequisite && remoteHasVideoFeature(); } diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java index cfd4bef78..2e548d60a 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java @@ -24,6 +24,7 @@ import eu.siacs.conversations.xmpp.jingle.stanzas.OmemoVerifiedIceUdpTransportIn import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription; import java.util.Collection; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java index 9bef9eb9c..2d2dc9570 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java @@ -99,7 +99,6 @@ public class SessionDescription { case 'm': if (currentMediaBuilder == null) { sessionDescriptionBuilder.setAttributes(attributeMap); - ; } else { currentMediaBuilder.setAttributes(attributeMap); mediaBuilder.add(currentMediaBuilder.createMedia()); diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java index 1f5b54cff..7356477ee 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java @@ -1,20 +1,24 @@ package eu.siacs.conversations.xmpp.jingle; import android.content.Context; +import android.media.ToneGenerator; import android.os.Build; import android.os.Handler; import android.os.Looper; import android.util.Log; -import android.media.ToneGenerator; import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.common.util.concurrent.Futures; import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.MoreExecutors; import com.google.common.util.concurrent.SettableFuture; -import com.google.common.collect.ImmutableMap; + +import eu.siacs.conversations.Config; +import eu.siacs.conversations.services.AppRTCAudioManager; +import eu.siacs.conversations.services.XmppConnectionService; import org.webrtc.AudioSource; import org.webrtc.AudioTrack; @@ -22,6 +26,7 @@ import org.webrtc.CandidatePairChangeEvent; import org.webrtc.DataChannel; import org.webrtc.DefaultVideoDecoderFactory; import org.webrtc.DefaultVideoEncoderFactory; +import org.webrtc.DtmfSender; import org.webrtc.EglBase; import org.webrtc.IceCandidate; import org.webrtc.MediaConstraints; @@ -35,25 +40,21 @@ import org.webrtc.SdpObserver; import org.webrtc.SessionDescription; import org.webrtc.VideoTrack; import org.webrtc.audio.JavaAudioDeviceModule; -import org.webrtc.DtmfSender; +import java.util.Collection; import java.util.LinkedList; import java.util.List; import java.util.Queue; -import java.util.Set; -import java.util.concurrent.Callable; import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.Nonnull; import javax.annotation.Nullable; -import eu.siacs.conversations.Config; -import eu.siacs.conversations.services.AppRTCAudioManager; -import eu.siacs.conversations.services.XmppConnectionService; - @SuppressWarnings("UnstableApiUsage") public class WebRTCWrapper { @@ -63,26 +64,6 @@ public class WebRTCWrapper { private final ExecutorService localDescriptionExecutorService = Executors.newSingleThreadExecutor(); - private static final Set HARDWARE_AEC_BLACKLIST = - new ImmutableSet.Builder() - .add("Pixel") - .add("Pixel XL") - .add("Moto G5") - .add("Moto G (5S) Plus") - .add("Moto G4") - .add("TA-1053") - .add("Mi A1") - .add("Mi A2") - .add("E5823") // Sony z5 compact - .add("Redmi Note 5") - .add("FP2") // Fairphone FP2 - .add("FP4") //Fairphone FP4 - .add("MI 5") - .add("GT-I9515") // Samsung Galaxy S4 Value Edition (jfvelte) - .add("GT-I9515L") // Samsung Galaxy S4 Value Edition (jfvelte) - .add("GT-I9505") // Samsung Galaxy S4 (jfltexx) - .build(); - private static final int TONE_DURATION = 500; private static final Map TONE_CODES; static { @@ -102,6 +83,26 @@ public class WebRTCWrapper { TONE_CODES = builder.build(); } + private static final Set HARDWARE_AEC_BLACKLIST = + new ImmutableSet.Builder() + .add("Pixel") + .add("Pixel XL") + .add("Moto G5") + .add("Moto G (5S) Plus") + .add("Moto G4") + .add("TA-1053") + .add("Mi A1") + .add("Mi A2") + .add("E5823") // Sony z5 compact + .add("Redmi Note 5") + .add("FP2") // Fairphone FP2 + .add("FP4") // Fairphone FP4 + .add("MI 5") + .add("GT-I9515") // Samsung Galaxy S4 Value Edition (jfvelte) + .add("GT-I9515L") // Samsung Galaxy S4 Value Edition (jfvelte) + .add("GT-I9505") // Samsung Galaxy S4 (jfltexx) + .build(); + private final EventCallback eventCallback; private final AtomicBoolean readyToReceivedIceCandidates = new AtomicBoolean(false); private final Queue iceCandidates = new LinkedList<>(); @@ -213,7 +214,6 @@ public class WebRTCWrapper { + ")"); if (track instanceof VideoTrack) { remoteVideoTrack = (VideoTrack) track; - eventCallback.onTrackModification(); } } @@ -263,8 +263,7 @@ public class WebRTCWrapper { PeerConnectionFactory.initialize( PeerConnectionFactory.InitializationOptions.builder(service) .setFieldTrials("WebRTC-BindUsingInterfaceName/Enabled/") - .createInitializationOptions() - ); + .createInitializationOptions()); } catch (final UnsatisfiedLinkError e) { throw new InitializationException("Unable to initialize PeerConnectionFactory", e); } @@ -370,8 +369,6 @@ public class WebRTCWrapper { } } - - private boolean addAudioTrack(final PeerConnection peerConnection) { final AudioSource audioSource = requirePeerConnectionFactory().createAudioSource(new MediaConstraints()); @@ -407,10 +404,7 @@ public class WebRTCWrapper { .createVideoTrack( TrackWrapper.id(VideoTrack.class), videoSourceWrapper.getVideoSource()); - // TODO do we want to create Transceiver manually and be able to set direction and keep a - // reference to it for later removal this.localVideoTrack = TrackWrapper.addTrack(peerConnection, videoTrack); - this.eventCallback.onTrackModification(); return true; } @@ -435,8 +429,6 @@ public class WebRTCWrapper { } } - - private static PeerConnection.RTCConfiguration buildConfiguration( final List iceServers, final boolean trickle) { final PeerConnection.RTCConfiguration rtcConfig = @@ -640,7 +632,7 @@ public class WebRTCWrapper { public void onSetSuccess() { final var delay = waitForCandidates - ? iceGatheringComplete + ? Futures.catching(Futures.withTimeout(iceGatheringComplete, 2, TimeUnit.SECONDS, JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE), Exception.class, (Exception e) -> { return null; }, MoreExecutors.directExecutor()) : Futures.immediateVoidFuture(); final var delayedSessionDescription = Futures.transformAsync( @@ -673,39 +665,6 @@ public class WebRTCWrapper { localDescriptionExecutorService); } - - synchronized ListenableFuture rollback() { - return Futures.transformAsync( - getPeerConnectionFuture(), - peerConnection -> { - final SettableFuture future = SettableFuture.create(); - if (peerConnection == null) { - return Futures.immediateFailedFuture( - new IllegalStateException("PeerConnection was null")); - } - peerConnection.setLocalDescription( - new SetSdpObserver() { - @Override - public void onSetSuccess() { - final SessionDescription description = - peerConnection.getLocalDescription(); - Log.d(EXTENDED_LOGGING_TAG, "rollback to local description:"); - logDescription(description); - future.set(description); - } - - @Override - public void onSetFailure(final String message) { - future.setException( - new FailureToSetDescriptionException(message)); - } - }, - new SessionDescription(SessionDescription.Type.ROLLBACK, "")); - return future; - }, - MoreExecutors.directExecutor()); - } - public static void logDescription(final SessionDescription sessionDescription) { for (final String line : sessionDescription.description.split( @@ -764,15 +723,6 @@ public class WebRTCWrapper { return peerConnection; } - @Nonnull - private PeerConnectionFactory requirePeerConnectionFactory() { - final PeerConnectionFactory peerConnectionFactory = this.peerConnectionFactory; - if (peerConnectionFactory == null) { - throw new IllegalStateException("Make sure PeerConnectionFactory is initialized"); - } - return peerConnectionFactory; - } - public boolean applyDtmfTone(String tone) { if (toneManager == null || peerConnection == null || localAudioTrack == null) { return false; @@ -782,6 +732,15 @@ public class WebRTCWrapper { return true; } + @Nonnull + private PeerConnectionFactory requirePeerConnectionFactory() { + final PeerConnectionFactory peerConnectionFactory = this.peerConnectionFactory; + if (peerConnectionFactory == null) { + throw new IllegalStateException("Make sure PeerConnectionFactory is initialized"); + } + return peerConnectionFactory; + } + void addIceCandidate(IceCandidate iceCandidate) { requirePeerConnection().addIceCandidate(iceCandidate); } @@ -830,7 +789,6 @@ public class WebRTCWrapper { mainHandler.post(() -> appRTCAudioManager.switchSpeakerPhonePreference(preference)); } - public interface EventCallback { void onIceCandidate(IceCandidate iceCandidate); @@ -841,8 +799,6 @@ public class WebRTCWrapper { Set availableAudioDevices); void onRenegotiationNeeded(); - - void onTrackModification(); } private abstract static class SetSdpObserver implements SdpObserver { diff --git a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java index ccaba56a6..8c8a9683d 100644 --- a/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java +++ b/src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java @@ -70,6 +70,9 @@ public class IceUdpTransportInfo extends GenericTransportInfo { for (final String iceOption : IceOption.of(media)) { iceUdpTransportInfo.addChild(new IceOption(iceOption)); } + for (final String candidate : media.attributes.get("candidate")) { + iceUdpTransportInfo.addChild(Candidate.fromSdpAttributeValue(candidate, ufrag)); + } return iceUdpTransportInfo; } @@ -96,7 +99,7 @@ public class IceUdpTransportInfo extends GenericTransportInfo { public List getIceOptions() { final ImmutableList.Builder optionBuilder = new ImmutableList.Builder<>(); - for (final Element child : this.children) { + for (final Element child : getChildren()) { if (Namespace.JINGLE_TRANSPORT_ICE_OPTION.equals(child.getNamespace()) && IceOption.WELL_KNOWN.contains(child.getName())) { optionBuilder.add(child.getName()); @@ -114,7 +117,7 @@ public class IceUdpTransportInfo extends GenericTransportInfo { public boolean isStub() { return Strings.isNullOrEmpty(this.getAttribute("ufrag")) && Strings.isNullOrEmpty(this.getAttribute("pwd")) - && this.children.isEmpty(); + && getChildren().isEmpty(); } public List getCandidates() {