Rework RTP + upgrade to Java 17 fixes crash when switch from audio to videocall

build.gradle

@@ -188,8 +188,8 @@ android {
 
     compileOptions {
         coreLibraryDesugaringEnabled true
-        sourceCompatibility JavaVersion.VERSION_1_9
+        sourceCompatibility JavaVersion.VERSION_17
-        targetCompatibility JavaVersion.VERSION_1_9
+        targetCompatibility JavaVersion.VERSION_17
     }
 
     flavorDimensions("distribution")

src/main/java/de/monocles/chat/ConnectionService.java

@@ -121,7 +121,7 @@ public class ConnectionService extends android.telecom.ConnectionService {
             );
         }
 
-        if (xmppConnectionService.getJingleConnectionManager().isBusy()) {
+        if (xmppConnectionService.getJingleConnectionManager().isBusy() != null) {
             return Connection.createFailedConnection(
                     new DisconnectCause(DisconnectCause.BUSY)
             );

src/main/java/eu/siacs/conversations/parser/MessageParser.java

@@ -448,7 +448,7 @@ public class MessageParser extends AbstractParser implements OnMessagePacketRece
                 }
                 if (id.startsWith(JingleRtpConnection.JINGLE_MESSAGE_PROCEED_ID_PREFIX)) {
                     final String sessionId = id.substring(JingleRtpConnection.JINGLE_MESSAGE_PROCEED_ID_PREFIX.length());
-                    mXmppConnectionService.getJingleConnectionManager().failProceed(account, from, sessionId);
+                    mXmppConnectionService.getJingleConnectionManager().failProceed(account, from, sessionId, extractErrorMessage(packet)); //TODO: Test it and check this changes again!
                     return true;
                 }
                 mXmppConnectionService.markMessage(account,

src/main/java/eu/siacs/conversations/ui/ConversationFragment.java

@@ -2351,8 +2351,9 @@ public class ConversationFragment extends XmppFragment
     }
 
     private void triggerRtpSession(final String action) {
-        if (activity.xmppConnectionService.getJingleConnectionManager().isBusy()) {
+        if (activity.xmppConnectionService.getJingleConnectionManager().isBusy() != null) {
-            Toast.makeText(getActivity(), R.string.only_one_call_at_a_time, Toast.LENGTH_LONG).show();
+            Toast.makeText(getActivity(), R.string.only_one_call_at_a_time, Toast.LENGTH_LONG)
+                    .show();
             return;
         }
         final Account account = conversation.getAccount();

src/main/java/eu/siacs/conversations/ui/util/CallManager.java

@@ -53,7 +53,7 @@ public class CallManager {
     }
 
     public static void triggerRtpSession(final String action, XmppActivity activity, Conversation conversation) {
-        if (activity.xmppConnectionService.getJingleConnectionManager().isBusy()) {
+        if (activity.xmppConnectionService.getJingleConnectionManager().isBusy() != null) {
             ToastCompat.makeText(activity, R.string.only_one_call_at_a_time, ToastCompat.LENGTH_LONG).show();
             return;
         }

src/main/java/eu/siacs/conversations/xmpp/jingle/JingleConnectionManager.java

@@ -36,8 +36,8 @@ import eu.siacs.conversations.entities.RtpSessionStatus;
 import eu.siacs.conversations.entities.Transferable;
 import eu.siacs.conversations.services.AbstractConnectionManager;
 import eu.siacs.conversations.services.XmppConnectionService;
-import eu.siacs.conversations.xml.Namespace;
 import eu.siacs.conversations.xml.Element;
+import eu.siacs.conversations.xml.Namespace;
 import eu.siacs.conversations.xmpp.Jid;
 import eu.siacs.conversations.xmpp.OnIqPacketReceived;
 import eu.siacs.conversations.xmpp.XmppConnection;
@@ -100,7 +100,7 @@ public class JingleConnectionManager extends AbstractConnectionManager {
                         this.terminatedSessions.asMap().containsKey(PersistableSessionId.of(id));
                 final boolean stranger =
                         isWithStrangerAndStrangerNotificationsAreOff(account, id.with);
-                final boolean busy = isBusy();
+                final boolean busy = isBusy() != null;
                 if (busy || sessionEnded || stranger) {
                     Log.d(
                             Config.LOGTAG,
@@ -145,26 +145,26 @@ public class JingleConnectionManager extends AbstractConnectionManager {
     }
 
     private boolean isUsingClearNet(final Account account) {
-        return !account.isOnion() && !mXmppConnectionService.useTorToConnect() && !account.isI2P() && !mXmppConnectionService.useI2PToConnect();
+        return !account.isOnion() && !mXmppConnectionService.useTorToConnect();
     }
 
-    public boolean isBusy() {
+    public String isBusy() {
         if (mXmppConnectionService.isPhoneInCall()) {
-            return true;
+            return "isPhoneInCall";
         }
         for (AbstractJingleConnection connection : this.connections.values()) {
             if (connection instanceof JingleRtpConnection) {
                 if (((JingleRtpConnection) connection).isTerminated()) {
                     continue;
                 }
-                return true;
+                return "connection !isTerminated";
             }
         }
         synchronized (this.rtpSessionProposals) {
-            return this.rtpSessionProposals.containsValue(DeviceDiscoveryState.DISCOVERED)
+            if (this.rtpSessionProposals.containsValue(DeviceDiscoveryState.DISCOVERED)) return "discovered";
-                    || this.rtpSessionProposals.containsValue(DeviceDiscoveryState.SEARCHING)
+            if (this.rtpSessionProposals.containsValue(DeviceDiscoveryState.SEARCHING)) return "searching";
-                    || this.rtpSessionProposals.containsValue(
+            if (this.rtpSessionProposals.containsValue(DeviceDiscoveryState.SEARCHING_ACKNOWLEDGED)) return "searching_acknolwedged";
-                    DeviceDiscoveryState.SEARCHING_ACKNOWLEDGED);
+            return null;
         }
     }
 
@@ -395,18 +395,20 @@ public class JingleConnectionManager extends AbstractConnectionManager {
                         this.connections.put(id, rtpConnection);
                         rtpConnection.setProposedMedia(ImmutableSet.copyOf(media));
                         rtpConnection.deliveryMessage(from, message, serverMsgId, timestamp);
+                        // TODO actually do the automatic accept?!
                     } else {
                         Log.d(
                                 Config.LOGTAG,
                                 account.getJid().asBareJid()
                                         + ": our session won tie break. waiting for other party to accept. winningSession="
                                         + ourSessionId);
+                        // TODO reject their session with <tie-break/>?
                     }
                     return;
                 }
                 final boolean stranger =
                         isWithStrangerAndStrangerNotificationsAreOff(account, id.with);
-                if (isBusy() || stranger) {
+                if (isBusy() != null || stranger) {
                     writeLogMissedIncoming(
                             account,
                             id.with.asBareJid(),
@@ -786,19 +788,23 @@ public class JingleConnectionManager extends AbstractConnectionManager {
                         final RtpEndUserState endUserState = preexistingState.toEndUserState();
                         toneManager.transition(endUserState, media);
                         mXmppConnectionService.notifyJingleRtpConnectionUpdate(
-                                account, with, proposal.sessionId, endUserState);
+                                account,
+                                with,
+                                proposal.sessionId,
+                                endUserState
+                        );
                         return proposal.sessionId;
                     }
                 }
             }
-            if (isBusy()) {
+            String busyCode = isBusy();
+            if (busyCode != null) {
                 String sessionId = hasMatchingRtpSession(account, with, media);
                 if (sessionId != null) {
                     Log.d(Config.LOGTAG, "ignoring request to propose jingle session because the other party already created one for us: " + sessionId);
                     return sessionId;
                 }
-                throw new IllegalStateException(
+                throw new IllegalStateException("There is already a running RTP session: " + busyCode);
-                        "There is already a running RTP session. This should have been caught by the UI");
             }
             final RtpSessionProposal proposal =
                     RtpSessionProposal.of(account, with.asBareJid(), media);
@@ -964,12 +970,12 @@ public class JingleConnectionManager extends AbstractConnectionManager {
         }
     }
 
-    public void failProceed(Account account, final Jid with, String sessionId) {
+    public void failProceed(Account account, final Jid with, final String sessionId, final String message) {
         final AbstractJingleConnection.Id id =
                 AbstractJingleConnection.Id.of(account, with, sessionId);
         final AbstractJingleConnection existingJingleConnection = connections.get(id);
         if (existingJingleConnection instanceof JingleRtpConnection) {
-            ((JingleRtpConnection) existingJingleConnection).deliverFailedProceed();
+            ((JingleRtpConnection) existingJingleConnection).deliverFailedProceed(message);
         }
     }
 

src/main/java/eu/siacs/conversations/xmpp/jingle/RtpContentMap.java

@@ -24,6 +24,7 @@ import eu.siacs.conversations.xmpp.jingle.stanzas.OmemoVerifiedIceUdpTransportIn
 import eu.siacs.conversations.xmpp.jingle.stanzas.RtpDescription;
 
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;

src/main/java/eu/siacs/conversations/xmpp/jingle/SessionDescription.java

@@ -99,7 +99,6 @@ public class SessionDescription {
                 case 'm':
                     if (currentMediaBuilder == null) {
                         sessionDescriptionBuilder.setAttributes(attributeMap);
-                        ;
                     } else {
                         currentMediaBuilder.setAttributes(attributeMap);
                         mediaBuilder.add(currentMediaBuilder.createMedia());

src/main/java/eu/siacs/conversations/xmpp/jingle/WebRTCWrapper.java

@@ -1,20 +1,24 @@
 package eu.siacs.conversations.xmpp.jingle;
 
 import android.content.Context;
+import android.media.ToneGenerator;
 import android.os.Build;
 import android.os.Handler;
 import android.os.Looper;
 import android.util.Log;
-import android.media.ToneGenerator;
 
 import com.google.common.base.Optional;
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.util.concurrent.Futures;
 import com.google.common.util.concurrent.ListenableFuture;
 import com.google.common.util.concurrent.MoreExecutors;
 import com.google.common.util.concurrent.SettableFuture;
-import com.google.common.collect.ImmutableMap;
+
+import eu.siacs.conversations.Config;
+import eu.siacs.conversations.services.AppRTCAudioManager;
+import eu.siacs.conversations.services.XmppConnectionService;
 
 import org.webrtc.AudioSource;
 import org.webrtc.AudioTrack;
@@ -22,6 +26,7 @@ import org.webrtc.CandidatePairChangeEvent;
 import org.webrtc.DataChannel;
 import org.webrtc.DefaultVideoDecoderFactory;
 import org.webrtc.DefaultVideoEncoderFactory;
+import org.webrtc.DtmfSender;
 import org.webrtc.EglBase;
 import org.webrtc.IceCandidate;
 import org.webrtc.MediaConstraints;
@@ -35,25 +40,21 @@ import org.webrtc.SdpObserver;
 import org.webrtc.SessionDescription;
 import org.webrtc.VideoTrack;
 import org.webrtc.audio.JavaAudioDeviceModule;
-import org.webrtc.DtmfSender;
 
+import java.util.Collection;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Queue;
-import java.util.Set;
-import java.util.concurrent.Callable;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 import javax.annotation.Nonnull;
 import javax.annotation.Nullable;
 
-import eu.siacs.conversations.Config;
-import eu.siacs.conversations.services.AppRTCAudioManager;
-import eu.siacs.conversations.services.XmppConnectionService;
-
 @SuppressWarnings("UnstableApiUsage")
 public class WebRTCWrapper {
 
@@ -63,26 +64,6 @@ public class WebRTCWrapper {
     private final ExecutorService localDescriptionExecutorService =
             Executors.newSingleThreadExecutor();
 
-    private static final Set<String> HARDWARE_AEC_BLACKLIST =
-            new ImmutableSet.Builder<String>()
-                    .add("Pixel")
-                    .add("Pixel XL")
-                    .add("Moto G5")
-                    .add("Moto G (5S) Plus")
-                    .add("Moto G4")
-                    .add("TA-1053")
-                    .add("Mi A1")
-                    .add("Mi A2")
-                    .add("E5823") // Sony z5 compact
-                    .add("Redmi Note 5")
-                    .add("FP2") // Fairphone FP2
-                    .add("FP4") //Fairphone FP4
-                    .add("MI 5")
-                    .add("GT-I9515") // Samsung Galaxy S4 Value Edition (jfvelte)
-                    .add("GT-I9515L") // Samsung Galaxy S4 Value Edition (jfvelte)
-                    .add("GT-I9505") // Samsung Galaxy S4 (jfltexx)
-                    .build();
-
     private static final int TONE_DURATION = 500;
     private static final Map<String,Integer> TONE_CODES;
     static {
@@ -102,6 +83,26 @@ public class WebRTCWrapper {
         TONE_CODES = builder.build();
     }
 
+    private static final Set<String> HARDWARE_AEC_BLACKLIST =
+            new ImmutableSet.Builder<String>()
+                    .add("Pixel")
+                    .add("Pixel XL")
+                    .add("Moto G5")
+                    .add("Moto G (5S) Plus")
+                    .add("Moto G4")
+                    .add("TA-1053")
+                    .add("Mi A1")
+                    .add("Mi A2")
+                    .add("E5823") // Sony z5 compact
+                    .add("Redmi Note 5")
+                    .add("FP2") // Fairphone FP2
+                    .add("FP4") // Fairphone FP4
+                    .add("MI 5")
+                    .add("GT-I9515") // Samsung Galaxy S4 Value Edition (jfvelte)
+                    .add("GT-I9515L") // Samsung Galaxy S4 Value Edition (jfvelte)
+                    .add("GT-I9505") // Samsung Galaxy S4 (jfltexx)
+                    .build();
+
     private final EventCallback eventCallback;
     private final AtomicBoolean readyToReceivedIceCandidates = new AtomicBoolean(false);
     private final Queue<IceCandidate> iceCandidates = new LinkedList<>();
@@ -213,7 +214,6 @@ public class WebRTCWrapper {
                                     + ")");
                     if (track instanceof VideoTrack) {
                         remoteVideoTrack = (VideoTrack) track;
-                        eventCallback.onTrackModification();
                     }
                 }
 
@@ -263,8 +263,7 @@ public class WebRTCWrapper {
             PeerConnectionFactory.initialize(
                     PeerConnectionFactory.InitializationOptions.builder(service)
                             .setFieldTrials("WebRTC-BindUsingInterfaceName/Enabled/")
-                            .createInitializationOptions()
+                            .createInitializationOptions());
-            );
         } catch (final UnsatisfiedLinkError e) {
             throw new InitializationException("Unable to initialize PeerConnectionFactory", e);
         }
@@ -370,8 +369,6 @@ public class WebRTCWrapper {
         }
     }
 
-
-
     private boolean addAudioTrack(final PeerConnection peerConnection) {
         final AudioSource audioSource =
                 requirePeerConnectionFactory().createAudioSource(new MediaConstraints());
@@ -407,10 +404,7 @@ public class WebRTCWrapper {
                         .createVideoTrack(
                                 TrackWrapper.id(VideoTrack.class),
                                 videoSourceWrapper.getVideoSource());
-        // TODO do we want to create Transceiver manually and be able to set direction and keep a
-        // reference to it for later removal
         this.localVideoTrack = TrackWrapper.addTrack(peerConnection, videoTrack);
-        this.eventCallback.onTrackModification();
         return true;
     }
 
@@ -435,8 +429,6 @@ public class WebRTCWrapper {
         }
     }
 
-
-
     private static PeerConnection.RTCConfiguration buildConfiguration(
             final List<PeerConnection.IceServer> iceServers, final boolean trickle) {
         final PeerConnection.RTCConfiguration rtcConfig =
@@ -640,7 +632,7 @@ public class WebRTCWrapper {
                                 public void onSetSuccess() {
                                     final var delay =
                                             waitForCandidates
-                                                    ? iceGatheringComplete
+                                                    ? Futures.catching(Futures.withTimeout(iceGatheringComplete, 2, TimeUnit.SECONDS, JingleConnectionManager.SCHEDULED_EXECUTOR_SERVICE), Exception.class, (Exception e) -> { return null; }, MoreExecutors.directExecutor())
                                                     : Futures.immediateVoidFuture();
                                     final var delayedSessionDescription =
                                             Futures.transformAsync(
@@ -673,39 +665,6 @@ public class WebRTCWrapper {
                 localDescriptionExecutorService);
     }
 
-
-    synchronized ListenableFuture<SessionDescription> rollback() {
-        return Futures.transformAsync(
-                getPeerConnectionFuture(),
-                peerConnection -> {
-                    final SettableFuture<SessionDescription> future = SettableFuture.create();
-                    if (peerConnection == null) {
-                        return Futures.immediateFailedFuture(
-                                new IllegalStateException("PeerConnection was null"));
-                    }
-                    peerConnection.setLocalDescription(
-                            new SetSdpObserver() {
-                                @Override
-                                public void onSetSuccess() {
-                                    final SessionDescription description =
-                                            peerConnection.getLocalDescription();
-                                    Log.d(EXTENDED_LOGGING_TAG, "rollback to local description:");
-                                    logDescription(description);
-                                    future.set(description);
-                                }
-
-                                @Override
-                                public void onSetFailure(final String message) {
-                                    future.setException(
-                                            new FailureToSetDescriptionException(message));
-                                }
-                            },
-                            new SessionDescription(SessionDescription.Type.ROLLBACK, ""));
-                    return future;
-                },
-                MoreExecutors.directExecutor());
-    }
-
     public static void logDescription(final SessionDescription sessionDescription) {
         for (final String line :
                 sessionDescription.description.split(
@@ -764,15 +723,6 @@ public class WebRTCWrapper {
         return peerConnection;
     }
 
-    @Nonnull
-    private PeerConnectionFactory requirePeerConnectionFactory() {
-        final PeerConnectionFactory peerConnectionFactory = this.peerConnectionFactory;
-        if (peerConnectionFactory == null) {
-            throw new IllegalStateException("Make sure PeerConnectionFactory is initialized");
-        }
-        return peerConnectionFactory;
-    }
-
     public boolean applyDtmfTone(String tone) {
         if (toneManager == null || peerConnection == null || localAudioTrack == null) {
             return false;
@@ -782,6 +732,15 @@ public class WebRTCWrapper {
         return true;
     }
 
+    @Nonnull
+    private PeerConnectionFactory requirePeerConnectionFactory() {
+        final PeerConnectionFactory peerConnectionFactory = this.peerConnectionFactory;
+        if (peerConnectionFactory == null) {
+            throw new IllegalStateException("Make sure PeerConnectionFactory is initialized");
+        }
+        return peerConnectionFactory;
+    }
+
     void addIceCandidate(IceCandidate iceCandidate) {
         requirePeerConnection().addIceCandidate(iceCandidate);
     }
@@ -830,7 +789,6 @@ public class WebRTCWrapper {
         mainHandler.post(() -> appRTCAudioManager.switchSpeakerPhonePreference(preference));
     }
 
-
     public interface EventCallback {
         void onIceCandidate(IceCandidate iceCandidate);
 
@@ -841,8 +799,6 @@ public class WebRTCWrapper {
                 Set<AppRTCAudioManager.AudioDevice> availableAudioDevices);
 
         void onRenegotiationNeeded();
-
-        void onTrackModification();
     }
 
     private abstract static class SetSdpObserver implements SdpObserver {

src/main/java/eu/siacs/conversations/xmpp/jingle/stanzas/IceUdpTransportInfo.java

@@ -70,6 +70,9 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
         for (final String iceOption : IceOption.of(media)) {
             iceUdpTransportInfo.addChild(new IceOption(iceOption));
         }
+        for (final String candidate : media.attributes.get("candidate")) {
+            iceUdpTransportInfo.addChild(Candidate.fromSdpAttributeValue(candidate, ufrag));
+        }
         return iceUdpTransportInfo;
     }
 
@@ -96,7 +99,7 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
 
     public List<String> getIceOptions() {
         final ImmutableList.Builder<String> optionBuilder = new ImmutableList.Builder<>();
-        for (final Element child : this.children) {
+        for (final Element child : getChildren()) {
             if (Namespace.JINGLE_TRANSPORT_ICE_OPTION.equals(child.getNamespace())
                     && IceOption.WELL_KNOWN.contains(child.getName())) {
                 optionBuilder.add(child.getName());
@@ -114,7 +117,7 @@ public class IceUdpTransportInfo extends GenericTransportInfo {
     public boolean isStub() {
         return Strings.isNullOrEmpty(this.getAttribute("ufrag"))
                 && Strings.isNullOrEmpty(this.getAttribute("pwd"))
-                && this.children.isEmpty();
+                && getChildren().isEmpty();
     }
 
     public List<Candidate> getCandidates() {