Unable to get other peer video

[harshapulikollu]

I'm using your demo code and firestore as signalling server. But i can't understand how to accept answer when other person called.
Here is my signalling server code:

import 'dart:convert';
import 'dart:async';
import 'dart:io';
import 'package:cloud_firestore/cloud_firestore.dart';
import 'package:flutter_webrtc/webrtc.dart';
import 'auth.dart';

enum SignalingState {
  CallStateNew,
  CallStateRinging,
  CallStateInvite,
  CallStateConnected,
  CallStateBye,
  ConnectionOpen,
  ConnectionClosed,
  ConnectionError,
}

typedef void SignalingStateCallback(SignalingState state);
typedef void StreamStateCallback(MediaStream stream);
typedef void OtherEventCallback(dynamic event);
typedef void DataChannelMessageCallback(RTCDataChannel dc, data);

class Signaling{
  Signaling(this._name, this.peerID);
  String _selfId = currentUserModel.id;
  var _session_id;
  var _url;
  var _name, peerID;
  var _peerConnections = new Map<String, RTCPeerConnection>();
  var _daChannels = new Map<int, RTCDataChannel>();
  Timer _timer;
  MediaStream _localStream;
  List<MediaStream> _remoteStreams;
  SignalingStateCallback onStateChange;
  StreamStateCallback onLocalStream;
  StreamStateCallback onAddRemoteStream;
  StreamStateCallback onRemoveRemoteStream;
  OtherEventCallback onPeersUpdate;
  DataChannelMessageCallback onDataChannel;

  final ref = Firestore.instance;
  StreamSubscription<DocumentSnapshot> streamSub;
  Map<String, dynamic> _iceServers = {
    'iceServers': [
      {'url': 'stun:stun.l.google.com:19302'},
    ]
  };

  final Map<String, dynamic> _config = {
    'mandatory': {},
    'optional': [
      {'DtlsSrtpKeyAgreement': true},
    ],
  };

  final Map<String, dynamic> _constraints = {
    'mandatory': {
      'OfferToReceiveAudio': true,
      'OfferToReceiveVideo': true,
    },
    'optional': [],
  };

  void invite(String peerId, String media) {
    //this._session_id = this._selfId + '-' + peerId;
    if (_selfId.hashCode <= peerID.hashCode) {
      this._session_id = this._selfId+'-'+peerId;//'$this._selfId-$peerID';
    } else {
      this._session_id = peerId+'-'+this._selfId;//'$peerID-$_selfId';
    }
      print('line 67 VCS  $_session_id');
    if (this.onStateChange != null) {
      this.onStateChange(SignalingState.CallStateNew);
    }
      print('line 76 $peerId');
    _createPeerConnection(peerId, media).then((pc) {
      _peerConnections[peerId] = pc;
      _createOffer(peerId, pc, media);
    });
  }

  _createPeerConnection(String id, String media) async{
    print('line 84 id is $id');
    _localStream = await createStream();
    RTCPeerConnection pc = await createPeerConnection(_iceServers, _config);
    print('line 87 vcs-cpc');
    pc.addStream(_localStream);
    pc.onIceCandidate = (candidate) {
      _send('candidate', {
        'to': id,
        'candidate': {
          'sdpMLineIndex': candidate.sdpMlineIndex,
          'sdpMid': candidate.sdpMid,
          'candidate': candidate.candidate,
        },
        'session_id': this._session_id,
      });
    };

    pc.onAddStream = (stream) {
      if(this.onAddRemoteStream != null)
        this.onAddRemoteStream(stream);
      //_remoteStreams.add(stream);
    };

    pc.onRemoveStream = (stream) {
      if(this.onRemoveRemoteStream != null)
        this.onRemoveRemoteStream(stream);
      _remoteStreams.removeWhere((it) {
        return (it.id == stream.id);
      });
    };

    pc.onDataChannel = (channel) {
      _addDataChannel(id, channel);
    };
    print('line 118 vcs-cpc');
    return pc;

  }

  void _createOffer(String id, pc, String media) async{
    try {
      RTCSessionDescription s = await pc.createOffer(_constraints);
      pc.setLocalDescription(s);
      _send('offer', {
        'to': id,
        'description': {'sdp': s.sdp, 'type': s.type},
        'session_id': this._session_id,
        'media': media,
      });
    } catch (e) {
      print(e.toString());
    }
  }

  _addDataChannel(id, RTCDataChannel channel) {
    channel.onDataChannelState = (e) {};
    channel.onMessage = (data) {
      if(this.onDataChannel != null)
        this.onDataChannel(channel, data);
    };
    _daChannels[id] = channel;
  }

  Future<MediaStream> createStream() async {
    final Map<String, dynamic> mediaConstraints = {
      'audio': true,
      'video': {
        'mandatory': {
          'minWidth':
          '640', // Provide your own width, height and frame rate here
          'minHeight': '480',
          'minFrameRate': '30',
        },
        'facingMode': 'user',
        'optional': [],
      }
    };

    MediaStream stream = await navigator.getUserMedia(mediaConstraints);
    if(this.onLocalStream != null){
      this.onLocalStream(stream);
    }
    return stream;
  }

  void bye() {
    _send('bye', {
      'session_id': this._session_id,
      'from': this._selfId,
    });
   // streamSub.cancel();
  }

  void onMessage(Map message) async {
    print('line 160 VCS $message');
    Map<String, dynamic> mapData = message;

    var data = mapData;

    print('line 169 VCS $data');
    switch(mapData['type']){

//      case 'peers':
//        {
//          List<dynamic> peers = data;
//          if(this.onPeersUpdate != null) {
//            Map<String, dynamic> event = new  Map<String, dynamic>();
//            event['self'] = _self_id;
//            event['peers'] = peers;
//            this.onPeersUpdate(event);
//          }
//        }
//        break;
      case 'offer':
        {
         
          var id = data['from'];
          var description = data['description'];
          var media = data['media'];
          var session_id = data['session_id'];
          this._session_id = session_id;

          if (this.onStateChange != null) {
            this.onStateChange(SignalingState.CallStateNew);
          }

          _createPeerConnection(id, media).then((pc) {
            _peerConnections[id] = pc;
            pc.setRemoteDescription(
                new RTCSessionDescription(description['sdp'], description['type']));
            _createAnswer(id, pc);
          });
         
        }
        break;
      case 'answer':
        {
   
          var id = data['from'];
          var description = data['description'];

          var pc = _peerConnections[id];
          if (pc != null) {
            pc.setRemoteDescription(
                new RTCSessionDescription(description['sdp'], description['type']));
          }
        
        }
        break;
      case 'candidate':
        {
       
          var id = data['from'];
          var candidateMap = data['candidate'];
          var pc = _peerConnections[id];

          if (pc != null) {
            RTCIceCandidate candidate = new RTCIceCandidate(
                candidateMap['candidate'],
                candidateMap['sdpMid'],
                candidateMap['sdpMLineIndex']);
            pc.addCandidate(candidate);
          }
          
        }
        break;
      case 'leave':
        {
          var id = data;
          _peerConnections.remove(id);
          _daChannels.remove(id);

          if (_localStream != null) {
            _localStream.dispose();
            _localStream = null;
          }

          var pc = _peerConnections[id];
          if (pc != null) {
            pc.close();
            _peerConnections.remove(id);
          }
          this._session_id = null;
          if (this.onStateChange != null) {
            this.onStateChange(SignalingState.CallStateBye);
          }
        }
        break;
      case 'bye':
        {          var from = data['from'];
        var to = data['to'];
        var session_id = data['session_id'];
        print('bye: ' + session_id);

        if (_localStream != null) {
          _localStream.dispose();
          _localStream = null;
        }


        var pc = _peerConnections[to];
        if (pc != null) {
          pc.close();
          _peerConnections.remove(to);
        }
        this._session_id = null;
        if (this.onStateChange != null) {
          this.onStateChange(SignalingState.CallStateBye);
        }
        }
        break;
      case 'keepalive':
        {
          print('keepalive response!');
        }
        break;
      default:
        break;
    }
  }

  _createAnswer(String id, RTCPeerConnection pc) async {
    print('line 306');
    try {
      RTCSessionDescription s = await pc.createAnswer(_constraints);
      pc.setLocalDescription(s);
      _send('answer', {
        'to': id,
        'description': {'sdp': s.sdp, 'type': s.type},
        'session_id': this._session_id,
      });
    } catch (e) {
      print(e.toString());
    }
  }

  connect() async{
    if (_selfId.hashCode <= peerID.hashCode) {
      this._session_id = this._selfId+'-'+peerID;//'$this._selfId-$peerID';
    } else {
      this._session_id = peerID+'-'+this._selfId;//'$peerID-$_selfId';
    }
    if (this.onStateChange != null) {
      this.onStateChange(SignalingState.ConnectionOpen);
    }
    _send('new', {
      'name': _name,
      'id': _selfId,
      'user_agent': 'flutter-webrtc/'+ Platform.operatingSystem +'-plugin 0.0.1'
    });

    streamSub = ref.collection('VideoCall').document(_session_id).snapshots().listen((snap){
        print('Recivied data: ' + snap.data.toString());
        this.onMessage(snap.data);
    });


  }



  _send(event, data) {
    if (_selfId.hashCode <= peerID.hashCode) {
      this._session_id = this._selfId+'-'+peerID;//'$this._selfId-$peerID';
    } else {
      this._session_id = peerID+'-'+this._selfId;//'$peerID-$_selfId';
    }
    data['type'] = event;
    JsonEncoder encoder = new JsonEncoder();
    ref.collection('VideoCall').document(_session_id).setData(data);
    print('send: ' + encoder.convert(data));

  }
}

and my videocall Screen code

import 'package:flutter/material.dart';
import 'package:flutter_webrtc/webrtc.dart';
import 'auth.dart';
import 'video_call_signaling.dart';


class VideoCall extends StatefulWidget{
  final String peerId, peerDisplayName, peerPhotoUrl;
  VideoCall(this.peerId, this.peerDisplayName, this.peerPhotoUrl);
  _VideoCallState createState() => new _VideoCallState();
}

class _VideoCallState extends State<VideoCall> {
  Signaling _signaling;
  String peerId , peerDisplayName, peerPhotoUrl;
  String _displayName = currentUserModel.displayName;
  String _selfId;
  bool inCall = false;

  RTCVideoRenderer _localRenderer = new RTCVideoRenderer();
  RTCVideoRenderer _remoteRenderer = new RTCVideoRenderer();


  @override
  void initState() {
    print('init video call class');
    _selfId = currentUserModel.id;
    peerId = widget.peerId;
    peerDisplayName = widget.peerDisplayName;
    peerPhotoUrl = widget.peerPhotoUrl;
    initRenders();
    _connect();
    _invitePeer(context, peerId);
  }

  @override
  void dispose() {
    _hangUp();
    super.dispose();
  }

  @override
  Widget build(BuildContext context) {
    return Scaffold(
      body: inCall? buildInCallView() : buildConnectingView(),
      floatingActionButton: inCall
          ? FloatingActionButton(
        onPressed: _hangUp,
        tooltip: 'Hangup',
        child: new Icon(Icons.call_end),
      )
          : null,
    );
  }

  buildInCallView() {
    return  OrientationBuilder(builder: (context, orientation) {
      return  Container(
        child:  Stack(children: <Widget>[
           Positioned(
              left: 0.0,
              right: 0.0,
              top: 0.0,
              bottom: 0.0,
              child:  Container(
                margin:  EdgeInsets.fromLTRB(0.0, 0.0, 0.0, 0.0),
                width: MediaQuery.of(context).size.width,
                height: MediaQuery.of(context).size.height,
                child: new RTCVideoView(_remoteRenderer),
                decoration: new BoxDecoration(color: Colors.black54),
              )),
           Positioned(
            left: 20.0,
            top: 20.0,
            child:  Container(
              width: orientation == Orientation.portrait ? 90.0 : 120.0,
              height:
              orientation == Orientation.portrait ? 120.0 : 90.0,
              child: new RTCVideoView(_localRenderer),
              decoration:  BoxDecoration(color: Colors.black54),
            ),
          ),
        ]),
      );
    });
  }

  buildConnectingView() {
    return OrientationBuilder(builder: (context, orientation){
      return Container(
        child: Stack(
          children: <Widget>[
            Container(
              child: Center(
                child: CircularProgressIndicator(),
              ),
            )
          ],
        ),
      );
    });
  }

  void initRenders() async{
    await _localRenderer.initialize();
    await _remoteRenderer.initialize();
  }



  _hangUp() {
    if (_signaling != null) {
      this.setState(() {
        _localRenderer.srcObject = null;
        _remoteRenderer.srcObject = null;
        inCall = false;

      });
      _signaling.bye();
    }
  }

  void _invitePeer(BuildContext context, String peerId) async{
    print('line 115invite peer, videocall');
    if (_signaling != null && peerId != _selfId) {
      _signaling.invite(peerId, 'video');
    }
  }

  void _connect() async{
    if (_signaling == null){
      _signaling = new Signaling( _displayName, peerId)
        ..connect();

      _signaling.onStateChange = (SignalingState state) {
        switch (state) {
          case SignalingState.CallStateNew:
            this.setState(() {
              inCall = true;
            });
            break;
          case SignalingState.CallStateBye:
            this.setState(() {
              _localRenderer.srcObject = null;
              _remoteRenderer.srcObject = null;
              inCall = false;
              print('line 140 videocall');
              Navigator.pop(context);
            });

            break;
          case SignalingState.CallStateInvite:
            print('line 152 videocall ');
            break;
          case SignalingState.CallStateConnected:
            print('line 155 videocall ');
            break;
          case SignalingState.CallStateInvite:
            print('line 158 videocall ');
            break;
          case SignalingState.CallStateRinging:
            print('line 161 videocall ');

            break;
          case SignalingState.ConnectionClosed:
            print('line 164 videocall ');
            break;
          case SignalingState.ConnectionError:
            print('line 167 videocall ');
            break;
          case SignalingState.ConnectionOpen:
            print('line 170 videocall ');
            break;
        }
        };
      _signaling.onLocalStream = ((stream) {
        _localRenderer.srcObject = stream;
      });
      _signaling.onAddRemoteStream = ((stream) {
        _remoteRenderer.srcObject = stream;
      });

      _signaling.onRemoveRemoteStream = ((stream) {
        _remoteRenderer.srcObject = null;
      });

    }
  }
}

I don't get how I have to show call lift screen such that both users can connect and see their other peer video.
Right now I'm getting only local stream video.

You have many callback unused like shown below. when and where to use them?

typedef void SignalingStateCallback(SignalingState state);
typedef void StreamStateCallback(MediaStream stream);
typedef void OtherEventCallback(dynamic event);
typedef void DataChannelMessageCallback(RTCDataChannel dc, data);

[rostopira]

Could you add some details?
What devices do you use?
On which ice connection state you stuck?
Does onAddStream called?

[harshapulikollu]

I'm using Xiaomi Redmi 5A and LG G6 as testing devices.
My firestore stops updating data when type = candidate, as It seen it screenshots.
As seen in screenshots I'm getting null value in to field sometimes and userId sometimes. (Don't understand exact reason).


Yes, on AddStream is called in signalling class around 100th line in my initial comment.

pc.onAddStream = (stream) {
      if(this.onAddRemoteStream != null)
        this.onAddRemoteStream(stream);
      //_remoteStreams.add(stream);
    };

[rostopira]

send function is completely wrong, because it's overwriting all data in session
It's called multiple times, first time with Offer/answer and then 2-10 times for each ice candidate,
Instead of document, you should use collection and append to it on each send call
Check out this article, it explains basic concept well:
https://websitebeaver.com/insanely-simple-webrtc-video-chat-using-firebase-with-codepen-demo

[harshapulikollu]

okay thank you. I'll try it then ping with the results.

[harshapulikollu]

@rostopira As you said, I tried to use collection by changing send method

_send(event ,Map data) async{
   if (_selfId.hashCode <= peerID.hashCode) {
     this._session_id = this._selfId+'-'+peerID;//'$this._selfId-$peerID';
   } else {
     this._session_id = peerID+'-'+this._selfId;//'$peerID-$_selfId';
   }
  data['type'] = event;
   Firestore.instance.collection('VideoCall').document(_session_id)
       .collection('VideoCall').add(new Map<String, dynamic>.from(data));
 }

And I'm listening like

Firestore.instance.collection('VideoCall').document(_session_id).collection('VideoCall').snapshots().listen((snapshot){
      snapshot.documents.forEach((docSnap){
        this.onMessage(docSnap.data);
      });

Haven't changed onMessage method just same as in my initial comment.
From that I have got new logs Log FIle Link which I haven't got in past but with in 1-3 seconds after showing local media all of sudden without any error app crashes and log cat won't register any error or not showing issue.