WebRTC gateway

// 1. Create peer connection
const config = {
  iceServers: [{ urls: 'stun:stun.l.google.com:19302' }]
};
const pc = new RTCPeerConnection(config);

// 2. Add audio track
const stream = await navigator.mediaDevices.getUserMedia({
  audio: {
    echoCancellation: true,
    noiseSuppression: true,
    autoGainControl: true,
    sampleRate: 48000
  }
});

stream.getTracks().forEach(track => {
  pc.addTrack(track, stream);
});

// 3. Handle incoming audio
pc.ontrack = (event) => {
  const audioElement = new Audio();
  audioElement.srcObject = event.streams[0];
  audioElement.play();
};

// 4. Create and send offer
const offer = await pc.createOffer();
await pc.setLocalDescription(offer);

ws.send(JSON.stringify({
  type: 'offer',
  sdp: offer.sdp
}));

// 5. Handle answer from backend
ws.onmessage = async (event) => {
  const msg = JSON.parse(event.data);
  
  if (msg.type === 'answer') {
    await pc.setRemoteDescription({
      type: 'answer',
      sdp: msg.sdp
    });
  } else if (msg.type === 'candidate') {
    await pc.addIceCandidate(msg.candidate);
  }
};

// 6. Send ICE candidates to backend
pc.onicecandidate = (event) => {
  if (event.candidate) {
    ws.send(JSON.stringify({
      type: 'candidate',
      candidate: event.candidate
    }));
  }
};

// Create peer connection with STUN server
var pcConfig = new RTCConfiguration {
    iceServers = new List<RTCIceServer> { 
        new RTCIceServer { 
            urls = "stun:stun.l.google.com:19302" 
        } 
    }
};
var peerConnection = new RTCPeerConnection(pcConfig);

// Add audio track with supported codec
var audioFormat = new AudioFormat(AudioCodecsEnum.OPUS, 111, 48000, 2, 
    "minptime=10;useinbandfec=1");
var track = new MediaStreamTrack(
    SDPMediaTypesEnum.audio, 
    false, 
    new List<SDPAudioVideoMediaFormat> { 
        new SDPAudioVideoMediaFormat(audioFormat) 
    }
);
peerConnection.addTrack(track);

// Handle RTP packets (incoming audio)
peerConnection.OnRtpPacketReceived += (ep, media, pkt) => {
    if (media == SDPMediaTypesEnum.audio) {
        // Pass encoded audio to AI agent
        BinaryMessageReceived?.Invoke(this, pkt.Payload);
    }
};

// Process offer and create answer
var offerInit = new RTCSessionDescriptionInit { 
    type = RTCSdpType.offer, 
    sdp = receivedSdp 
};
peerConnection.setRemoteDescription(offerInit);

var answer = peerConnection.createAnswer(null);
await peerConnection.setLocalDescription(answer);

// Send answer back via WebSocket
SendSignaling(new { type = "answer", sdp = answer.sdp });

// Client specifies preferred audio settings
const audioConfig = {
  input: {
    codec: 'OPUS',
    sampleRate: 48000,
    bitsPerSample: 16,
    channels: 1  // Mono for efficiency
  },
  output: {
    codec: 'OPUS',
    sampleRate: 48000,
    bitsPerSample: 16,
    channels: 1,
    frameDurationMs: 20  // 20ms frames
  }
};

{
  "type": "offer",
  "sdp": "v=0\r\no=- 1234567890 2 IN IP4 127.0.0.1\r\n..."
}

{
  "type": "answer",
  "sdp": "v=0\r\no=- 9876543210 2 IN IP4 127.0.0.1\r\n..."
}

{
  "type": "candidate",
  "candidate": {
    "candidate": "candidate:1 1 UDP 2130706431 192.168.1.100 54321 typ host",
    "sdpMid": "0",
    "sdpMLineIndex": 0,
    "usernameFragment": "abc123"
  }
}

private async Task StartSignalingLoop(CancellationToken cancellationToken) {
    var buffer = new ArraySegment<byte>(new byte[8192]);
    
    while (_signalingSocket.State == WebSocketState.Open) {
        var result = await _signalingSocket.ReceiveAsync(buffer, cancellationToken);
        
        if (result.MessageType == WebSocketMessageType.Text) {
            string message = Encoding.UTF8.GetString(buffer.Array, 0, result.Count);
            await HandleSignalingMessageAsync(message);
        }
    }
}

_peerConnection.ondatachannel += (dc) => {
    _logger.LogInformation($"Data channel established: {dc.label}");
    
    dc.onmessage += (rtcDc, proto, data) => {
        string text = Encoding.UTF8.GetString(data);
        TextMessageReceived?.Invoke(this, text);
    };
    
    // Send confirmation
    dc.send("Backend connected");
};

// Create data channel before creating offer
const dataChannel = pc.createDataChannel('chat');

dataChannel.onopen = () => {
  console.log('Data channel open');
  dataChannel.send('Hello from client');
};

dataChannel.onmessage = (event) => {
  console.log('Received:', event.data);
};

// Send text during conversation
dataChannel.send(JSON.stringify({
  type: 'metadata',
  userId: 'user-123'
}));

pc.onconnectionstatechange = () => {
  console.log('Connection state:', pc.connectionState);
  
  switch (pc.connectionState) {
    case 'connected':
      // Fully connected, media flowing
      showStatus('Connected to AI agent');
      break;
      
    case 'disconnected':
      // Temporary network issue
      showStatus('Connection interrupted');
      break;
      
    case 'failed':
      // Connection failed, retry needed
      showStatus('Connection failed');
      reconnect();
      break;
      
    case 'closed':
      // Clean shutdown
      showStatus('Call ended');
      break;
  }
};

pc.oniceconnectionstatechange = () => {
  console.log('ICE state:', pc.iceConnectionState);
};

peerConnection.onconnectionstatechange += (state) => {
    _logger.LogInformation($"WebRTC connection state: {state}");
    
    if (state == RTCPeerConnectionState.failed || 
        state == RTCPeerConnectionState.closed) {
        Disconnected?.Invoke(this, $"WebRTC State: {state}");
    }
};

public Task SendBinaryAsync(
    byte[] data, 
    int sampleRate, 
    int bitsPerSample, 
    int frameDurationMs, 
    CancellationToken cancellationToken
) {
    // Calculate RTP timestamp increment
    uint durationRtpUnits = (uint)(sampleRate * frameDurationMs) / 1000;
    
    // Send encoded audio via RTP
    _peerConnection.SendAudio(durationRtpUnits, data);
    
    return Task.CompletedTask;
}

private void OnRtpPacketHandler(
    IPEndPoint ep, 
    SDPMediaTypesEnum media, 
    RTPPacket pkt
) {
    if (media == SDPMediaTypesEnum.audio) {
        // Extract encoded payload from RTP packet
        // Pass to AI agent's audio decoder
        BinaryMessageReceived?.Invoke(this, pkt.Payload);
    }
}

import { RTCPeerConnection, mediaDevices } from 'react-native-webrtc';

const setupWebRTC = async () => {
  // Get microphone access
  const stream = await mediaDevices.getUserMedia({
    audio: true,
    video: false
  });

  const pc = new RTCPeerConnection({
    iceServers: [{ urls: 'stun:stun.l.google.com:19302' }]
  });

  // Add local audio track
  stream.getTracks().forEach(track => {
    pc.addTrack(track, stream);
  });

  // Create offer and follow same signaling flow
  const offer = await pc.createOffer();
  await pc.setLocalDescription(offer);
  
  // Send to backend via WebSocket
  ws.send(JSON.stringify({ type: 'offer', sdp: offer.sdp }));
};

import WebRTC

let config = RTCConfiguration()
config.iceServers = [RTCIceServer(urlStrings: ["stun:stun.l.google.com:19302"])]

let peerConnection = RTCPeerConnectionFactory().peerConnection(
    with: config,
    constraints: RTCMediaConstraints(mandatoryConstraints: nil, 
                                     optionalConstraints: nil),
    delegate: self
)

// Add audio track
let audioTrack = createAudioTrack()
peerConnection.add(audioTrack, streamIds: ["stream-id"])

// Create and send offer
peerConnection.offer(for: RTCMediaConstraints()) { sdp, error in
    guard let sdp = sdp else { return }
    peerConnection.setLocalDescription(sdp) { error in
        // Send SDP to backend
        sendSignaling(["type": "offer", "sdp": sdp.sdp])
    }
}

var validatedSessionTokenResult = CallWebsocketTokenGenerator.ValidateHmacToken(
    sessionToken, 
    sessionId, 
    clientId, 
    _backendAppConfig.WebhookTokenSecret, 
    out var validationError
);

if (!validatedSessionTokenResult) {
    return result.SetFailureResult(
        "AssignWebSocketToClientAsync:VALIDATION_FAILED", 
        validationError
    );
}

pc.onicegatheringstatechange = () => {
  console.log('ICE gathering state:', pc.iceGatheringState);
};

pc.onicecandidate = (event) => {
  if (event.candidate) {
    console.log('New ICE candidate:', event.candidate.candidate);
  } else {
    console.log('ICE gathering complete');
  }
};

setInterval(async () => {
  const stats = await pc.getStats();
  stats.forEach(report => {
    if (report.type === 'inbound-rtp' && report.mediaType === 'audio') {
      console.log('Packets lost:', report.packetsLost);
      console.log('Jitter:', report.jitter);
    }
  });
}, 5000);

pc.createOffer().then(offer => {
  console.log('Offer SDP:', offer.sdp);
});

ws.onmessage = (event) => {
  const msg = JSON.parse(event.data);
  if (msg.type === 'answer') {
    console.log('Answer SDP:', msg.sdp);
  }
};