perf(audio-stream): reduce memory allocation on send audio frame

Author: H4ad

src/app/Media/VoIPMediaSession.cs

@@ -338,13 +338,13 @@ protected void RtpMediaPacketReceived(IPEndPoint remoteEndPoint, SDPMediaTypesEn
             {
                 logger.LogTrace(nameof(RtpMediaPacketReceived) + " audio RTP packet received from {RemoteEndPoint} ssrc {SyncSource} seqnum {SequenceNumber} timestamp {Timestamp} payload type {PayloadType}.", remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType);
 
-                Media.AudioSink.GotAudioRtp(remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType, marker, rtpPacket.Payload);
+                Media.AudioSink.GotAudioRtp(remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType, marker, rtpPacket.GetPayloadBytes());
             }
             else if (mediaType == SDPMediaTypesEnum.text && Media.TextSink != null)
             {
                 logger.LogTrace(nameof(RtpMediaPacketReceived) + " text RTP packet received from {RemoteEndPoint} ssrc {SyncSource} seqnum {SequenceNumber} timestamp {Timestamp} payload type {PayloadType}.", remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType);
 
-                Media.TextSink.GotTextRtp(remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType, hdr.MarkerBit, rtpPacket.Payload);
+                Media.TextSink.GotTextRtp(remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType, hdr.MarkerBit, rtpPacket.GetPayloadBytes());
             }
         }
 

src/net/RTCP/RTCPSession.cs

@@ -234,7 +234,7 @@ public void RecordRtpPacketReceived(RTPPacket rtpPacket)
             LastActivityAt = DateTime.Now;
             IsTimedOut = false;
             PacketsReceivedCount++;
-            OctetsReceivedCount += (uint)rtpPacket.Payload.Length;
+            OctetsReceivedCount += rtpPacket.GetPayloadLength();
 
             if (m_receptionReport == null)
             {
@@ -272,7 +272,7 @@ public void RecordRtpPacketSend(RTPPacket rtpPacket)
             }
 
             PacketsSentCount++;
-            OctetsSentCount += (uint)rtpPacket.Payload.Length;
+            OctetsSentCount += rtpPacket.GetPayloadLength();
             LastSeqNum = rtpPacket.Header.SequenceNumber;
             LastRtpTimestampSent = rtpPacket.Header.Timestamp;
             LastNtpTimestampSent = DateTimeToNtpTimestamp(DateTime.Now);

src/net/RTP/AudioStream.cs

@@ -61,7 +61,7 @@ public bool HasAudio
         /// <param name="durationRtpUnits">The duration in RTP timestamp units of the audio sample. This
         /// value is added to the previous RTP timestamp when building the RTP header.</param>
         /// <param name="sample">The audio sample to set as the RTP packet payload.</param>
-        public void SendAudio(uint durationRtpUnits, byte[] sample)
+        public void SendAudio(uint durationRtpUnits, ArraySegment<byte> sample)
         {
             if (!sendingFormatFound)
             {
@@ -71,14 +71,25 @@ public void SendAudio(uint durationRtpUnits, byte[] sample)
             SendAudioFrame(durationRtpUnits, NegotiatedFormat.ID, sample);
         }
 
+        /// <summary>
+        /// Sends an audio sample to the remote peer.
+        /// </summary>
+        /// <param name="durationRtpUnits">The duration in RTP timestamp units of the audio sample. This
+        /// value is added to the previous RTP timestamp when building the RTP header.</param>
+        /// <param name="sample">The audio sample to set as the RTP packet payload.</param>
+        public void SendAudio(uint durationRtpUnits, byte[] sample)
+        {
+            SendAudio(durationRtpUnits, new ArraySegment<byte>(sample));
+        }
+
         /// <summary>
         /// Sends an audio packet to the remote party.
         /// </summary>
         /// <param name="duration">The duration of the audio payload in timestamp units. This value
         /// gets added onto the timestamp being set in the RTP header.</param>
         /// <param name="payloadTypeID">The payload ID to set in the RTP header.</param>
-        /// <param name="buffer">The audio payload to send.</param>
-        public void SendAudioFrame(uint duration, int payloadTypeID, byte[] buffer)
+        /// <param name="bufferSegment">The audio payload to send.</param>
+        public void SendAudioFrame(uint duration, int payloadTypeID, ArraySegment<byte> bufferSegment)
         {
             if (CheckIfCanSendRtpRaw())
             {
@@ -99,29 +110,30 @@ public void SendAudioFrame(uint duration, int payloadTypeID, byte[] buffer)
                     // See https://github.com/sipsorcery/sipsorcery/issues/394.
 
                     int maxPayload = RTPSession.RTP_MAX_PAYLOAD;
-                    int totalPackets = (buffer.Length + maxPayload - 1) / maxPayload;
+                    int totalPackets = (bufferSegment.Count + maxPayload - 1) / maxPayload;
 
                     uint totalIncrement = 0;
                     uint startTimestamp = LocalTrack.Timestamp; // Keep track of where we started.
 
                     for (int index = 0; index < totalPackets; index++)
                     {
                         int offset = index * maxPayload;
-                        int payloadLength = Math.Min(maxPayload, buffer.Length - offset);
+                        int payloadLength = Math.Min(maxPayload, bufferSegment.Count - offset);
 
-                        double fraction = (double)payloadLength / buffer.Length;
+                        double fraction = (double)payloadLength / bufferSegment.Count;
                         uint packetDuration = (uint)Math.Round(fraction * duration);
 
-                        byte[] payload = new byte[payloadLength];
-                        Buffer.BlockCopy(buffer, offset, payload, 0, payloadLength);
-
                         // RFC3551 specifies that for audio the marker bit should always be 0 except for when returning
                         // from silence suppression. For video the marker bit DOES get set to 1 for the last packet
                         // in a frame.
                         int markerBit = 0;
-
+#if NETCOREAPP2_1_OR_GREATER && !NETFRAMEWORK
+                        var memorySegment = bufferSegment.Slice(offset, payloadLength);
+#else
+                        var memorySegment = new ArraySegment<byte>(bufferSegment.Array!, offset, payloadLength);
+#endif
                         // Send this packet at the current LocalTrack.Timestamp
-                        SendRtpRaw(payload, LocalTrack.Timestamp, markerBit, payloadTypeID, true);
+                        SendRtpRaw(memorySegment, LocalTrack.Timestamp, markerBit, payloadTypeID, true);
 
                         // After sending, increment the timestamp by this packet's portion.
                         // This ensures the timestamp increments for the next packet, including the first one.
@@ -143,6 +155,18 @@ public void SendAudioFrame(uint duration, int payloadTypeID, byte[] buffer)
             }
         }
 
+        /// <summary>
+        /// Sends an audio packet to the remote party.
+        /// </summary>
+        /// <param name="duration">The duration of the audio payload in timestamp units. This value
+        /// gets added onto the timestamp being set in the RTP header.</param>
+        /// <param name="payloadTypeID">The payload ID to set in the RTP header.</param>
+        /// <param name="buffer">The audio payload to send.</param>
+        public void SendAudioFrame(uint duration, int payloadTypeID, byte[] buffer)
+        {
+            SendAudioFrame(duration, payloadTypeID, new ArraySegment<byte>(buffer));
+        }
+
         /// <summary>
         /// Sends an RTP event for a DTMF tone as per RFC2833. Sending the event requires multiple packets to be sent.
         /// This method will hold onto the socket until all the packets required for the event have been sent. The send

src/net/RTP/MediaStream.cs

@@ -377,14 +377,15 @@ private static byte[] Combine(params byte[][] arrays)
             return rv;
         }
 
-        protected void SendRtpRaw(byte[] data, uint timestamp, int markerBit, int payloadType, Boolean checkDone, ushort? seqNum = null)
+        protected void SendRtpRaw(ArraySegment<byte> data, uint timestamp, int markerBit, int payloadType, Boolean checkDone, ushort? seqNum = null)
         {
             if (checkDone || CheckIfCanSendRtpRaw())
             {
                 ProtectRtpPacket protectRtpPacket = SecureContext?.ProtectRtpPacket;
                 int srtpProtectionLength = (protectRtpPacket != null) ? RTPSession.SRTP_MAX_PREFIX_LENGTH : 0;
 
-                RTPPacket rtpPacket = new RTPPacket(data.Length + srtpProtectionLength);
+                RTPPacket rtpPacket = new RTPPacket(data, srtpProtectionLength);
+
                 rtpPacket.Header.SyncSource = LocalTrack.Ssrc;
                 rtpPacket.Header.SequenceNumber = seqNum ?? LocalTrack.GetNextSeqNum();
                 rtpPacket.Header.Timestamp = timestamp;
@@ -452,8 +453,6 @@ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
                     rtpPacket.Header.HeaderExtensionFlag = 0;
                 }
 
-                Buffer.BlockCopy(data, 0, rtpPacket.Payload, 0, data.Length);
-
                 var rtpBuffer = rtpPacket.GetBytes();
 
                 if (protectRtpPacket == null)
@@ -478,6 +477,11 @@ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
             }
         }
 
+        protected void SendRtpRaw(byte[] data, uint timestamp, int markerBit, int payloadType, Boolean checkDone, ushort? seqNum = null)
+        {
+            SendRtpRaw(new ArraySegment<byte>(data), timestamp, markerBit, payloadType, checkDone, seqNum);
+        }
+
         /// <summary>
         /// To set a new value to a RTP Header extension.
         /// 
@@ -679,7 +683,7 @@ public void OnReceiveRTPPacket(RTPHeader hdr, int localPort, IPEndPoint remoteEn
                     return;
                 }
 
-                RaiseOnRtpEventByIndex(remoteEndPoint, new RTPEvent(rtpPacket.Payload), rtpPacket.Header);
+                RaiseOnRtpEventByIndex(remoteEndPoint, new RTPEvent(rtpPacket.GetPayloadBytes()), rtpPacket.Header);
                 return;
             }
 

src/net/RTP/Packetisation/RtpVideoFramer.cs

@@ -63,7 +63,7 @@ public RtpVideoFramer(VideoCodecsEnum codec, int maxFrameSize)
 
         public byte[] GotRtpPacket(RTPPacket rtpPacket)
         {
-            var payload = rtpPacket.Payload;
+            var payload = rtpPacket.GetPayloadBytes();
 
             var hdr = rtpPacket.Header;
 

src/net/RTP/RTPPacket.cs

@@ -15,13 +15,24 @@
 //-----------------------------------------------------------------------------
 
 using System;
+#if !NETCOREAPP2_1_OR_GREATER || NETFRAMEWORK
+using System.Linq;
+#endif
 
 namespace SIPSorcery.Net
 {
     public class RTPPacket
     {
         public RTPHeader Header;
-        public byte[] Payload;
+        private byte[] _payload;
+        private ArraySegment<byte> _payloadSegment;
+        private int _srtpProtectionLength = 0;
+
+        public byte[] Payload
+        {
+            get { return _payload; }
+            set { _payload = value; }
+        }
 
         public RTPPacket()
         {
@@ -31,23 +42,81 @@ public RTPPacket()
         public RTPPacket(int payloadSize)
         {
             Header = new RTPHeader();
-            Payload = new byte[payloadSize];
+            _payload = new byte[payloadSize];
         }
 
         public RTPPacket(byte[] packet)
         {
             Header = new RTPHeader(packet);
-            Payload = new byte[Header.PayloadSize];
-            Array.Copy(packet, Header.Length, Payload, 0, Payload.Length);
+            _payload = new byte[Header.PayloadSize];
+            Array.Copy(packet, Header.Length, _payload, 0, _payload.Length);
+        }
+
+        public RTPPacket(ArraySegment<byte> packet, int srtpProtectionLength)
+        {
+            Header = new RTPHeader();
+            _payloadSegment = packet;
+            _srtpProtectionLength = srtpProtectionLength;
+        }
+
+        public uint GetPayloadLength()
+        {
+            return (uint)(_payload?.Length ?? _payloadSegment.Count);
+        }
+
+        public byte[] GetPayloadBytes()
+        {
+            Payload ??= _payloadSegment.ToArray();
+
+            return Payload;
+        }
+
+        public byte GetPayloadByteAt(int index)
+        {
+#if NETCOREAPP2_1_OR_GREATER && !NETFRAMEWORK
+            return _payload?[index] ?? _payloadSegment[index];
+#else
+            return _payload?[index] ?? _payloadSegment.ElementAt(index);
+#endif
+        }
+
+        public ArraySegment<byte> GetPayloadSegment(int offset, int length)
+        {
+            if (_payload != null)
+            {
+                return new ArraySegment<byte>(_payload, offset, length);
+            }
+
+#if NETCOREAPP2_1_OR_GREATER && !NETFRAMEWORK
+            return _payloadSegment.Slice(offset, length);
+#else
+            return new ArraySegment<byte>(_payloadSegment.Array!, offset + _payloadSegment.Offset, length);
+#endif
         }
 
         public byte[] GetBytes()
         {
             byte[] header = Header.GetBytes();
-            byte[] packet = new byte[header.Length + Payload.Length];
+            byte[] packet = new byte[header.Length + (_payload?.Length ?? _payloadSegment.Count) + _srtpProtectionLength];
 
             Array.Copy(header, packet, header.Length);
-            Array.Copy(Payload, 0, packet, header.Length, Payload.Length);
+
+            if (_payloadSegment != null)
+            {
+#if NETCOREAPP2_1_OR_GREATER && !NETFRAMEWORK
+                _payloadSegment.CopyTo(packet, header.Length);
+#else
+                Array.Copy(_payloadSegment.Array!, _payloadSegment.Offset, packet, header.Length, _payloadSegment.Count);
+#endif
+            }
+            else if (_payload != null)
+            {
+                Array.Copy(_payload, 0, packet, header.Length, _payload.Length);
+            }
+            else
+            {
+                throw new ApplicationException("Either _payloadSegment or _payload should be defined");
+            }
 
             return packet;
         }
@@ -66,15 +135,15 @@ private byte[] GetNullPayload(int numBytes)
 
         public static bool TryParse(
             ReadOnlySpan<byte> buffer,
-             RTPPacket packet,
+            RTPPacket packet,
             out int consumed)
         {
             consumed = 0;
             if (RTPHeader.TryParse(buffer, out var header, out var headerConsumed))
             {
                 packet.Header = header;
                 consumed += headerConsumed;
-                packet.Payload = buffer.Slice(headerConsumed, header.PayloadSize).ToArray();
+                packet._payload = buffer.Slice(headerConsumed, header.PayloadSize).ToArray();
                 consumed += header.PayloadSize;
                 return true;
             }
@@ -91,4 +160,4 @@ public static bool TryParse(
             return TryParse(buffer, packet, out consumed);
         }
     }
-}
+}