perf(audio-stream): reduce memory allocation on send audio frame

Author: H4ad

src/app/Media/VoIPMediaSession.cs

@@ -338,13 +338,13 @@ protected void RtpMediaPacketReceived(IPEndPoint remoteEndPoint, SDPMediaTypesEn
             {
                 logger.LogTrace(nameof(RtpMediaPacketReceived) + " audio RTP packet received from {RemoteEndPoint} ssrc {SyncSource} seqnum {SequenceNumber} timestamp {Timestamp} payload type {PayloadType}.", remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType);
 
-                Media.AudioSink.GotAudioRtp(remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType, marker, rtpPacket.Payload);
+                Media.AudioSink.GotAudioRtp(remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType, marker, rtpPacket.GetPayloadBytes());
             }
             else if (mediaType == SDPMediaTypesEnum.text && Media.TextSink != null)
             {
                 logger.LogTrace(nameof(RtpMediaPacketReceived) + " text RTP packet received from {RemoteEndPoint} ssrc {SyncSource} seqnum {SequenceNumber} timestamp {Timestamp} payload type {PayloadType}.", remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType);
 
-                Media.TextSink.GotTextRtp(remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType, hdr.MarkerBit, rtpPacket.Payload);
+                Media.TextSink.GotTextRtp(remoteEndPoint, hdr.SyncSource, hdr.SequenceNumber, hdr.Timestamp, hdr.PayloadType, hdr.MarkerBit, rtpPacket.GetPayloadBytes());
             }
         }
 

src/net/RTCP/RTCPSession.cs

@@ -234,7 +234,7 @@ public void RecordRtpPacketReceived(RTPPacket rtpPacket)
             LastActivityAt = DateTime.Now;
             IsTimedOut = false;
             PacketsReceivedCount++;
-            OctetsReceivedCount += (uint)rtpPacket.Payload.Length;
+            OctetsReceivedCount += rtpPacket.GetPayloadLength();
 
             if (m_receptionReport == null)
             {
@@ -272,7 +272,7 @@ public void RecordRtpPacketSend(RTPPacket rtpPacket)
             }
 
             PacketsSentCount++;
-            OctetsSentCount += (uint)rtpPacket.Payload.Length;
+            OctetsSentCount += rtpPacket.GetPayloadLength();
             LastSeqNum = rtpPacket.Header.SequenceNumber;
             LastRtpTimestampSent = rtpPacket.Header.Timestamp;
             LastNtpTimestampSent = DateTimeToNtpTimestamp(DateTime.Now);

src/net/RTP/AudioStream.cs

@@ -61,7 +61,7 @@ public bool HasAudio
         /// <param name="durationRtpUnits">The duration in RTP timestamp units of the audio sample. This
         /// value is added to the previous RTP timestamp when building the RTP header.</param>
         /// <param name="sample">The audio sample to set as the RTP packet payload.</param>
-        public void SendAudio(uint durationRtpUnits, byte[] sample)
+        public void SendAudio(uint durationRtpUnits, ArraySegment<byte> sample)
         {
             if (!sendingFormatFound)
             {
@@ -71,14 +71,25 @@ public void SendAudio(uint durationRtpUnits, byte[] sample)
             SendAudioFrame(durationRtpUnits, NegotiatedFormat.ID, sample);
         }
 
+        /// <summary>
+        /// Sends an audio sample to the remote peer.
+        /// </summary>
+        /// <param name="durationRtpUnits">The duration in RTP timestamp units of the audio sample. This
+        /// value is added to the previous RTP timestamp when building the RTP header.</param>
+        /// <param name="sample">The audio sample to set as the RTP packet payload.</param>
+        public void SendAudio(uint durationRtpUnits, byte[] sample)
+        {
+            SendAudio(durationRtpUnits, new ArraySegment<byte>(sample));
+        }
+
         /// <summary>
         /// Sends an audio packet to the remote party.
         /// </summary>
         /// <param name="duration">The duration of the audio payload in timestamp units. This value
         /// gets added onto the timestamp being set in the RTP header.</param>
         /// <param name="payloadTypeID">The payload ID to set in the RTP header.</param>
-        /// <param name="buffer">The audio payload to send.</param>
-        public void SendAudioFrame(uint duration, int payloadTypeID, byte[] buffer)
+        /// <param name="bufferSegment">The audio payload to send.</param>
+        public void SendAudioFrame(uint duration, int payloadTypeID, ArraySegment<byte> bufferSegment)
         {
             if (CheckIfCanSendRtpRaw())
             {
@@ -99,29 +110,30 @@ public void SendAudioFrame(uint duration, int payloadTypeID, byte[] buffer)
                     // See https://github.com/sipsorcery/sipsorcery/issues/394.
 
                     int maxPayload = RTPSession.RTP_MAX_PAYLOAD;
-                    int totalPackets = (buffer.Length + maxPayload - 1) / maxPayload;
+                    int totalPackets = (bufferSegment.Count + maxPayload - 1) / maxPayload;
 
                     uint totalIncrement = 0;
                     uint startTimestamp = LocalTrack.Timestamp; // Keep track of where we started.
 
                     for (int index = 0; index < totalPackets; index++)
                     {
                         int offset = index * maxPayload;
-                        int payloadLength = Math.Min(maxPayload, buffer.Length - offset);
+                        int payloadLength = Math.Min(maxPayload, bufferSegment.Count - offset);
 
-                        double fraction = (double)payloadLength / buffer.Length;
+                        double fraction = (double)payloadLength / bufferSegment.Count;
                         uint packetDuration = (uint)Math.Round(fraction * duration);
 
-                        byte[] payload = new byte[payloadLength];
-                        Buffer.BlockCopy(buffer, offset, payload, 0, payloadLength);
-
                         // RFC3551 specifies that for audio the marker bit should always be 0 except for when returning
                         // from silence suppression. For video the marker bit DOES get set to 1 for the last packet
                         // in a frame.
                         int markerBit = 0;
-
+#if NETCOREAPP2_1_OR_GREATER && !NETFRAMEWORK
+                        var memorySegment = bufferSegment.Slice(offset, payloadLength);
+#else
+                        var memorySegment = new ArraySegment<byte>(bufferSegment.Array!, offset, payloadLength);
+#endif
                         // Send this packet at the current LocalTrack.Timestamp
-                        SendRtpRaw(payload, LocalTrack.Timestamp, markerBit, payloadTypeID, true);
+                        SendRtpRaw(memorySegment, LocalTrack.Timestamp, markerBit, payloadTypeID, true);
 
                         // After sending, increment the timestamp by this packet's portion.
                         // This ensures the timestamp increments for the next packet, including the first one.
@@ -143,6 +155,18 @@ public void SendAudioFrame(uint duration, int payloadTypeID, byte[] buffer)
             }
         }
 
+        /// <summary>
+        /// Sends an audio packet to the remote party.
+        /// </summary>
+        /// <param name="duration">The duration of the audio payload in timestamp units. This value
+        /// gets added onto the timestamp being set in the RTP header.</param>
+        /// <param name="payloadTypeID">The payload ID to set in the RTP header.</param>
+        /// <param name="buffer">The audio payload to send.</param>
+        public void SendAudioFrame(uint duration, int payloadTypeID, byte[] buffer)
+        {
+            SendAudioFrame(duration, payloadTypeID, new ArraySegment<byte>(buffer));
+        }
+
         /// <summary>
         /// Sends an RTP event for a DTMF tone as per RFC2833. Sending the event requires multiple packets to be sent.
         /// This method will hold onto the socket until all the packets required for the event have been sent. The send

src/net/RTP/MediaStream.cs

@@ -377,14 +377,15 @@ private static byte[] Combine(params byte[][] arrays)
             return rv;
         }
 
-        protected void SendRtpRaw(byte[] data, uint timestamp, int markerBit, int payloadType, Boolean checkDone, ushort? seqNum = null)
+        protected void SendRtpRaw(ArraySegment<byte> data, uint timestamp, int markerBit, int payloadType, Boolean checkDone, ushort? seqNum = null)
         {
             if (checkDone || CheckIfCanSendRtpRaw())
             {
                 ProtectRtpPacket protectRtpPacket = SecureContext?.ProtectRtpPacket;
                 int srtpProtectionLength = (protectRtpPacket != null) ? RTPSession.SRTP_MAX_PREFIX_LENGTH : 0;
 
-                RTPPacket rtpPacket = new RTPPacket(data.Length + srtpProtectionLength);
+                RTPPacket rtpPacket = new RTPPacket(data, srtpProtectionLength);
+
                 rtpPacket.Header.SyncSource = LocalTrack.Ssrc;
                 rtpPacket.Header.SequenceNumber = seqNum ?? LocalTrack.GetNextSeqNum();
                 rtpPacket.Header.Timestamp = timestamp;
@@ -452,8 +453,6 @@ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
                     rtpPacket.Header.HeaderExtensionFlag = 0;
                 }
 
-                Buffer.BlockCopy(data, 0, rtpPacket.Payload, 0, data.Length);
-
                 var rtpBuffer = rtpPacket.GetBytes();
 
                 if (protectRtpPacket == null)
@@ -478,6 +477,11 @@ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
             }
         }
 
+        protected void SendRtpRaw(byte[] data, uint timestamp, int markerBit, int payloadType, Boolean checkDone, ushort? seqNum = null)
+        {
+            SendRtpRaw(new ArraySegment<byte>(data), timestamp, markerBit, payloadType, checkDone, seqNum);
+        }
+
         /// <summary>
         /// To set a new value to a RTP Header extension.
         /// 
@@ -679,7 +683,7 @@ public void OnReceiveRTPPacket(RTPHeader hdr, int localPort, IPEndPoint remoteEn
                     return;
                 }
 
-                RaiseOnRtpEventByIndex(remoteEndPoint, new RTPEvent(rtpPacket.Payload), rtpPacket.Header);
+                RaiseOnRtpEventByIndex(remoteEndPoint, new RTPEvent(rtpPacket.GetPayloadBytes()), rtpPacket.Header);
                 return;
             }
 

src/net/RTP/Packetisation/RtpVideoFramer.cs

@@ -63,7 +63,7 @@ public RtpVideoFramer(VideoCodecsEnum codec, int maxFrameSize)
 
         public byte[] GotRtpPacket(RTPPacket rtpPacket)
         {
-            var payload = rtpPacket.Payload;
+            var payload = rtpPacket.GetPayloadBytes();
 
             var hdr = rtpPacket.Header;
 

src/net/RTP/RTPPacket.cs

@@ -21,7 +21,9 @@ namespace SIPSorcery.Net
     public class RTPPacket
     {
         public RTPHeader Header;
-        public byte[] Payload;
+        private byte[] _payload;
+        private ArraySegment<byte> _payloadSegment;
+        private int _srtpProtectionLength = 0;
 
         public RTPPacket()
         {
@@ -31,23 +33,66 @@ public RTPPacket()
         public RTPPacket(int payloadSize)
         {
             Header = new RTPHeader();
-            Payload = new byte[payloadSize];
+            _payload = new byte[payloadSize];
         }
 
         public RTPPacket(byte[] packet)
         {
             Header = new RTPHeader(packet);
-            Payload = new byte[Header.PayloadSize];
-            Array.Copy(packet, Header.Length, Payload, 0, Payload.Length);
+            _payload = new byte[Header.PayloadSize];
+            Array.Copy(packet, Header.Length, _payload, 0, _payload.Length);
+        }
+
+        public RTPPacket(ArraySegment<byte> packet, int srtpProtectionLength)
+        {
+            Header = new RTPHeader();
+            _payloadSegment = packet;
+            _srtpProtectionLength = srtpProtectionLength;
+        }
+
+        public uint GetPayloadLength()
+        {
+            return (uint)(_payload?.Length ?? _payloadSegment.Count);
+        }
+
+        public byte[] GetPayloadBytes()
+        {
+            _payload = _payload ?? _payloadSegment.ToArray();
+
+            return _payload;
+        }
+
+        public byte GetPayloadByteAt(int index)
+        {
+            return _payload?[index] ?? _payloadSegment[index];
+        }
+
+        public ArraySegment<byte> GetPayloadSegment(int offset, int length)
+        {
+            return _payload != null
+                ? new ArraySegment<byte>(_payload, offset, length)
+                : _payloadSegment.Slice(offset, length);
         }
 
         public byte[] GetBytes()
         {
             byte[] header = Header.GetBytes();
-            byte[] packet = new byte[header.Length + Payload.Length];
+            byte[] packet = new byte[header.Length + (_payload?.Length ?? _payloadSegment.Count) + _srtpProtectionLength];
 
             Array.Copy(header, packet, header.Length);
-            Array.Copy(Payload, 0, packet, header.Length, Payload.Length);
+
+            if (_payloadSegment != null)
+            {
+                _payloadSegment.CopyTo(packet, header.Length);
+            }
+            else if (_payload != null)
+            {
+                Array.Copy(_payload, 0, packet, header.Length, _payload.Length);
+            }
+            else
+            {
+                throw new ApplicationException("Either _payloadSegment or _payload should be defined");
+            }
 
             return packet;
         }
@@ -66,15 +111,15 @@ private byte[] GetNullPayload(int numBytes)
 
         public static bool TryParse(
             ReadOnlySpan<byte> buffer,
-             RTPPacket packet,
+            RTPPacket packet,
             out int consumed)
         {
             consumed = 0;
             if (RTPHeader.TryParse(buffer, out var header, out var headerConsumed))
             {
                 packet.Header = header;
                 consumed += headerConsumed;
-                packet.Payload = buffer.Slice(headerConsumed, header.PayloadSize).ToArray();
+                packet._payload = buffer.Slice(headerConsumed, header.PayloadSize).ToArray();
                 consumed += header.PayloadSize;
                 return true;
             }

src/net/RTSP/Mjpeg.cs

@@ -414,8 +414,8 @@ public static byte[] ProcessMjpegFrame(List<RTPPacket> framePackets)
 
                     //Decode RtpJpeg Header
 
-                    TypeSpecific = (uint)(packet.Payload[offset++]);
-                    FragmentOffset = (uint)(packet.Payload[offset++] << 16 | packet.Payload[offset++] << 8 | packet.Payload[offset++]);
+                    TypeSpecific = packet.GetPayloadByteAt(offset++);
+                    FragmentOffset = (uint)(packet.GetPayloadByteAt(offset++) << 16 | packet.GetPayloadByteAt(offset++) << 8 | packet.GetPayloadByteAt(offset++));
 
                     #region RFC2435 -  The Type Field
 
@@ -498,16 +498,16 @@ doubling the height of the image.
 
                     #endregion
 
-                    Type = (uint)(packet.Payload[offset++]);
+                    Type = packet.GetPayloadByteAt(offset++);
                     type = Type & 1;
                     if (type > 3 || type > 6)
                     {
                         throw new ArgumentException("Type numbers 2-5 are reserved and SHOULD NOT be used.  Applications on RFC 2035 should be updated to indicate the presence of restart markers with type 64 or 65 and the Restart Marker header.");
                     }
 
-                    Quality = (uint)packet.Payload[offset++];
-                    Width = (uint)(packet.Payload[offset++] * 8); // This should have been 128 or > and the standard would have worked for all resolutions
-                    Height = (uint)(packet.Payload[offset++] * 8);// Now in certain highres profiles you will need an OnVif extension before the RtpJpeg Header
+                    Quality = packet.GetPayloadByteAt(offset++);
+                    Width = (uint)(packet.GetPayloadByteAt(offset++) * 8);  // This should have been 128 or > and the standard would have worked for all resolutions
+                    Height = (uint)(packet.GetPayloadByteAt(offset++) * 8); // Now in certain highres profiles you will need an OnVif extension before the RtpJpeg Header
                                                                   //It is worth noting Rtp does not care what you send and more tags such as comments and or higher resolution pictures may be sent and these values will simply be ignored.
 
                     if (Width == 0 || Height == 0)
@@ -529,8 +529,8 @@ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
                            |       Restart Interval        |F|L|       Restart Count       |
                            +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
                          */
-                        RestartInterval = (ushort)(packet.Payload[offset++] << 8 | packet.Payload[offset++]);
-                        RestartCount = (ushort)((packet.Payload[offset++] << 8 | packet.Payload[offset++]) & 0x3fff);
+                        RestartInterval = (ushort)(packet.GetPayloadByteAt(offset++) << 8 | packet.GetPayloadByteAt(offset++));
+                        RestartCount = (ushort)((packet.GetPayloadByteAt(offset++) << 8 | packet.GetPayloadByteAt(offset++)) & 0x3fff);
                     }
 
                     //QTables Only occur in the first packet
@@ -539,7 +539,7 @@ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
                         //If the quality > 127 there are usually Quantization Tables
                         if (Quality > 127)
                         {
-                            if ((packet.Payload[offset++]) != 0)
+                            if ((packet.GetPayloadByteAt(offset++)) != 0)
                             {
                                 //Must Be Zero is Not Zero
                                 if (System.Diagnostics.Debugger.IsAttached)
@@ -549,7 +549,7 @@ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
                             }
 
                             //Precision
-                            PrecisionTable = (packet.Payload[offset++]);
+                            PrecisionTable = (packet.GetPayloadByteAt(offset++));
 
                             #region RFC2435 Length Field
 
@@ -584,15 +584,15 @@ those corresponding to the tables needed by the type in use MUST be
                             #endregion
 
                             //Length of all tables
-                            ushort Length = (ushort)(packet.Payload[offset++] << 8 | packet.Payload[offset++]);
+                            ushort Length = (ushort)(packet.GetPayloadByteAt(offset++) << 8 | packet.GetPayloadByteAt(offset++));
 
                             //If there is Table Data Read it
                             if (Length > 0)
                             {
-                                tables = new ArraySegment<byte>(packet.Payload, offset, (int)Length);
+                                tables = packet.GetPayloadSegment(offset, Length);
                                 offset += (int)Length;
                             }
-                            else if (Length > packet.Payload.Length - offset)
+                            else if (Length > packet.GetPayloadLength() - offset)
                             {
                                 continue; // The packet must be discarded
                             }
@@ -611,7 +611,7 @@ those corresponding to the tables needed by the type in use MUST be
                     }
 
                     //Write the Payload data from the offset
-                    Buffer.Write(packet.Payload, offset, packet.Payload.Length - offset);
+                    Buffer.Write(packet.GetPayloadBytes(), offset, (int)packet.GetPayloadLength() - offset);
                 }
 
                 //Check for EOI Marker