diff options
Diffstat (limited to 'third_party/libwebrtc/modules/rtp_rtcp/source')
18 files changed, 1156 insertions, 101 deletions
diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/flexfec_header_reader_writer.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/flexfec_header_reader_writer.cc index cfca7cb066..3e6d04d59c 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/flexfec_header_reader_writer.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/flexfec_header_reader_writer.cc @@ -138,9 +138,9 @@ bool FlexfecHeaderReader::ReadFecHeader( mask_part0 <<= 1; ByteWriter<uint16_t>::WriteBigEndian(&data[byte_index], mask_part0); byte_index += kFlexfecPacketMaskSizes[0]; - if (k_bit0) { - // The first K-bit is set, and the packet mask is thus only 2 bytes long. - // We have finished reading the properties for current ssrc. + if (!k_bit0) { + // The first K-bit is clear, and the packet mask is thus only 2 bytes + // long. We have finished reading the properties for current ssrc. fec_packet->protected_streams[i].packet_mask_size = kFlexfecPacketMaskSizes[0]; } else { @@ -162,8 +162,8 @@ bool FlexfecHeaderReader::ReadFecHeader( mask_part1 <<= 2; ByteWriter<uint32_t>::WriteBigEndian(&data[byte_index], mask_part1); byte_index += kFlexfecPacketMaskSizes[1] - kFlexfecPacketMaskSizes[0]; - if (k_bit1) { - // The first K-bit is clear, but the second K-bit is set. The packet + if (!k_bit1) { + // The first K-bit is set, but the second K-bit is clear. The packet // mask is thus 6 bytes long. We have finished reading the properties // for current ssrc. fec_packet->protected_streams[i].packet_mask_size = @@ -273,8 +273,9 @@ void FlexfecHeaderWriter::FinalizeFecHeader( tmp_mask_part0 >>= 1; // Shift, thus clearing K-bit 0. ByteWriter<uint16_t>::WriteBigEndian(write_at, tmp_mask_part0); + *write_at |= 0x80; // Set K-bit 0. write_at += kFlexfecPacketMaskSizes[0]; - tmp_mask_part1 >>= 2; // Shift, thus clearing K-bit 1 and bit 15. + tmp_mask_part1 >>= 2; // Shift twice, thus clearing K-bit 1 and bit 15. ByteWriter<uint32_t>::WriteBigEndian(write_at, tmp_mask_part1); bool bit15 = (protected_stream.packet_mask[1] & 0x01) != 0; @@ -284,9 +285,9 @@ void FlexfecHeaderWriter::FinalizeFecHeader( bool bit46 = (protected_stream.packet_mask[5] & 0x02) != 0; bool bit47 = (protected_stream.packet_mask[5] & 0x01) != 0; if (!bit46 && !bit47) { - *write_at |= 0x80; // Set K-bit 1. write_at += kFlexfecPacketMaskSizes[1] - kFlexfecPacketMaskSizes[0]; } else { + *write_at |= 0x80; // Set K-bit 1. write_at += kFlexfecPacketMaskSizes[1] - kFlexfecPacketMaskSizes[0]; // Clear all trailing bits. memset(write_at, 0, @@ -307,14 +308,13 @@ void FlexfecHeaderWriter::FinalizeFecHeader( ByteWriter<uint16_t>::WriteBigEndian(write_at, tmp_mask_part0); bool bit15 = (protected_stream.packet_mask[1] & 0x01) != 0; if (!bit15) { - *write_at |= 0x80; // Set K-bit 0. write_at += kFlexfecPacketMaskSizes[0]; } else { + *write_at |= 0x80; // Set K-bit 0. write_at += kFlexfecPacketMaskSizes[0]; // Clear all trailing bits. memset(write_at, 0U, kFlexfecPacketMaskSizes[1] - kFlexfecPacketMaskSizes[0]); - *write_at |= 0x80; // Set K-bit 1. *write_at |= 0x40; // Set bit 15. write_at += kFlexfecPacketMaskSizes[1] - kFlexfecPacketMaskSizes[0]; } diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/flexfec_header_reader_writer_unittest.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/flexfec_header_reader_writer_unittest.cc index 6995ba3871..f25e0d8d2a 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/flexfec_header_reader_writer_unittest.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/flexfec_header_reader_writer_unittest.cc @@ -36,11 +36,12 @@ using ReceivedFecPacket = ForwardErrorCorrection::ReceivedFecPacket; using ::testing::Each; using ::testing::ElementsAreArray; -constexpr uint8_t kMask0[] = {0xAB, 0xCD}; // First K bit is set. -constexpr uint8_t kMask1[] = {0x12, 0x34, // First K bit cleared. - 0xF6, 0x78, 0x9A, 0xBC}; // Second K bit set. -constexpr uint8_t kMask2[] = {0x12, 0x34, // First K bit cleared. - 0x56, 0x78, 0x9A, 0xBC, // Second K bit cleared. +constexpr uint8_t kKBit = 1 << 7; +constexpr uint8_t kMask0[] = {0x2B, 0xCD}; // First K bit is cleared. +constexpr uint8_t kMask1[] = {0x92, 0x34, // First K bit set. + 0x76, 0x78, 0x9A, 0xBC}; // Second K bit cleared. +constexpr uint8_t kMask2[] = {0x92, 0x34, // First K bit set. + 0xD6, 0x78, 0x9A, 0xBC, // Second K bit set. 0xDE, 0xF0, 0x12, 0x34, 0x56, 0x78, 0x9A, 0xBC}; constexpr size_t kMediaPacketLength = 1234; @@ -186,11 +187,10 @@ void VerifyWrittenAndReadHeaders( } // namespace -TEST(FlexfecHeaderReaderTest, ReadsHeaderWithKBit0SetSingleStream) { - constexpr uint8_t kKBit0 = 1 << 7; +TEST(FlexfecHeaderReaderTest, ReadsHeaderWithKBit0ClearSingleStream) { constexpr size_t kExpectedFecHeaderSize = 12; constexpr uint16_t kSnBase = 0x0102; - constexpr uint8_t kFlexfecPktMask[] = {kKBit0 | 0x08, 0x81}; + constexpr uint8_t kFlexfecPktMask[] = {0x08, 0x81}; constexpr uint8_t kUlpfecPacketMask[] = {0x11, 0x02}; constexpr uint8_t kPacketData[] = { kFlexible, kPtRecovery, kLengthRecovery[0], kLengthRecovery[1], @@ -215,13 +215,11 @@ TEST(FlexfecHeaderReaderTest, ReadsHeaderWithKBit0SetSingleStream) { VerifyReadHeaders(kExpectedFecHeaderSize, read_packet, expected); } -TEST(FlexfecHeaderReaderTest, ReadsHeaderWithKBit1SetSingleStream) { - constexpr uint8_t kKBit0 = 0 << 7; - constexpr uint8_t kKBit1 = 1 << 7; +TEST(FlexfecHeaderReaderTest, ReadsHeaderWithKBit1ClearSingleStream) { constexpr size_t kExpectedFecHeaderSize = 16; constexpr uint16_t kSnBase = 0x0102; - constexpr uint8_t kFlexfecPktMask[] = {kKBit0 | 0x48, 0x81, // - kKBit1 | 0x02, 0x11, 0x00, 0x21}; + constexpr uint8_t kFlexfecPktMask[] = {kKBit | 0x48, 0x81, // + 0x02, 0x11, 0x00, 0x21}; constexpr uint8_t kUlpfecPacketMask[] = {0x91, 0x02, // 0x08, 0x44, 0x00, 0x84}; constexpr uint8_t kPacketData[] = { @@ -250,15 +248,13 @@ TEST(FlexfecHeaderReaderTest, ReadsHeaderWithKBit1SetSingleStream) { VerifyReadHeaders(kExpectedFecHeaderSize, read_packet, expected); } -TEST(FlexfecHeaderReaderTest, ReadsHeaderWithNoKBitsSetSingleStream) { - constexpr uint8_t kKBit0 = 0 << 7; - constexpr uint8_t kKBit1 = 0 << 7; +TEST(FlexfecHeaderReaderTest, ReadsHeaderWithBothKBitsSetSingleStream) { constexpr size_t kExpectedFecHeaderSize = 24; constexpr uint16_t kSnBase = 0x0102; - constexpr uint8_t kFlexfecPacketMask[] = {kKBit0 | 0x48, 0x81, // - kKBit1 | 0x02, 0x11, 0x00, 0x21, // - 0x01, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11}; + constexpr uint8_t kFlexfecPacketMask[] = {kKBit | 0x48, 0x81, // + kKBit | 0x02, 0x11, 0x00, 0x21, // + 0x01, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x11, 0x11}; constexpr uint8_t kUlpfecPacketMask[] = {0x91, 0x02, // 0x08, 0x44, 0x00, 0x84, // 0x04, 0x44, 0x44, 0x44, @@ -309,14 +305,13 @@ TEST(FlexfecHeaderReaderTest, ReadsHeaderWithNoKBitsSetSingleStream) { VerifyReadHeaders(kExpectedFecHeaderSize, read_packet, expected); } -TEST(FlexfecHeaderReaderTest, ReadsHeaderWithKBit0Set2Streams) { - constexpr uint8_t kKBit0 = 1 << 7; +TEST(FlexfecHeaderReaderTest, ReadsHeaderWithKBit0Clear2Streams) { constexpr size_t kExpectedFecHeaderSize = 16; constexpr uint16_t kSnBase0 = 0x0102; constexpr uint16_t kSnBase1 = 0x0304; - constexpr uint8_t kFlexfecPktMask1[] = {kKBit0 | 0x08, 0x81}; + constexpr uint8_t kFlexfecPktMask1[] = {0x08, 0x81}; constexpr uint8_t kUlpfecPacketMask1[] = {0x11, 0x02}; - constexpr uint8_t kFlexfecPktMask2[] = {kKBit0 | 0x04, 0x41}; + constexpr uint8_t kFlexfecPktMask2[] = {0x04, 0x41}; constexpr uint8_t kUlpfecPacketMask2[] = {0x08, 0x82}; constexpr uint8_t kPacketData[] = { @@ -349,18 +344,16 @@ TEST(FlexfecHeaderReaderTest, ReadsHeaderWithKBit0Set2Streams) { VerifyReadHeaders(kExpectedFecHeaderSize, read_packet, expected); } -TEST(FlexfecHeaderReaderTest, ReadsHeaderWithKBit1Set2Streams) { - constexpr uint8_t kKBit0 = 0 << 7; - constexpr uint8_t kKBit1 = 1 << 7; +TEST(FlexfecHeaderReaderTest, ReadsHeaderWithKBit1Clear2Streams) { constexpr size_t kExpectedFecHeaderSize = 24; constexpr uint16_t kSnBase0 = 0x0102; constexpr uint16_t kSnBase1 = 0x0304; - constexpr uint8_t kFlexfecPktMask1[] = {kKBit0 | 0x48, 0x81, // - kKBit1 | 0x02, 0x11, 0x00, 0x21}; + constexpr uint8_t kFlexfecPktMask1[] = {kKBit | 0x48, 0x81, // + 0x02, 0x11, 0x00, 0x21}; constexpr uint8_t kUlpfecPacketMask1[] = {0x91, 0x02, // 0x08, 0x44, 0x00, 0x84}; - constexpr uint8_t kFlexfecPktMask2[] = {kKBit0 | 0x57, 0x82, // - kKBit1 | 0x04, 0x33, 0x00, 0x51}; + constexpr uint8_t kFlexfecPktMask2[] = {kKBit | 0x57, 0x82, // + 0x04, 0x33, 0x00, 0x51}; constexpr uint8_t kUlpfecPacketMask2[] = {0xAF, 0x04, // 0x10, 0xCC, 0x01, 0x44}; constexpr uint8_t kPacketData[] = { @@ -398,24 +391,22 @@ TEST(FlexfecHeaderReaderTest, ReadsHeaderWithKBit1Set2Streams) { VerifyReadHeaders(kExpectedFecHeaderSize, read_packet, expected); } -TEST(FlexfecHeaderReaderTest, ReadsHeaderWithNoKBitsSet2Streams) { - constexpr uint8_t kKBit0 = 0 << 7; - constexpr uint8_t kKBit1 = 0 << 7; +TEST(FlexfecHeaderReaderTest, ReadsHeaderWithBothKBitsSet2Streams) { constexpr size_t kExpectedFecHeaderSize = 40; constexpr uint16_t kSnBase0 = 0x0102; constexpr uint16_t kSnBase1 = 0x0304; - constexpr uint8_t kFlexfecPktMask1[] = {kKBit0 | 0x48, 0x81, // - kKBit1 | 0x02, 0x11, 0x00, 0x21, // - 0x01, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11}; + constexpr uint8_t kFlexfecPktMask1[] = {kKBit | 0x48, 0x81, // + kKBit | 0x02, 0x11, 0x00, 0x21, // + 0x01, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x11, 0x11}; constexpr uint8_t kUlpfecPacketMask1[] = {0x91, 0x02, // 0x08, 0x44, 0x00, 0x84, // 0x04, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44}; - constexpr uint8_t kFlexfecPktMask2[] = {kKBit0 | 0x32, 0x84, // - kKBit1 | 0x05, 0x23, 0x00, 0x55, // - 0xA3, 0x22, 0x22, 0x22, - 0x22, 0x22, 0x22, 0x35}; + constexpr uint8_t kFlexfecPktMask2[] = {kKBit | 0x32, 0x84, // + kKBit | 0x05, 0x23, 0x00, 0x55, // + 0xA3, 0x22, 0x22, 0x22, + 0x22, 0x22, 0x22, 0x35}; constexpr uint8_t kUlpfecPacketMask2[] = {0x65, 0x08, // 0x14, 0x8C, 0x01, 0x56, // 0x8C, 0x88, 0x88, 0x88, @@ -490,29 +481,27 @@ TEST(FlexfecHeaderReaderTest, ReadsHeaderWithNoKBitsSet2Streams) { } TEST(FlexfecHeaderReaderTest, ReadsHeaderWithMultipleStreamsMultipleMasks) { - constexpr uint8_t kBit0 = 0 << 7; - constexpr uint8_t kBit1 = 1 << 7; constexpr size_t kExpectedFecHeaderSize = 44; constexpr uint16_t kSnBase0 = 0x0102; constexpr uint16_t kSnBase1 = 0x0304; constexpr uint16_t kSnBase2 = 0x0506; constexpr uint16_t kSnBase3 = 0x0708; - constexpr uint8_t kFlexfecPacketMask1[] = {kBit1 | 0x29, 0x91}; + constexpr uint8_t kFlexfecPacketMask1[] = {0x29, 0x91}; constexpr uint8_t kUlpfecPacketMask1[] = {0x53, 0x22}; - constexpr uint8_t kFlexfecPacketMask2[] = {kBit0 | 0x32, 0xA1, // - kBit1 | 0x02, 0x11, 0x00, 0x21}; + constexpr uint8_t kFlexfecPacketMask2[] = {kKBit | 0x32, 0xA1, // + 0x02, 0x11, 0x00, 0x21}; constexpr uint8_t kUlpfecPacketMask2[] = {0x65, 0x42, // 0x08, 0x44, 0x00, 0x84}; - constexpr uint8_t kFlexfecPacketMask3[] = {kBit0 | 0x48, 0x81, // - kBit0 | 0x02, 0x11, 0x00, 0x21, // + constexpr uint8_t kFlexfecPacketMask3[] = {kKBit | 0x48, 0x81, // + kKBit | 0x02, 0x11, 0x00, 0x21, // 0x01, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11, 0x11}; constexpr uint8_t kUlpfecPacketMask3[] = {0x91, 0x02, // 0x08, 0x44, 0x00, 0x84, // 0x04, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44, 0x44}; - constexpr uint8_t kFlexfecPacketMask4[] = {kBit0 | 0x32, 0x84, // - kBit1 | 0x05, 0x23, 0x00, 0x55}; + constexpr uint8_t kFlexfecPacketMask4[] = {kKBit | 0x32, 0x84, // + 0x05, 0x23, 0x00, 0x55}; constexpr uint8_t kUlpfecPacketMask4[] = {0x65, 0x08, // 0x14, 0x8C, 0x01, 0x54}; constexpr uint8_t kPacketData[] = {kFlexible, @@ -642,7 +631,7 @@ TEST(FlexfecHeaderReaderTest, ReadShortPacketWithKBit0SetShouldFail) { EXPECT_FALSE(reader.ReadFecHeader(&read_packet)); } -TEST(FlexfecHeaderReaderTest, ReadShortPacketWithKBit1SetShouldFail) { +TEST(FlexfecHeaderReaderTest, ReadShortPacketWithKBit1ClearShouldFail) { // Simulate short received packet. constexpr uint8_t kPacketData[] = { kFlexible, kPtRecovery, kLengthRecovery[0], kLengthRecovery[1], @@ -659,7 +648,7 @@ TEST(FlexfecHeaderReaderTest, ReadShortPacketWithKBit1SetShouldFail) { EXPECT_FALSE(reader.ReadFecHeader(&read_packet)); } -TEST(FlexfecHeaderReaderTest, ReadShortPacketWithKBit1ClearedShouldFail) { +TEST(FlexfecHeaderReaderTest, ReadShortPacketWithKBit1SetShouldFail) { // Simulate short received packet. constexpr uint8_t kPacketData[] = { kFlexible, kPtRecovery, kLengthRecovery[0], kLengthRecovery[1], @@ -698,8 +687,8 @@ TEST(FlexfecHeaderReaderTest, ReadShortPacketMultipleStreamsShouldFail) { EXPECT_FALSE(reader.ReadFecHeader(&read_packet)); } -TEST(FlexfecHeaderWriterTest, FinalizesHeaderWithKBit0SetSingleStream) { - constexpr uint8_t kFlexfecPacketMask[] = {0x88, 0x81}; +TEST(FlexfecHeaderWriterTest, FinalizesHeaderWithKBit0ClearSingleStream) { + constexpr uint8_t kFlexfecPacketMask[] = {0x08, 0x81}; constexpr uint8_t kUlpfecPacketMask[] = {0x11, 0x02}; constexpr uint16_t kMediaStartSeqNum = 1234; Packet written_packet = WritePacket({{.ssrc = 0x01, @@ -714,8 +703,8 @@ TEST(FlexfecHeaderWriterTest, FinalizesHeaderWithKBit0SetSingleStream) { VerifyFinalizedHeaders(written_packet, expected); } -TEST(FlexfecHeaderWriterTest, FinalizesHeaderWithKBit1SetSingleStream) { - constexpr uint8_t kFlexfecPacketMask[] = {0x48, 0x81, 0x82, 0x11, 0x00, 0x21}; +TEST(FlexfecHeaderWriterTest, FinalizesHeaderWithKBit1ClearSingleStream) { + constexpr uint8_t kFlexfecPacketMask[] = {0xC8, 0x81, 0x02, 0x11, 0x00, 0x21}; constexpr uint8_t kUlpfecPacketMask[] = {0x91, 0x02, 0x08, 0x44, 0x00, 0x84}; constexpr uint16_t kMediaStartSeqNum = 1234; Packet written_packet = WritePacket({{.ssrc = 0x01, @@ -730,10 +719,10 @@ TEST(FlexfecHeaderWriterTest, FinalizesHeaderWithKBit1SetSingleStream) { VerifyFinalizedHeaders(written_packet, expected); } -TEST(FlexfecHeaderWriterTest, FinalizesHeaderWithNoKBitsSetSingleStream) { +TEST(FlexfecHeaderWriterTest, FinalizesHeaderWithBothKBitsSetSingleStream) { constexpr uint8_t kFlexfecPacketMask[] = { - 0x11, 0x11, // K-bit 0 clear. - 0x11, 0x11, 0x11, 0x10, // K-bit 1 clear. + 0x91, 0x11, // K-bit 0 set. + 0x91, 0x11, 0x11, 0x10, // K-bit 1 set. 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // }; constexpr uint8_t kUlpfecPacketMask[] = {0x22, 0x22, 0x44, 0x44, 0x44, 0x41}; @@ -752,22 +741,22 @@ TEST(FlexfecHeaderWriterTest, FinalizesHeaderWithNoKBitsSetSingleStream) { TEST(FlexfecHeaderWriterTest, FinalizesHeaderMultipleStreamsMultipleMasks) { constexpr uint8_t kFlexfecPacketMask1[] = { - 0x11, 0x11, // K-bit 0 clear. - 0x11, 0x11, 0x11, 0x10, // K-bit 1 clear. + 0x91, 0x11, // K-bit 0 set. + 0x91, 0x11, 0x11, 0x10, // K-bit 1 set. 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // }; constexpr uint8_t kUlpfecPacketMask1[] = {0x22, 0x22, 0x44, 0x44, 0x44, 0x41}; constexpr uint16_t kMediaStartSeqNum1 = 1234; - constexpr uint8_t kFlexfecPacketMask2[] = {0x88, 0x81}; + constexpr uint8_t kFlexfecPacketMask2[] = {0x08, 0x81}; constexpr uint8_t kUlpfecPacketMask2[] = {0x11, 0x02}; constexpr uint16_t kMediaStartSeqNum2 = 2345; - constexpr uint8_t kFlexfecPacketMask3[] = {0x48, 0x81, 0x82, + constexpr uint8_t kFlexfecPacketMask3[] = {0xC8, 0x81, 0x02, 0x11, 0x00, 0x21}; constexpr uint8_t kUlpfecPacketMask3[] = {0x91, 0x02, 0x08, 0x44, 0x00, 0x84}; constexpr uint16_t kMediaStartSeqNum3 = 3456; constexpr uint8_t kFlexfecPacketMask4[] = { - 0x55, 0xAA, // K-bit 0 clear. - 0x22, 0xAB, 0xCD, 0xEF, // K-bit 1 clear. + 0xD5, 0xAA, // K-bit 0 set. + 0xA2, 0xAB, 0xCD, 0xEF, // K-bit 1 set. 0xC0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 // }; constexpr uint8_t kUlpfecPacketMask4[] = {0xAB, 0x54, 0x8A, 0xAF, 0x37, 0xBF}; diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format.cc index 2c11a29bfa..c7534dee40 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_format.cc @@ -22,6 +22,9 @@ #include "modules/video_coding/codecs/vp8/include/vp8_globals.h" #include "modules/video_coding/codecs/vp9/include/vp9_globals.h" #include "rtc_base/checks.h" +#ifdef RTC_ENABLE_H265 +#include "modules/rtp_rtcp/source/rtp_packetizer_h265.h" +#endif namespace webrtc { @@ -57,7 +60,11 @@ std::unique_ptr<RtpPacketizer> RtpPacketizer::Create( return std::make_unique<RtpPacketizerAv1>( payload, limits, rtp_video_header.frame_type, rtp_video_header.is_last_frame_in_picture); - // TODO(bugs.webrtc.org/13485): Implement RtpPacketizerH265. +#ifdef RTC_ENABLE_H265 + case kVideoCodecH265: { + return std::make_unique<RtpPacketizerH265>(payload, limits); + } +#endif default: { return std::make_unique<RtpPacketizerGeneric>(payload, limits, rtp_video_header); diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_av1.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_av1.cc index 95dbaf364c..859b529a47 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_av1.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_av1.cc @@ -74,8 +74,7 @@ RtpPacketizerAv1::RtpPacketizerAv1(rtc::ArrayView<const uint8_t> payload, std::vector<RtpPacketizerAv1::Obu> RtpPacketizerAv1::ParseObus( rtc::ArrayView<const uint8_t> payload) { std::vector<Obu> result; - rtc::ByteBufferReader payload_reader( - reinterpret_cast<const char*>(payload.data()), payload.size()); + rtc::ByteBufferReader payload_reader(payload); while (payload_reader.Length() > 0) { Obu obu; payload_reader.ReadUInt8(&obu.header); diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_h265.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_h265.cc new file mode 100644 index 0000000000..313680cc87 --- /dev/null +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_h265.cc @@ -0,0 +1,350 @@ +/* + * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/rtp_packetizer_h265.h" + +#include <vector> + +#include "absl/types/optional.h" +#include "common_video/h264/h264_common.h" +#include "common_video/h265/h265_common.h" +#include "modules/rtp_rtcp/source/byte_io.h" +#include "rtc_base/logging.h" + +namespace webrtc { +namespace { + +// The payload header consists of the same +// fields (F, Type, LayerId and TID) as the NAL unit header. Refer to +// section 4.2 in RFC 7798. +constexpr size_t kH265PayloadHeaderSize = 2; +// Unlike H.264, H265 NAL header is 2-bytes. +constexpr size_t kH265NalHeaderSize = 2; +// H265's FU is constructed of 2-byte payload header, 1-byte FU header and FU +// payload. +constexpr size_t kH265FuHeaderSize = 1; +// The NALU size for H265 RTP aggregated packet indicates the size of the NAL +// unit is 2-bytes. +constexpr size_t kH265LengthFieldSize = 2; + +enum H265NalHdrMasks { + kH265FBit = 0x80, + kH265TypeMask = 0x7E, + kH265LayerIDHMask = 0x1, + kH265LayerIDLMask = 0xF8, + kH265TIDMask = 0x7, + kH265TypeMaskN = 0x81, + kH265TypeMaskInFuHeader = 0x3F +}; + +// Bit masks for FU headers. +enum H265FuBitmasks { + kH265SBitMask = 0x80, + kH265EBitMask = 0x40, + kH265FuTypeBitMask = 0x3F +}; + +} // namespace + +RtpPacketizerH265::RtpPacketizerH265(rtc::ArrayView<const uint8_t> payload, + PayloadSizeLimits limits) + : limits_(limits), num_packets_left_(0) { + for (const auto& nalu : + H264::FindNaluIndices(payload.data(), payload.size())) { + input_fragments_.push_back( + payload.subview(nalu.payload_start_offset, nalu.payload_size)); + } + + if (!GeneratePackets()) { + // If failed to generate all the packets, discard already generated + // packets in case the caller would ignore return value and still try to + // call NextPacket(). + num_packets_left_ = 0; + while (!packets_.empty()) { + packets_.pop(); + } + } +} + +RtpPacketizerH265::~RtpPacketizerH265() = default; + +size_t RtpPacketizerH265::NumPackets() const { + return num_packets_left_; +} + +bool RtpPacketizerH265::GeneratePackets() { + for (size_t i = 0; i < input_fragments_.size();) { + int fragment_len = input_fragments_[i].size(); + int single_packet_capacity = limits_.max_payload_len; + if (input_fragments_.size() == 1) { + single_packet_capacity -= limits_.single_packet_reduction_len; + } else if (i == 0) { + single_packet_capacity -= limits_.first_packet_reduction_len; + } else if (i + 1 == input_fragments_.size()) { + // Pretend that last fragment is larger instead of making last packet + // smaller. + single_packet_capacity -= limits_.last_packet_reduction_len; + } + if (fragment_len > single_packet_capacity) { + if (!PacketizeFu(i)) { + return false; + } + ++i; + } else { + i = PacketizeAp(i); + } + } + return true; +} + +bool RtpPacketizerH265::PacketizeFu(size_t fragment_index) { + // Fragment payload into packets (FU). + // Strip out the original header and leave room for the FU header. + rtc::ArrayView<const uint8_t> fragment = input_fragments_[fragment_index]; + PayloadSizeLimits limits = limits_; + // Refer to section 4.4.3 in RFC7798, each FU fragment will have a 2-bytes + // payload header and a one-byte FU header. DONL is not supported so ignore + // its size when calculating max_payload_len. + limits.max_payload_len -= kH265FuHeaderSize + kH265PayloadHeaderSize; + + // Update single/first/last packet reductions unless it is single/first/last + // fragment. + if (input_fragments_.size() != 1) { + // if this fragment is put into a single packet, it might still be the + // first or the last packet in the whole sequence of packets. + if (fragment_index == input_fragments_.size() - 1) { + limits.single_packet_reduction_len = limits_.last_packet_reduction_len; + } else if (fragment_index == 0) { + limits.single_packet_reduction_len = limits_.first_packet_reduction_len; + } else { + limits.single_packet_reduction_len = 0; + } + } + if (fragment_index != 0) { + limits.first_packet_reduction_len = 0; + } + if (fragment_index != input_fragments_.size() - 1) { + limits.last_packet_reduction_len = 0; + } + + // Strip out the original header. + size_t payload_left = fragment.size() - kH265NalHeaderSize; + int offset = kH265NalHeaderSize; + + std::vector<int> payload_sizes = SplitAboutEqually(payload_left, limits); + if (payload_sizes.empty()) { + return false; + } + + for (size_t i = 0; i < payload_sizes.size(); ++i) { + int packet_length = payload_sizes[i]; + RTC_CHECK_GT(packet_length, 0); + uint16_t header = (fragment[0] << 8) | fragment[1]; + packets_.push({.source_fragment = fragment.subview(offset, packet_length), + .first_fragment = (i == 0), + .last_fragment = (i == payload_sizes.size() - 1), + .aggregated = false, + .header = header}); + offset += packet_length; + payload_left -= packet_length; + } + num_packets_left_ += payload_sizes.size(); + RTC_CHECK_EQ(payload_left, 0); + return true; +} + +int RtpPacketizerH265::PacketizeAp(size_t fragment_index) { + // Aggregate fragments into one packet. + size_t payload_size_left = limits_.max_payload_len; + if (input_fragments_.size() == 1) { + payload_size_left -= limits_.single_packet_reduction_len; + } else if (fragment_index == 0) { + payload_size_left -= limits_.first_packet_reduction_len; + } + int aggregated_fragments = 0; + size_t fragment_headers_length = 0; + rtc::ArrayView<const uint8_t> fragment = input_fragments_[fragment_index]; + RTC_CHECK_GE(payload_size_left, fragment.size()); + ++num_packets_left_; + + auto payload_size_needed = [&] { + size_t fragment_size = fragment.size() + fragment_headers_length; + if (input_fragments_.size() == 1) { + // Single fragment, single packet, payload_size_left already adjusted + // with limits_.single_packet_reduction_len. + return fragment_size; + } + if (fragment_index == input_fragments_.size() - 1) { + // Last fragment, so this might be the last packet. + return fragment_size + limits_.last_packet_reduction_len; + } + return fragment_size; + }; + + while (payload_size_left >= payload_size_needed()) { + RTC_CHECK_GT(fragment.size(), 0); + packets_.push({.source_fragment = fragment, + .first_fragment = (aggregated_fragments == 0), + .last_fragment = false, + .aggregated = true, + .header = fragment[0]}); + payload_size_left -= fragment.size(); + payload_size_left -= fragment_headers_length; + + fragment_headers_length = kH265LengthFieldSize; + // If we are going to try to aggregate more fragments into this packet + // we need to add the AP NALU header and a length field for the first + // NALU of this packet. + if (aggregated_fragments == 0) { + fragment_headers_length += kH265PayloadHeaderSize + kH265LengthFieldSize; + } + ++aggregated_fragments; + + // Next fragment. + ++fragment_index; + if (fragment_index == input_fragments_.size()) { + break; + } + fragment = input_fragments_[fragment_index]; + } + RTC_CHECK_GT(aggregated_fragments, 0); + packets_.back().last_fragment = true; + return fragment_index; +} + +bool RtpPacketizerH265::NextPacket(RtpPacketToSend* rtp_packet) { + RTC_DCHECK(rtp_packet); + + if (packets_.empty()) { + return false; + } + + PacketUnit packet = packets_.front(); + + if (packet.first_fragment && packet.last_fragment) { + // Single NAL unit packet. Do not support DONL for single NAL unit packets, + // DONL field is not present. + size_t bytes_to_send = packet.source_fragment.size(); + uint8_t* buffer = rtp_packet->AllocatePayload(bytes_to_send); + memcpy(buffer, packet.source_fragment.data(), bytes_to_send); + packets_.pop(); + input_fragments_.pop_front(); + } else if (packet.aggregated) { + NextAggregatePacket(rtp_packet); + } else { + NextFragmentPacket(rtp_packet); + } + rtp_packet->SetMarker(packets_.empty()); + --num_packets_left_; + return true; +} + +void RtpPacketizerH265::NextAggregatePacket(RtpPacketToSend* rtp_packet) { + size_t payload_capacity = rtp_packet->FreeCapacity(); + RTC_CHECK_GE(payload_capacity, kH265PayloadHeaderSize); + uint8_t* buffer = rtp_packet->AllocatePayload(payload_capacity); + RTC_CHECK(buffer); + PacketUnit* packet = &packets_.front(); + RTC_CHECK(packet->first_fragment); + + /* + +---------------+---------------+ + |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7| + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |F| Type | LayerId | TID | + +-------------+-----------------+ + */ + // Refer to section section 4.4.2 for aggregation packets and modify type to + // 48 in PayloadHdr for aggregate packet. Do not support DONL for aggregation + // packets, DONL field is not present. + uint8_t payload_hdr_h = packet->header >> 8; + uint8_t payload_hdr_l = packet->header & 0xFF; + uint8_t layer_id_h = payload_hdr_h & kH265LayerIDHMask; + payload_hdr_h = (payload_hdr_h & kH265TypeMaskN) | + (H265::NaluType::kAp << 1) | layer_id_h; + buffer[0] = payload_hdr_h; + buffer[1] = payload_hdr_l; + + int index = kH265PayloadHeaderSize; + bool is_last_fragment = packet->last_fragment; + while (packet->aggregated) { + // Add NAL unit length field. + rtc::ArrayView<const uint8_t> fragment = packet->source_fragment; + ByteWriter<uint16_t>::WriteBigEndian(&buffer[index], fragment.size()); + index += kH265LengthFieldSize; + // Add NAL unit. + memcpy(&buffer[index], fragment.data(), fragment.size()); + index += fragment.size(); + packets_.pop(); + input_fragments_.pop_front(); + if (is_last_fragment) { + break; + } + packet = &packets_.front(); + is_last_fragment = packet->last_fragment; + } + RTC_CHECK(is_last_fragment); + rtp_packet->SetPayloadSize(index); +} + +void RtpPacketizerH265::NextFragmentPacket(RtpPacketToSend* rtp_packet) { + PacketUnit* packet = &packets_.front(); + // NAL unit fragmented over multiple packets (FU). + // We do not send original NALU header, so it will be replaced by the + // PayloadHdr of the first packet. + /* + +---------------+---------------+ + |0|1|2|3|4|5|6|7|0|1|2|3|4|5|6|7| + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |F| Type | LayerId | TID | + +-------------+-----------------+ + */ + // Refer to section section 4.4.3 for aggregation packets and modify type to + // 49 in PayloadHdr for aggregate packet. + uint8_t payload_hdr_h = + packet->header >> 8; // 1-bit F, 6-bit type, 1-bit layerID highest-bit + uint8_t payload_hdr_l = packet->header & 0xFF; + uint8_t layer_id_h = payload_hdr_h & kH265LayerIDHMask; + uint8_t fu_header = 0; + /* + +---------------+ + |0|1|2|3|4|5|6|7| + +-+-+-+-+-+-+-+-+ + |S|E| FuType | + +---------------+ + */ + // S bit indicates the start of a fragmented NAL unit. + // E bit indicates the end of a fragmented NAL unit. + // FuType must be equal to the field type value of the fragmented NAL unit. + fu_header |= (packet->first_fragment ? kH265SBitMask : 0); + fu_header |= (packet->last_fragment ? kH265EBitMask : 0); + uint8_t type = (payload_hdr_h & kH265TypeMask) >> 1; + fu_header |= type; + // Now update payload_hdr_h with FU type. + payload_hdr_h = (payload_hdr_h & kH265TypeMaskN) | + (H265::NaluType::kFu << 1) | layer_id_h; + rtc::ArrayView<const uint8_t> fragment = packet->source_fragment; + uint8_t* buffer = rtp_packet->AllocatePayload( + kH265FuHeaderSize + kH265PayloadHeaderSize + fragment.size()); + RTC_CHECK(buffer); + buffer[0] = payload_hdr_h; + buffer[1] = payload_hdr_l; + buffer[2] = fu_header; + + // Do not support DONL for fragmentation units, DONL field is not present. + memcpy(buffer + kH265FuHeaderSize + kH265PayloadHeaderSize, fragment.data(), + fragment.size()); + if (packet->last_fragment) { + input_fragments_.pop_front(); + } + packets_.pop(); +} + +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_h265.h b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_h265.h new file mode 100644 index 0000000000..95442f795c --- /dev/null +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_h265.h @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef MODULES_RTP_RTCP_SOURCE_RTP_PACKETIZER_H265_H_ +#define MODULES_RTP_RTCP_SOURCE_RTP_PACKETIZER_H265_H_ + +#include <deque> +#include <queue> +#include <string> + +#include "api/array_view.h" +#include "modules/rtp_rtcp/source/rtp_format.h" +#include "modules/rtp_rtcp/source/rtp_packet_to_send.h" + +namespace webrtc { + +class RtpPacketizerH265 : public RtpPacketizer { + public: + // Initialize with payload from encoder. + // The payload_data must be exactly one encoded H.265 frame. + // For H265 we only support tx-mode SRST. + RtpPacketizerH265(rtc::ArrayView<const uint8_t> payload, + PayloadSizeLimits limits); + + RtpPacketizerH265(const RtpPacketizerH265&) = delete; + RtpPacketizerH265& operator=(const RtpPacketizerH265&) = delete; + + ~RtpPacketizerH265() override; + + size_t NumPackets() const override; + + // Get the next payload with H.265 payload header. + // Write payload and set marker bit of the `packet`. + // Returns true on success or false if there was no payload to packetize. + bool NextPacket(RtpPacketToSend* rtp_packet) override; + + private: + struct PacketUnit { + rtc::ArrayView<const uint8_t> source_fragment; + bool first_fragment = false; + bool last_fragment = false; + bool aggregated = false; + uint16_t header = 0; + }; + std::deque<rtc::ArrayView<const uint8_t>> input_fragments_; + std::queue<PacketUnit> packets_; + + bool GeneratePackets(); + bool PacketizeFu(size_t fragment_index); + int PacketizeAp(size_t fragment_index); + + void NextAggregatePacket(RtpPacketToSend* rtp_packet); + void NextFragmentPacket(RtpPacketToSend* rtp_packet); + + const PayloadSizeLimits limits_; + size_t num_packets_left_ = 0; +}; +} // namespace webrtc +#endif // MODULES_RTP_RTCP_SOURCE_RTP_PACKETIZER_H265_H_ diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_h265_unittest.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_h265_unittest.cc new file mode 100644 index 0000000000..cb1de334c0 --- /dev/null +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_packetizer_h265_unittest.cc @@ -0,0 +1,525 @@ +/* + * Copyright (c) 2023 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "modules/rtp_rtcp/source/rtp_packetizer_h265.h" + +#include <vector> + +#include "common_video/h265/h265_common.h" +#include "modules/rtp_rtcp/mocks/mock_rtp_rtcp.h" +#include "modules/rtp_rtcp/source/byte_io.h" +#include "test/gmock.h" +#include "test/gtest.h" + +namespace webrtc { +namespace { + +using ::testing::Each; +using ::testing::ElementsAre; +using ::testing::ElementsAreArray; +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::SizeIs; + +constexpr RtpPacketToSend::ExtensionManager* kNoExtensions = nullptr; +constexpr size_t kMaxPayloadSize = 1200; +constexpr size_t kLengthFieldLength = 2; +constexpr RtpPacketizer::PayloadSizeLimits kNoLimits; + +constexpr size_t kNalHeaderSize = 2; +constexpr size_t kFuHeaderSize = 3; + +constexpr uint8_t kNaluTypeMask = 0x7E; + +// Bit masks for FU headers. +constexpr uint8_t kH265SBit = 0x80; +constexpr uint8_t kH265EBit = 0x40; + +// Creates Buffer that looks like nal unit of given size. +rtc::Buffer GenerateNalUnit(size_t size) { + RTC_CHECK_GT(size, 0); + rtc::Buffer buffer(size); + // Set some valid header with type TRAIL_R and temporal id + buffer[0] = 2; + buffer[1] = 2; + for (size_t i = 2; i < size; ++i) { + buffer[i] = static_cast<uint8_t>(i); + } + // Last byte shouldn't be 0, or it may be counted as part of next 4-byte start + // sequence. + buffer[size - 1] |= 0x10; + return buffer; +} + +// Create frame consisting of nalus of given size. +rtc::Buffer CreateFrame(std::initializer_list<size_t> nalu_sizes) { + static constexpr int kStartCodeSize = 3; + rtc::Buffer frame(absl::c_accumulate(nalu_sizes, size_t{0}) + + kStartCodeSize * nalu_sizes.size()); + size_t offset = 0; + for (size_t nalu_size : nalu_sizes) { + EXPECT_GE(nalu_size, 1u); + // Insert nalu start code + frame[offset] = 0; + frame[offset + 1] = 0; + frame[offset + 2] = 1; + // Set some valid header. + frame[offset + 3] = 2; + // Fill payload avoiding accidental start codes + if (nalu_size > 1) { + memset(frame.data() + offset + 4, 0x3f, nalu_size - 1); + } + offset += (kStartCodeSize + nalu_size); + } + return frame; +} + +// Create frame consisting of given nalus. +rtc::Buffer CreateFrame(rtc::ArrayView<const rtc::Buffer> nalus) { + static constexpr int kStartCodeSize = 3; + int frame_size = 0; + for (const rtc::Buffer& nalu : nalus) { + frame_size += (kStartCodeSize + nalu.size()); + } + rtc::Buffer frame(frame_size); + size_t offset = 0; + for (const rtc::Buffer& nalu : nalus) { + // Insert nalu start code + frame[offset] = 0; + frame[offset + 1] = 0; + frame[offset + 2] = 1; + // Copy the nalu unit. + memcpy(frame.data() + offset + 3, nalu.data(), nalu.size()); + offset += (kStartCodeSize + nalu.size()); + } + return frame; +} + +std::vector<RtpPacketToSend> FetchAllPackets(RtpPacketizerH265* packetizer) { + std::vector<RtpPacketToSend> result; + size_t num_packets = packetizer->NumPackets(); + result.reserve(num_packets); + RtpPacketToSend packet(kNoExtensions); + while (packetizer->NextPacket(&packet)) { + result.push_back(packet); + } + EXPECT_THAT(result, SizeIs(num_packets)); + return result; +} + +// Single nalu tests. +TEST(RtpPacketizerH265Test, SingleNalu) { + const uint8_t frame[] = {0, 0, 1, H265::kIdrWRadl, 0xFF}; + + RtpPacketizerH265 packetizer(frame, kNoLimits); + std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer); + + ASSERT_THAT(packets, SizeIs(1)); + EXPECT_THAT(packets[0].payload(), ElementsAre(H265::kIdrWRadl, 0xFF)); +} + +TEST(RtpPacketizerH265Test, SingleNaluTwoPackets) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = kMaxPayloadSize; + rtc::Buffer nalus[] = {GenerateNalUnit(kMaxPayloadSize), + GenerateNalUnit(100)}; + rtc::Buffer frame = CreateFrame(nalus); + + RtpPacketizerH265 packetizer(frame, limits); + std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer); + + ASSERT_THAT(packets, SizeIs(2)); + EXPECT_THAT(packets[0].payload(), ElementsAreArray(nalus[0])); + EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[1])); +} + +TEST(RtpPacketizerH265Test, + SingleNaluFirstPacketReductionAppliesOnlyToFirstFragment) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 200; + limits.first_packet_reduction_len = 5; + rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/195), + GenerateNalUnit(/*size=*/200), + GenerateNalUnit(/*size=*/200)}; + rtc::Buffer frame = CreateFrame(nalus); + + RtpPacketizerH265 packetizer(frame, limits); + std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer); + + ASSERT_THAT(packets, SizeIs(3)); + EXPECT_THAT(packets[0].payload(), ElementsAreArray(nalus[0])); + EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[1])); + EXPECT_THAT(packets[2].payload(), ElementsAreArray(nalus[2])); +} + +TEST(RtpPacketizerH265Test, + SingleNaluLastPacketReductionAppliesOnlyToLastFragment) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 200; + limits.last_packet_reduction_len = 5; + rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/200), + GenerateNalUnit(/*size=*/200), + GenerateNalUnit(/*size=*/195)}; + rtc::Buffer frame = CreateFrame(nalus); + + RtpPacketizerH265 packetizer(frame, limits); + std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer); + + ASSERT_THAT(packets, SizeIs(3)); + EXPECT_THAT(packets[0].payload(), ElementsAreArray(nalus[0])); + EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[1])); + EXPECT_THAT(packets[2].payload(), ElementsAreArray(nalus[2])); +} + +TEST(RtpPacketizerH265Test, + SingleNaluFirstAndLastPacketReductionSumsForSinglePacket) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 200; + limits.first_packet_reduction_len = 20; + limits.last_packet_reduction_len = 30; + rtc::Buffer frame = CreateFrame({150}); + + RtpPacketizerH265 packetizer(frame, limits); + std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer); + + EXPECT_THAT(packets, SizeIs(1)); +} + +// Aggregation tests. +TEST(RtpPacketizerH265Test, ApRespectsNoPacketReduction) { + rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2), + GenerateNalUnit(/*size=*/2), + GenerateNalUnit(/*size=*/0x123)}; + rtc::Buffer frame = CreateFrame(nalus); + + RtpPacketizerH265 packetizer(frame, kNoLimits); + std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer); + + ASSERT_THAT(packets, SizeIs(1)); + auto payload = packets[0].payload(); + int type = H265::ParseNaluType(payload[0]); + EXPECT_EQ(payload.size(), + kNalHeaderSize + 3 * kLengthFieldLength + 2 + 2 + 0x123); + + EXPECT_EQ(type, H265::NaluType::kAp); + payload = payload.subview(kNalHeaderSize); + // 1st fragment. + EXPECT_THAT(payload.subview(0, kLengthFieldLength), + ElementsAre(0, 2)); // Size. + EXPECT_THAT(payload.subview(kLengthFieldLength, 2), + ElementsAreArray(nalus[0])); + payload = payload.subview(kLengthFieldLength + 2); + // 2nd fragment. + EXPECT_THAT(payload.subview(0, kLengthFieldLength), + ElementsAre(0, 2)); // Size. + EXPECT_THAT(payload.subview(kLengthFieldLength, 2), + ElementsAreArray(nalus[1])); + payload = payload.subview(kLengthFieldLength + 2); + // 3rd fragment. + EXPECT_THAT(payload.subview(0, kLengthFieldLength), + ElementsAre(0x1, 0x23)); // Size. + EXPECT_THAT(payload.subview(kLengthFieldLength), ElementsAreArray(nalus[2])); +} + +TEST(RtpPacketizerH265Test, ApRespectsFirstPacketReduction) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 1000; + limits.first_packet_reduction_len = 100; + const size_t kFirstFragmentSize = + limits.max_payload_len - limits.first_packet_reduction_len; + rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/kFirstFragmentSize), + GenerateNalUnit(/*size=*/2), + GenerateNalUnit(/*size=*/2)}; + rtc::Buffer frame = CreateFrame(nalus); + + RtpPacketizerH265 packetizer(frame, limits); + std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer); + + ASSERT_THAT(packets, SizeIs(2)); + // Expect 1st packet is single nalu. + EXPECT_THAT(packets[0].payload(), ElementsAreArray(nalus[0])); + // Expect 2nd packet is aggregate of last two fragments. + // The size of H265 nal_unit_header is 2 bytes, according to 7.3.1.2 + // in H265 spec. Aggregation packet type is 48, and nuh_temporal_id_plus1 + // is 2, so the nal_unit_header should be "01100000 00000010", + // which is 96 and 2. + EXPECT_THAT(packets[1].payload(), + ElementsAre(96, 2, // + 0, 2, nalus[1][0], nalus[1][1], // + 0, 2, nalus[2][0], nalus[2][1])); +} + +TEST(RtpPacketizerH265Test, ApRespectsLastPacketReduction) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 1000; + limits.last_packet_reduction_len = 100; + const size_t kLastFragmentSize = + limits.max_payload_len - limits.last_packet_reduction_len; + rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2), + GenerateNalUnit(/*size=*/2), + GenerateNalUnit(/*size=*/kLastFragmentSize)}; + rtc::Buffer frame = CreateFrame(nalus); + + RtpPacketizerH265 packetizer(frame, limits); + std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer); + + ASSERT_THAT(packets, SizeIs(2)); + // Expect 1st packet is aggregate of 1st two fragments. + EXPECT_THAT(packets[0].payload(), + ElementsAre(96, 2, // + 0, 2, nalus[0][0], nalus[0][1], // + 0, 2, nalus[1][0], nalus[1][1])); + // Expect 2nd packet is single nalu. + EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[2])); +} + +TEST(RtpPacketizerH265Test, TooSmallForApHeaders) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 1000; + const size_t kLastFragmentSize = + limits.max_payload_len - 3 * kLengthFieldLength - 4; + rtc::Buffer nalus[] = {GenerateNalUnit(/*size=*/2), + GenerateNalUnit(/*size=*/2), + GenerateNalUnit(/*size=*/kLastFragmentSize)}; + rtc::Buffer frame = CreateFrame(nalus); + + RtpPacketizerH265 packetizer(frame, limits); + std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer); + + ASSERT_THAT(packets, SizeIs(2)); + // Expect 1st packet is aggregate of 1st two fragments. + EXPECT_THAT(packets[0].payload(), + ElementsAre(96, 2, // + 0, 2, nalus[0][0], nalus[0][1], // + 0, 2, nalus[1][0], nalus[1][1])); + // Expect 2nd packet is single nalu. + EXPECT_THAT(packets[1].payload(), ElementsAreArray(nalus[2])); +} + +TEST(RtpPacketizerH265Test, LastFragmentFitsInSingleButNotLastPacket) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 1178; + limits.first_packet_reduction_len = 0; + limits.last_packet_reduction_len = 20; + limits.single_packet_reduction_len = 20; + // Actual sizes, which triggered this bug. + rtc::Buffer frame = CreateFrame({20, 8, 18, 1161}); + + RtpPacketizerH265 packetizer(frame, limits); + std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer); + + // Last packet has to be of correct size. + // Incorrect implementation might miss this constraint and not split the last + // fragment in two packets. + EXPECT_LE(static_cast<int>(packets.back().payload_size()), + limits.max_payload_len - limits.last_packet_reduction_len); +} + +// Splits frame with payload size `frame_payload_size` without fragmentation, +// Returns sizes of the payloads excluding FU headers. +std::vector<int> TestFu(size_t frame_payload_size, + const RtpPacketizer::PayloadSizeLimits& limits) { + rtc::Buffer nalu[] = {GenerateNalUnit(kNalHeaderSize + frame_payload_size)}; + rtc::Buffer frame = CreateFrame(nalu); + + RtpPacketizerH265 packetizer(frame, limits); + std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer); + + EXPECT_GE(packets.size(), 2u); // Single packet indicates it is not FU. + std::vector<uint16_t> fu_header; + std::vector<int> payload_sizes; + + for (const RtpPacketToSend& packet : packets) { + auto payload = packet.payload(); + EXPECT_GT(payload.size(), kFuHeaderSize); + // FU header is after the 2-bytes size PayloadHdr according to 4.4.3 in spec + fu_header.push_back(payload[2]); + payload_sizes.push_back(payload.size() - kFuHeaderSize); + } + + EXPECT_TRUE(fu_header.front() & kH265SBit); + EXPECT_TRUE(fu_header.back() & kH265EBit); + // Clear S and E bits before testing all are duplicating same original header. + fu_header.front() &= ~kH265SBit; + fu_header.back() &= ~kH265EBit; + uint8_t nalu_type = (nalu[0][0] & kNaluTypeMask) >> 1; + EXPECT_THAT(fu_header, Each(Eq(nalu_type))); + + return payload_sizes; +} + +// Fragmentation tests. +TEST(RtpPacketizerH265Test, FuOddSize) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 1200; + EXPECT_THAT(TestFu(1200, limits), ElementsAre(600, 600)); +} + +TEST(RtpPacketizerH265Test, FuWithFirstPacketReduction) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 1200; + limits.first_packet_reduction_len = 4; + limits.single_packet_reduction_len = 4; + EXPECT_THAT(TestFu(1198, limits), ElementsAre(597, 601)); +} + +TEST(RtpPacketizerH265Test, FuWithLastPacketReduction) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 1200; + limits.last_packet_reduction_len = 4; + limits.single_packet_reduction_len = 4; + EXPECT_THAT(TestFu(1198, limits), ElementsAre(601, 597)); +} + +TEST(RtpPacketizerH265Test, FuWithSinglePacketReduction) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 1199; + limits.single_packet_reduction_len = 200; + EXPECT_THAT(TestFu(1000, limits), ElementsAre(500, 500)); +} + +TEST(RtpPacketizerH265Test, FuEvenSize) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 1200; + EXPECT_THAT(TestFu(1201, limits), ElementsAre(600, 601)); +} + +TEST(RtpPacketizerH265Test, FuRounding) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 1448; + EXPECT_THAT(TestFu(10123, limits), + ElementsAre(1265, 1265, 1265, 1265, 1265, 1266, 1266, 1266)); +} + +TEST(RtpPacketizerH265Test, FuBig) { + RtpPacketizer::PayloadSizeLimits limits; + limits.max_payload_len = 1200; + // Generate 10 full sized packets, leave room for FU headers. + EXPECT_THAT( + TestFu(10 * (1200 - kFuHeaderSize), limits), + ElementsAre(1197, 1197, 1197, 1197, 1197, 1197, 1197, 1197, 1197, 1197)); +} + +struct PacketInfo { + bool first_fragment = false; + bool last_fragment = false; + bool aggregated = false; + int nalu_index = 0; + int nalu_number = 0; + int payload_size = 0; + int start_offset = 0; +}; + +struct MixedApFuTestParams { + std::vector<int> nalus; + int expect_packetsSize = 0; + std::vector<PacketInfo> expected_packets; +}; + +class RtpPacketizerH265ParametrizedTest + : public ::testing::TestWithParam<MixedApFuTestParams> {}; + +// Fragmentation + aggregation mixed testing. +TEST_P(RtpPacketizerH265ParametrizedTest, MixedApFu) { + RtpPacketizer::PayloadSizeLimits limits; + const MixedApFuTestParams params = GetParam(); + limits.max_payload_len = 100; + std::vector<rtc::Buffer> nalus; + nalus.reserve(params.nalus.size()); + + // Generate nalus according to size specified in paramters + for (size_t index = 0; index < params.nalus.size(); index++) { + nalus.push_back(GenerateNalUnit(params.nalus[index])); + } + rtc::Buffer frame = CreateFrame(nalus); + + RtpPacketizerH265 packetizer(frame, limits); + std::vector<RtpPacketToSend> packets = FetchAllPackets(&packetizer); + + ASSERT_THAT(packets, SizeIs(params.expect_packetsSize)); + for (int i = 0; i < params.expect_packetsSize; i++) { + PacketInfo expected_packet = params.expected_packets[i]; + if (expected_packet.aggregated) { + int type = H265::ParseNaluType(packets[i].payload()[0]); + EXPECT_THAT(type, H265::NaluType::kAp); + auto payload = packets[i].payload().subview(kNalHeaderSize); + int offset = 0; + // Generated AP packet header and payload align + for (int j = expected_packet.nalu_index; j < expected_packet.nalu_number; + j++) { + EXPECT_THAT(payload.subview(0, kLengthFieldLength), + ElementsAre(0, nalus[j].size())); + EXPECT_THAT( + payload.subview(offset + kLengthFieldLength, nalus[j].size()), + ElementsAreArray(nalus[j])); + offset += kLengthFieldLength + nalus[j].size(); + } + } else { + uint8_t fu_header = 0; + fu_header |= (expected_packet.first_fragment ? kH265SBit : 0); + fu_header |= (expected_packet.last_fragment ? kH265EBit : 0); + fu_header |= H265::NaluType::kTrailR; + EXPECT_THAT(packets[i].payload().subview(0, kFuHeaderSize), + ElementsAre(98, 2, fu_header)); + EXPECT_THAT( + packets[i].payload().subview(kFuHeaderSize), + ElementsAreArray(nalus[expected_packet.nalu_index].data() + + kNalHeaderSize + expected_packet.start_offset, + expected_packet.payload_size)); + } + } +} + +INSTANTIATE_TEST_SUITE_P( + RtpPacketizerH265Test, + RtpPacketizerH265ParametrizedTest, + testing::Values( + // FU + AP + FU. + // GenerateNalUnit will include 2 bytes nalu header, for FU packet split + // calculation, this 2-byte nalu header length should be excluded. + MixedApFuTestParams{.nalus = {140, 20, 20, 160}, + .expect_packetsSize = 5, + .expected_packets = {{.first_fragment = true, + .nalu_index = 0, + .payload_size = 69, + .start_offset = 0}, + {.last_fragment = true, + .nalu_index = 0, + .payload_size = 69, + .start_offset = 69}, + {.aggregated = true, + .nalu_index = 1, + .nalu_number = 2}, + {.first_fragment = true, + .nalu_index = 3, + .payload_size = 79, + .start_offset = 0}, + {.last_fragment = true, + .nalu_index = 3, + .payload_size = 79, + .start_offset = 79}}}, + // AP + FU + AP + MixedApFuTestParams{ + .nalus = {20, 20, 160, 30, 30}, + .expect_packetsSize = 4, + .expected_packets = { + {.aggregated = true, .nalu_index = 0, .nalu_number = 2}, + {.first_fragment = true, + .nalu_index = 2, + .payload_size = 79, + .start_offset = 0}, + {.last_fragment = true, + .nalu_index = 2, + .payload_size = 79, + .start_offset = 79}, + {.aggregated = true, .nalu_index = 3, .nalu_number = 2}}})); + +} // namespace +} // namespace webrtc diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc index ff482b39b6..31e8b71117 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_rtcp_impl2.cc @@ -53,8 +53,8 @@ RTCPSender::Configuration AddRtcpSendEvaluationCallback( RtpPacketHistory::PaddingMode GetPaddingMode( const FieldTrialsView* field_trials) { - if (field_trials && - field_trials->IsEnabled("WebRTC-PaddingMode-RecentLargePacket")) { + if (!field_trials || + !field_trials->IsDisabled("WebRTC-PaddingMode-RecentLargePacket")) { return RtpPacketHistory::PaddingMode::kRecentLargePacket; } return RtpPacketHistory::PaddingMode::kPriority; diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_audio.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_audio.cc index b826c30e07..9d2258dc66 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_audio.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_audio.cc @@ -254,7 +254,8 @@ bool RTPSenderAudio::SendAudio(const RtpAudioFrame& frame) { return false; } - std::unique_ptr<RtpPacketToSend> packet = rtp_sender_->AllocatePacket(); + std::unique_ptr<RtpPacketToSend> packet = + rtp_sender_->AllocatePacket(frame.csrcs); packet->SetMarker(MarkerBit(frame.type, frame.payload_id)); packet->SetPayloadType(frame.payload_id); packet->SetTimestamp(frame.rtp_timestamp); diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_audio.h b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_audio.h index 662f908216..83a2cb211f 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_audio.h +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_audio.h @@ -61,6 +61,9 @@ class RTPSenderAudio { // header-extension-for-audio-level-indication. // Valid range is [0,127]. Actual value is negative. absl::optional<int> audio_level_dbov; + + // Contributing sources list. + rtc::ArrayView<const uint32_t> csrcs; }; bool SendAudio(const RtpAudioFrame& frame); diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc index 0db610c149..724cd3a5e0 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_audio_unittest.cc @@ -222,4 +222,19 @@ TEST_F(RtpSenderAudioTest, CheckMarkerBitForTelephoneEvents) { EXPECT_FALSE(transport_.last_sent_packet().Marker()); } +TEST_F(RtpSenderAudioTest, SendsCsrcs) { + const char payload_name[] = "audio"; + const uint8_t payload_type = 127; + ASSERT_EQ(0, rtp_sender_audio_->RegisterAudioPayload( + payload_name, payload_type, 48000, 0, 1500)); + uint8_t payload[] = {47, 11, 32, 93, 89}; + + std::vector<uint32_t> csrcs({123, 456, 789}); + + ASSERT_TRUE(rtp_sender_audio_->SendAudio( + {.payload = payload, .payload_id = payload_type, .csrcs = csrcs})); + + EXPECT_EQ(transport_.last_sent_packet().Csrcs(), csrcs); +} + } // namespace webrtc diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc index 9d7c58d19a..ae9eb6b4bd 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc @@ -19,10 +19,17 @@ #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" #include "rtc_base/checks.h" #include "rtc_base/event.h" +#include "rtc_base/logging.h" namespace webrtc { namespace { +// Using a reasonable default of 10ms for the retransmission delay for frames +// not coming from this sender's encoder. This is usually taken from an +// estimate of the RTT of the link,so 10ms should be a reasonable estimate for +// frames being re-transmitted to a peer, probably on the same network. +const TimeDelta kDefaultRetransmissionsTime = TimeDelta::Millis(10); + class TransformableVideoSenderFrame : public TransformableVideoFrameInterface { public: TransformableVideoSenderFrame(const EncodedImage& encoded_image, @@ -155,6 +162,17 @@ bool RTPSenderVideoFrameTransformerDelegate::TransformFrame( const EncodedImage& encoded_image, RTPVideoHeader video_header, TimeDelta expected_retransmission_time) { + { + MutexLock lock(&sender_lock_); + if (short_circuit_) { + sender_->SendVideo(payload_type, codec_type, rtp_timestamp, + encoded_image.CaptureTime(), + *encoded_image.GetEncodedData(), encoded_image.size(), + video_header, expected_retransmission_time, + /*csrcs=*/{}); + return true; + } + } frame_transformer_->Transform(std::make_unique<TransformableVideoSenderFrame>( encoded_image, video_header, payload_type, codec_type, rtp_timestamp, expected_retransmission_time, ssrc_, @@ -177,6 +195,11 @@ void RTPSenderVideoFrameTransformerDelegate::OnTransformedFrame( }); } +void RTPSenderVideoFrameTransformerDelegate::StartShortCircuiting() { + MutexLock lock(&sender_lock_); + short_circuit_ = true; +} + void RTPSenderVideoFrameTransformerDelegate::SendVideo( std::unique_ptr<TransformableFrameInterface> transformed_frame) const { RTC_DCHECK_RUN_ON(transformation_queue_.get()); @@ -200,15 +223,17 @@ void RTPSenderVideoFrameTransformerDelegate::SendVideo( auto* transformed_video_frame = static_cast<TransformableVideoFrameInterface*>(transformed_frame.get()); VideoFrameMetadata metadata = transformed_video_frame->Metadata(); - sender_->SendVideo( - transformed_video_frame->GetPayloadType(), metadata.GetCodec(), - transformed_video_frame->GetTimestamp(), - /*capture_time=*/Timestamp::MinusInfinity(), - transformed_video_frame->GetData(), - transformed_video_frame->GetData().size(), - RTPVideoHeader::FromMetadata(metadata), - /*expected_retransmission_time=*/TimeDelta::PlusInfinity(), - metadata.GetCsrcs()); + // TODO(bugs.webrtc.org/14708): Use an actual RTT estimate for the + // retransmission time instead of a const default, in the same way as a + // locally encoded frame. + sender_->SendVideo(transformed_video_frame->GetPayloadType(), + metadata.GetCodec(), + transformed_video_frame->GetTimestamp(), + /*capture_time=*/Timestamp::MinusInfinity(), + transformed_video_frame->GetData(), + transformed_video_frame->GetData().size(), + RTPVideoHeader::FromMetadata(metadata), + kDefaultRetransmissionsTime, metadata.GetCsrcs()); } } @@ -253,13 +278,14 @@ std::unique_ptr<TransformableVideoFrameInterface> CloneSenderVideoFrame( ? VideoFrameType::kVideoFrameKey : VideoFrameType::kVideoFrameDelta; // TODO(bugs.webrtc.org/14708): Fill in other EncodedImage parameters - + // TODO(bugs.webrtc.org/14708): Use an actual RTT estimate for the + // retransmission time instead of a const default, in the same way as a + // locally encoded frame. VideoFrameMetadata metadata = original->Metadata(); RTPVideoHeader new_header = RTPVideoHeader::FromMetadata(metadata); return std::make_unique<TransformableVideoSenderFrame>( encoded_image, new_header, original->GetPayloadType(), new_header.codec, - original->GetTimestamp(), - /*expected_retransmission_time=*/TimeDelta::PlusInfinity(), + original->GetTimestamp(), kDefaultRetransmissionsTime, original->GetSsrc(), metadata.GetCsrcs(), original->GetRid()); } diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h index 3379ead364..1f70a23ccc 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.h @@ -76,6 +76,8 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { void OnTransformedFrame( std::unique_ptr<TransformableFrameInterface> frame) override; + void StartShortCircuiting() override; + // Delegates the call to RTPSendVideo::SendVideo on the `encoder_queue_`. void SendVideo(std::unique_ptr<TransformableFrameInterface> frame) const RTC_RUN_ON(transformation_queue_); @@ -109,6 +111,7 @@ class RTPSenderVideoFrameTransformerDelegate : public TransformedFrameCallback { // Used when the encoded frames arrives without a current task queue. This can // happen if a hardware encoder was used. std::unique_ptr<TaskQueueBase, TaskQueueDeleter> transformation_queue_; + bool short_circuit_ RTC_GUARDED_BY(sender_lock_) = false; }; // Method to support cloning a Sender frame from another frame diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc index a376be77b4..6790fc3a71 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate_unittest.cc @@ -83,7 +83,7 @@ class RtpSenderVideoFrameTransformerDelegateTest : public ::testing::Test { delegate->TransformFrame( /*payload_type=*/1, VideoCodecType::kVideoCodecVP8, /*rtp_timestamp=*/2, encoded_image, RTPVideoHeader::FromMetadata(metadata), - /*expected_retransmission_time=*/TimeDelta::PlusInfinity()); + /*expected_retransmission_time=*/TimeDelta::Millis(10)); return frame; } @@ -123,7 +123,7 @@ TEST_F(RtpSenderVideoFrameTransformerDelegateTest, delegate->TransformFrame( /*payload_type=*/1, VideoCodecType::kVideoCodecVP8, /*rtp_timestamp=*/2, encoded_image, RTPVideoHeader(), - /*expected_retransmission_time=*/TimeDelta::PlusInfinity()); + /*expected_retransmission_time=*/TimeDelta::Millis(10)); } TEST_F(RtpSenderVideoFrameTransformerDelegateTest, @@ -260,7 +260,7 @@ TEST_F(RtpSenderVideoFrameTransformerDelegateTest, test_sender_, SendVideo(payload_type, absl::make_optional(kVideoCodecVP8), timestamp, /*capture_time=*/Timestamp::MinusInfinity(), buffer, _, _, - /*expected_retransmission_time=*/TimeDelta::PlusInfinity(), + /*expected_retransmission_time=*/TimeDelta::Millis(10), frame_csrcs)) .WillOnce(WithoutArgs([&] { event.Set(); @@ -289,5 +289,29 @@ TEST_F(RtpSenderVideoFrameTransformerDelegateTest, SettingRTPTimestamp) { EXPECT_EQ(video_frame.GetTimestamp(), rtp_timestamp); } +TEST_F(RtpSenderVideoFrameTransformerDelegateTest, + ShortCircuitingSkipsTransform) { + auto delegate = rtc::make_ref_counted<RTPSenderVideoFrameTransformerDelegate>( + &test_sender_, frame_transformer_, + /*ssrc=*/1111, time_controller_.CreateTaskQueueFactory().get()); + EXPECT_CALL(*frame_transformer_, + RegisterTransformedFrameSinkCallback(_, 1111)); + delegate->Init(); + + delegate->StartShortCircuiting(); + + // Will not call the actual transformer. + EXPECT_CALL(*frame_transformer_, Transform).Times(0); + // Will pass the frame straight to the reciever. + EXPECT_CALL(test_sender_, SendVideo); + + EncodedImage encoded_image; + encoded_image.SetEncodedData(EncodedImageBuffer::Create(1)); + delegate->TransformFrame( + /*payload_type=*/1, VideoCodecType::kVideoCodecVP8, /*rtp_timestamp=*/2, + encoded_image, RTPVideoHeader(), + /*expected_retransmission_time=*/TimeDelta::Millis(10)); +} + } // namespace } // namespace webrtc diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc index 94c9249e16..7af945c623 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.cc @@ -17,6 +17,7 @@ #include "absl/memory/memory.h" #include "modules/rtp_rtcp/source/rtp_descriptor_authentication.h" #include "rtc_base/checks.h" +#include "rtc_base/logging.h" #include "rtc_base/thread.h" namespace webrtc { @@ -123,9 +124,14 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::Reset() { void RtpVideoStreamReceiverFrameTransformerDelegate::TransformFrame( std::unique_ptr<RtpFrameObject> frame) { RTC_DCHECK_RUN_ON(&network_sequence_checker_); - frame_transformer_->Transform( - std::make_unique<TransformableVideoReceiverFrame>(std::move(frame), ssrc_, - receiver_)); + if (short_circuit_) { + // Just pass the frame straight back. + receiver_->ManageFrame(std::move(frame)); + } else { + frame_transformer_->Transform( + std::make_unique<TransformableVideoReceiverFrame>(std::move(frame), + ssrc_, receiver_)); + } } void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame( @@ -138,6 +144,20 @@ void RtpVideoStreamReceiverFrameTransformerDelegate::OnTransformedFrame( }); } +void RtpVideoStreamReceiverFrameTransformerDelegate::StartShortCircuiting() { + rtc::scoped_refptr<RtpVideoStreamReceiverFrameTransformerDelegate> delegate( + this); + network_thread_->PostTask([delegate = std::move(delegate)]() mutable { + delegate->StartShortCircuitingOnNetworkSequence(); + }); +} + +void RtpVideoStreamReceiverFrameTransformerDelegate:: + StartShortCircuitingOnNetworkSequence() { + RTC_DCHECK_RUN_ON(&network_sequence_checker_); + short_circuit_ = true; +} + void RtpVideoStreamReceiverFrameTransformerDelegate::ManageFrame( std::unique_ptr<TransformableFrameInterface> frame) { RTC_DCHECK_RUN_ON(&network_sequence_checker_); diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h index 20f9a5caa9..02f2e53923 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate.h @@ -54,6 +54,8 @@ class RtpVideoStreamReceiverFrameTransformerDelegate void OnTransformedFrame( std::unique_ptr<TransformableFrameInterface> frame) override; + void StartShortCircuiting() override; + // Delegates the call to RtpVideoFrameReceiver::ManageFrame on the // `network_thread_`. void ManageFrame(std::unique_ptr<TransformableFrameInterface> frame); @@ -62,6 +64,8 @@ class RtpVideoStreamReceiverFrameTransformerDelegate ~RtpVideoStreamReceiverFrameTransformerDelegate() override = default; private: + void StartShortCircuitingOnNetworkSequence(); + RTC_NO_UNIQUE_ADDRESS SequenceChecker network_sequence_checker_; RtpVideoFrameReceiver* receiver_ RTC_GUARDED_BY(network_sequence_checker_); rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_ @@ -69,6 +73,7 @@ class RtpVideoStreamReceiverFrameTransformerDelegate TaskQueueBase* const network_thread_; const uint32_t ssrc_; Clock* const clock_; + bool short_circuit_ RTC_GUARDED_BY(network_sequence_checker_) = false; }; } // namespace webrtc diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc index f403c91a74..cf3062610f 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/rtp_video_stream_receiver_frame_transformer_delegate_unittest.cc @@ -349,5 +349,28 @@ TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); } +TEST(RtpVideoStreamReceiverFrameTransformerDelegateTest, + ShortCircuitingSkipsTransform) { + rtc::AutoThread main_thread_; + TestRtpVideoFrameReceiver receiver; + auto mock_frame_transformer = + rtc::make_ref_counted<NiceMock<MockFrameTransformer>>(); + SimulatedClock clock(0); + auto delegate = + rtc::make_ref_counted<RtpVideoStreamReceiverFrameTransformerDelegate>( + &receiver, &clock, mock_frame_transformer, rtc::Thread::Current(), + 1111); + delegate->Init(); + + delegate->StartShortCircuiting(); + rtc::ThreadManager::ProcessAllMessageQueuesForTesting(); + + // Will not call the actual transformer. + EXPECT_CALL(*mock_frame_transformer, Transform).Times(0); + // Will pass the frame straight to the reciever. + EXPECT_CALL(receiver, ManageFrame); + delegate->TransformFrame(CreateRtpFrameObject()); +} + } // namespace } // namespace webrtc diff --git a/third_party/libwebrtc/modules/rtp_rtcp/source/video_rtp_depacketizer_av1.cc b/third_party/libwebrtc/modules/rtp_rtcp/source/video_rtp_depacketizer_av1.cc index 870f788538..30bbbc5000 100644 --- a/third_party/libwebrtc/modules/rtp_rtcp/source/video_rtp_depacketizer_av1.cc +++ b/third_party/libwebrtc/modules/rtp_rtcp/source/video_rtp_depacketizer_av1.cc @@ -188,8 +188,7 @@ VectorObuInfo ParseObus( VectorObuInfo obu_infos; bool expect_continues_obu = false; for (rtc::ArrayView<const uint8_t> rtp_payload : rtp_payloads) { - rtc::ByteBufferReader payload( - reinterpret_cast<const char*>(rtp_payload.data()), rtp_payload.size()); + rtc::ByteBufferReader payload(rtp_payload); uint8_t aggregation_header; if (!payload.ReadUInt8(&aggregation_header)) { RTC_DLOG(LS_WARNING) |