diff --git a/RELEASENOTES.md b/RELEASENOTES.md index 4678b2a8fdf..1c4dbbc843f 100644 --- a/RELEASENOTES.md +++ b/RELEASENOTES.md @@ -127,6 +127,8 @@ ([#10165](https://github.com/google/ExoPlayer/issues/10165)). * Add RTP reader for VP9 ([#47](https://github.com/androidx/media/pull/64)). + * Add RTP reader for OPUS + ([#53](https://github.com/androidx/media/pull/53)). * Session: * Fix NPE in MediaControllerImplLegacy ([#59](https://github.com/androidx/media/pull/59)). diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java index 5f3e382a4bf..39b7d6f0eb8 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java @@ -46,6 +46,7 @@ public final class RtpPayloadFormat { private static final String RTP_MEDIA_MPEG4_VIDEO = "MP4V-ES"; private static final String RTP_MEDIA_H264 = "H264"; private static final String RTP_MEDIA_H265 = "H265"; + private static final String RTP_MEDIA_OPUS = "OPUS"; private static final String RTP_MEDIA_PCM_L8 = "L8"; private static final String RTP_MEDIA_PCM_L16 = "L16"; private static final String RTP_MEDIA_PCMA = "PCMA"; @@ -63,6 +64,7 @@ public static boolean isFormatSupported(MediaDescription mediaDescription) { case RTP_MEDIA_H265: case RTP_MEDIA_MPEG4_VIDEO: case RTP_MEDIA_MPEG4_GENERIC: + case RTP_MEDIA_OPUS: case RTP_MEDIA_PCM_L8: case RTP_MEDIA_PCM_L16: case RTP_MEDIA_PCMA: @@ -92,6 +94,8 @@ public static String getMimeTypeFromRtpMediaType(String mediaType) { return MimeTypes.AUDIO_AMR_WB; case RTP_MEDIA_MPEG4_GENERIC: return MimeTypes.AUDIO_AAC; + case RTP_MEDIA_OPUS: + return MimeTypes.AUDIO_OPUS; case RTP_MEDIA_PCM_L8: case RTP_MEDIA_PCM_L16: return MimeTypes.AUDIO_RAW; diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java index ad4b3e26e7c..2a7310c4708 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java @@ -101,6 +101,9 @@ */ private static final int DEFAULT_VP8_HEIGHT = 240; + /** RFC7587 Section 6.1 Sampling rate for OPUS is fixed at 48KHz. */ + private static final int OPUS_CLOCK_RATE = 48_000; + /** * Default width for VP9. * @@ -201,6 +204,12 @@ public int hashCode() { !fmtpParameters.containsKey(PARAMETER_AMR_INTERLEAVING), "Interleaving mode is not currently supported."); break; + case MimeTypes.AUDIO_OPUS: + checkArgument(channelCount != C.INDEX_UNSET); + // RFC7587 Section 6.1: the RTP timestamp is incremented with a 48000 Hz clock rate + // for all modes of Opus and all sampling rates. + checkArgument(clockRate == OPUS_CLOCK_RATE, "Invalid OPUS clock rate."); + break; case MimeTypes.VIDEO_MP4V: checkArgument(!fmtpParameters.isEmpty()); processMPEG4FmtpAttribute(formatBuilder, fmtpParameters); diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java index 3e3b8c0beb7..7c09884475b 100644 --- a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java @@ -39,6 +39,8 @@ public RtpPayloadReader createPayloadReader(RtpPayloadFormat payloadFormat) { case MimeTypes.AUDIO_AMR_NB: case MimeTypes.AUDIO_AMR_WB: return new RtpAmrReader(payloadFormat); + case MimeTypes.AUDIO_OPUS: + return new RtpOpusReader(payloadFormat); case MimeTypes.AUDIO_RAW: case MimeTypes.AUDIO_ALAW: case MimeTypes.AUDIO_MLAW: diff --git a/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReader.java b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReader.java new file mode 100644 index 00000000000..9b8994ae6db --- /dev/null +++ b/libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReader.java @@ -0,0 +1,157 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.rtsp.reader; + +import static androidx.media3.common.util.Assertions.checkArgument; +import static androidx.media3.common.util.Assertions.checkStateNotNull; + +import androidx.media3.common.C; +import androidx.media3.common.Format; +import androidx.media3.common.util.Log; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.common.util.Util; +import androidx.media3.exoplayer.rtsp.RtpPacket; +import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; +import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.extractor.OpusUtil; +import androidx.media3.extractor.TrackOutput; +import java.util.List; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Parses an OPUS byte stream carried on RTP packets and extracts individual samples. Refer to + * RFC7845 for more details. + */ +/* package */ final class RtpOpusReader implements RtpPayloadReader { + private static final String TAG = "RtpOpusReader"; + /* Opus uses a fixed 48KHz media clock RFC7845 Section 4. */ + private static final long MEDIA_CLOCK_FREQUENCY = 48_000; + + private final RtpPayloadFormat payloadFormat; + + private @MonotonicNonNull TrackOutput trackOutput; + + /** + * First received RTP timestamp. All RTP timestamps are dimension-less, the time base is defined + * by {@link #MEDIA_CLOCK_FREQUENCY}. + */ + private long firstReceivedTimestamp; + + private long startTimeOffsetUs; + private int previousSequenceNumber; + private boolean foundOpusIDHeader; + private boolean foundOpusCommentHeader; + + /** Creates an instance. */ + public RtpOpusReader(RtpPayloadFormat payloadFormat) { + this.payloadFormat = payloadFormat; + this.firstReceivedTimestamp = C.INDEX_UNSET; + this.previousSequenceNumber = C.INDEX_UNSET; + } + + // RtpPayloadReader implementation. + + @Override + public void createTracks(ExtractorOutput extractorOutput, int trackId) { + trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_AUDIO); + trackOutput.format(payloadFormat.format); + } + + @Override + public void onReceivingFirstPacket(long timestamp, int sequenceNumber) { + this.firstReceivedTimestamp = timestamp; + } + + @Override + public void consume( + ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) { + checkStateNotNull(trackOutput); + + /* RFC7845 Section 3. + * +---------+ +----------------+ +--------------------+ +----- + * |ID Header| | Comment Header | |Audio Data Packet 1 | | ... + * +---------+ +----------------+ +--------------------+ +----- + */ + if (!foundOpusIDHeader) { + validateOpusIdHeader(data); + List initializationData = OpusUtil.buildInitializationData(data.getData()); + Format.Builder formatBuilder = payloadFormat.format.buildUpon(); + formatBuilder.setInitializationData(initializationData); + trackOutput.format(formatBuilder.build()); + foundOpusIDHeader = true; + } else if (!foundOpusCommentHeader) { + // Comment Header RFC7845 Section 5.2. + int sampleSize = data.limit(); + checkArgument(sampleSize >= 8, "Comment Header has insufficient data"); + String header = data.readString(8); + checkArgument(header.equals("OpusTags"), "Comment Header should follow ID Header"); + foundOpusCommentHeader = true; + } else { + // Check that this packet is in the sequence of the previous packet. + int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber); + if (sequenceNumber != expectedSequenceNumber) { + Log.w( + TAG, + Util.formatInvariant( + "Received RTP packet with unexpected sequence number. Expected: %d; received: %d.", + expectedSequenceNumber, sequenceNumber)); + } + + // sending opus data. + int size = data.bytesLeft(); + trackOutput.sampleData(data, size); + long timeUs = toSampleTimeUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp); + trackOutput.sampleMetadata( + timeUs, C.BUFFER_FLAG_KEY_FRAME, size, /* offset*/ 0, /* cryptoData*/ null); + } + previousSequenceNumber = sequenceNumber; + } + + @Override + public void seek(long nextRtpTimestamp, long timeUs) { + firstReceivedTimestamp = nextRtpTimestamp; + startTimeOffsetUs = timeUs; + } + + // Internal methods. + + /** + * Validates the OPUS ID Header at {@code data}'s current position, throws {@link + * IllegalArgumentException} if the header is invalid. + * + *

{@code data}'s position does not change after returning. + */ + private static void validateOpusIdHeader(ParsableByteArray data) { + int currPosition = data.getPosition(); + int sampleSize = data.limit(); + checkArgument(sampleSize > 18, "ID Header has insufficient data"); + String header = data.readString(8); + // Identification header RFC7845 Section 5.1. + checkArgument(header.equals("OpusHead"), "ID Header missing"); + checkArgument(data.readUnsignedByte() == 1, "version number must always be 1"); + data.setPosition(currPosition); + } + + /** Returns the correct sample time from RTP timestamp, accounting for the OPUS sampling rate. */ + private static long toSampleTimeUs( + long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) { + return startTimeOffsetUs + + Util.scaleLargeTimestamp( + rtpTimestamp - firstReceivedRtpTimestamp, + /* multiplier= */ C.MICROS_PER_SECOND, + /* divisor= */ MEDIA_CLOCK_FREQUENCY); + } +} diff --git a/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReaderTest.java b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReaderTest.java new file mode 100644 index 00000000000..1b2ed3a50b0 --- /dev/null +++ b/libraries/exoplayer_rtsp/src/test/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReaderTest.java @@ -0,0 +1,200 @@ +/* + * Copyright 2022 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package androidx.media3.exoplayer.rtsp.reader; + +import static androidx.media3.common.util.Util.getBytesFromHexString; +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.when; + +import androidx.media3.common.Format; +import androidx.media3.common.MimeTypes; +import androidx.media3.common.util.ParsableByteArray; +import androidx.media3.exoplayer.rtsp.RtpPacket; +import androidx.media3.exoplayer.rtsp.RtpPayloadFormat; +import androidx.media3.extractor.ExtractorOutput; +import androidx.media3.test.utils.FakeTrackOutput; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import com.google.common.collect.ImmutableMap; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Mock; +import org.mockito.junit.MockitoJUnit; +import org.mockito.junit.MockitoRule; + +/** Unit test for {@link RtpOpusReader}. */ +@RunWith(AndroidJUnit4.class) +public final class RtpOpusReaderTest { + + private static final RtpPayloadFormat OPUS_FORMAT = + new RtpPayloadFormat( + new Format.Builder() + .setChannelCount(6) + .setSampleMimeType(MimeTypes.AUDIO_OPUS) + .setSampleRate(48_000) + .build(), + /* rtpPayloadType= */ 97, + /* clockRate= */ 48_000, + /* fmtpParameters= */ ImmutableMap.of()); + + private static final RtpPacket OPUS_HEADER = + createRtpPacket( + /* timestamp= */ 2599168056L, + /* sequenceNumber= */ 40289, + /* payloadData= */ getBytesFromHexString("4F707573486561640102000000000000000000")); + private static final RtpPacket OPUS_TAGS = + createRtpPacket( + /* timestamp= */ 2599168056L, + /* sequenceNumber= */ 40290, + /* payloadData= */ getBytesFromHexString("4F707573546167730000000000000000000000")); + private static final RtpPacket OPUS_FRAME_1 = + createRtpPacket( + /* timestamp= */ 2599168056L, + /* sequenceNumber= */ 40292, + /* payloadData= */ getBytesFromHexString("010203")); + private static final RtpPacket OPUS_FRAME_2 = + createRtpPacket( + /* timestamp= */ 2599169592L, + /* sequenceNumber= */ 40293, + /* payloadData= */ getBytesFromHexString("04050607")); + + @Rule public final MockitoRule mockito = MockitoJUnit.rule(); + + private ParsableByteArray packetData; + private RtpOpusReader opusReader; + private FakeTrackOutput trackOutput; + @Mock private ExtractorOutput extractorOutput; + + @Before + public void setUp() { + packetData = new ParsableByteArray(); + trackOutput = new FakeTrackOutput(/* deduplicateConsecutiveFormats= */ true); + when(extractorOutput.track(anyInt(), anyInt())).thenReturn(trackOutput); + opusReader = new RtpOpusReader(OPUS_FORMAT); + opusReader.createTracks(extractorOutput, /* trackId= */ 0); + } + + @Test + public void consume_validPackets() { + opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber); + consume(OPUS_HEADER); + consume(OPUS_TAGS); + consume(OPUS_FRAME_1); + consume(OPUS_FRAME_2); + + assertThat(trackOutput.getSampleCount()).isEqualTo(2); + assertThat(trackOutput.getSampleData(0)).isEqualTo(getBytesFromHexString("010203")); + assertThat(trackOutput.getSampleTimeUs(0)).isEqualTo(0); + assertThat(trackOutput.getSampleData(1)).isEqualTo(getBytesFromHexString("04050607")); + assertThat(trackOutput.getSampleTimeUs(1)).isEqualTo(32000); + } + + @Test + public void consume_opusHeaderWithInvalidHeader_throwsIllegalArgumentException() { + opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber); + assertThrows( + IllegalArgumentException.class, + () -> + consume( + createRtpPacket( + /* timestamp= */ 2599168056L, + /* sequenceNumber= */ 40289, + // Modify "OpusHead" -> "OrusHead" (First 8 bytes). + /* payloadData= */ getBytesFromHexString( + "4F727573486561640102000000000000000000")))); + } + + @Test + public void consume_opusHeaderWithInvalidSampleSize_throwsIllegalArgumentException() { + opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber); + + assertThrows( + IllegalArgumentException.class, + () -> + consume( + createRtpPacket( + /* timestamp= */ 2599168056L, + /* sequenceNumber= */ 40289, + // Truncate the opusHeader payload data. + /* payloadData= */ getBytesFromHexString("4F707573486561640102")))); + } + + @Test + public void consume_opusHeaderWithInvalidVersionNumber_throwsIllegalArgumentException() { + opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber); + assertThrows( + IllegalArgumentException.class, + () -> + consume( + createRtpPacket( + /* timestamp= */ 2599168056L, + /* sequenceNumber= */ 40289, + // Modify version 1 -> 2 (9th byte) + /* payloadData= */ getBytesFromHexString( + "4f707573486561640202000000000000000000")))); + } + + @Test + public void consume_invalidOpusTags_throwsIllegalArgumentException() { + opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber); + consume(OPUS_HEADER); + assertThrows( + IllegalArgumentException.class, + () -> + consume( + createRtpPacket( + /* timestamp= */ 2599168056L, + /* sequenceNumber= */ 40290, + // Modify "OpusTags" -> "OpusTggs" (First 8 bytes) + /* payloadData= */ getBytesFromHexString("4F70757354676773")))); + } + + @Test + public void consume_skipOpusTags_throwsIllegalArgumentException() { + opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber); + consume(OPUS_HEADER); + assertThrows(IllegalArgumentException.class, () -> consume(OPUS_FRAME_1)); + } + + @Test + public void consume_skipOpusHeader_throwsIllegalArgumentException() { + opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber); + assertThrows(IllegalArgumentException.class, () -> consume(OPUS_TAGS)); + } + + @Test + public void consume_skipOpusHeaderAndOpusTags_throwsIllegalArgumentException() { + opusReader.onReceivingFirstPacket(OPUS_HEADER.timestamp, OPUS_HEADER.sequenceNumber); + assertThrows(IllegalArgumentException.class, () -> consume(OPUS_FRAME_1)); + } + + private static RtpPacket createRtpPacket(long timestamp, int sequenceNumber, byte[] payloadData) { + return new RtpPacket.Builder() + .setTimestamp(timestamp) + .setSequenceNumber(sequenceNumber) + .setMarker(false) + .setPayloadData(payloadData) + .build(); + } + + private void consume(RtpPacket rtpPacket) { + packetData.reset(rtpPacket.payloadData); + opusReader.consume(packetData, rtpPacket.timestamp, rtpPacket.sequenceNumber, rtpPacket.marker); + } +}