Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for RTSP Opus #53

Merged
merged 5 commits into from
Jun 9, 2022
Merged

Add support for RTSP Opus #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
be108b9
Add support for RTSP Opus
ManishaJajoo Mar 3, 2022
3a87039
Fixed review comments in RtpOpusReader
ManishaJajoo Mar 23, 2022
f0d7d96
Merge branch 'main' into rtp_opus
ManishaJajoo Apr 21, 2022
0bf1973
Add a missing break in RtspMediaTrack and update RtpPayloadFormat.java
ManishaJajoo Apr 21, 2022
165e706
Add RTP Opus Reader Test
basantwanishraddha Apr 8, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtpPayloadFormat.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ public final class RtpPayloadFormat {
private static final String RTP_MEDIA_MPEG4_VIDEO = "MP4V-ES";
private static final String RTP_MEDIA_H264 = "H264";
private static final String RTP_MEDIA_H265 = "H265";
private static final String RTP_MEDIA_OPUS = "OPUS";
private static final String RTP_MEDIA_PCM_L8 = "L8";
private static final String RTP_MEDIA_PCM_L16 = "L16";
private static final String RTP_MEDIA_PCMA = "PCMA";
Expand All @@ -62,6 +63,7 @@ public static boolean isFormatSupported(MediaDescription mediaDescription) {
case RTP_MEDIA_H265:
case RTP_MEDIA_MPEG4_VIDEO:
case RTP_MEDIA_MPEG4_GENERIC:
case RTP_MEDIA_OPUS:
case RTP_MEDIA_PCM_L8:
case RTP_MEDIA_PCM_L16:
case RTP_MEDIA_PCMA:
Expand Down Expand Up @@ -90,6 +92,8 @@ public static String getMimeTypeFromRtpMediaType(String mediaType) {
return MimeTypes.AUDIO_AMR_WB;
case RTP_MEDIA_MPEG4_GENERIC:
return MimeTypes.AUDIO_AAC;
case RTP_MEDIA_OPUS:
return MimeTypes.AUDIO_OPUS;
case RTP_MEDIA_PCM_L8:
case RTP_MEDIA_PCM_L16:
return MimeTypes.AUDIO_RAW;
Expand Down
10 changes: 10 additions & 0 deletions libraries/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/RtspMediaTrack.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,9 @@
*/
private static final int DEFAULT_VP8_HEIGHT = 240;

/** RFC7587 Section 6.1 Sampling rate for OPUS is fixed at 48KHz. */
private static final int OPUS_SAMPLING_RATE = 48000;

/** The track's associated {@link RtpPayloadFormat}. */
public final RtpPayloadFormat payloadFormat;
/** The track's URI. */
Expand Down Expand Up @@ -180,6 +183,13 @@ public int hashCode() {
!fmtpParameters.containsKey(PARAMETER_AMR_INTERLEAVING),
"Interleaving mode is not currently supported.");
break;
case MimeTypes.AUDIO_OPUS:
checkArgument(channelCount != C.INDEX_UNSET);
// RFC7587 Section 6.1.
// the RTP timestamp is incremented with a 48000 Hz clock rate
// for all modes of Opus and all sampling rates.
checkArgument(clockRate == OPUS_SAMPLING_RATE, "Invalid sampling rate");
break;
case MimeTypes.VIDEO_MP4V:
checkArgument(!fmtpParameters.isEmpty());
processMPEG4FmtpAttribute(formatBuilder, fmtpParameters);
Expand Down
2 changes: 2 additions & 0 deletions ...p/src/main/java/androidx/media3/exoplayer/rtsp/reader/DefaultRtpPayloadReaderFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ public RtpPayloadReader createPayloadReader(RtpPayloadFormat payloadFormat) {
case MimeTypes.AUDIO_AMR_NB:
case MimeTypes.AUDIO_AMR_WB:
return new RtpAmrReader(payloadFormat);
case MimeTypes.AUDIO_OPUS:
return new RtpOpusReader(payloadFormat);
case MimeTypes.AUDIO_RAW:
case MimeTypes.AUDIO_ALAW:
case MimeTypes.AUDIO_MLAW:
Expand Down
146 changes: 146 additions & 0 deletions ...ies/exoplayer_rtsp/src/main/java/androidx/media3/exoplayer/rtsp/reader/RtpOpusReader.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
/*
* Copyright 2022 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package androidx.media3.exoplayer.rtsp.reader;

import static androidx.media3.common.util.Assertions.checkArgument;
import static androidx.media3.common.util.Assertions.checkStateNotNull;

import androidx.media3.common.C;
import androidx.media3.common.Format;
import androidx.media3.common.util.Log;
import androidx.media3.common.util.ParsableByteArray;
import androidx.media3.common.util.Util;
import androidx.media3.exoplayer.rtsp.RtpPacket;
import androidx.media3.exoplayer.rtsp.RtpPayloadFormat;
import androidx.media3.extractor.ExtractorOutput;
import androidx.media3.extractor.OpusUtil;
import androidx.media3.extractor.TrackOutput;
import java.util.List;
import org.checkerframework.checker.nullness.qual.MonotonicNonNull;

/**
* Parses an OPUS byte stream carried on RTP packets and extracts individual samples. Refer to
* RFC7845 for more details.
*/
/* package */ final class RtpOpusReader implements RtpPayloadReader {
private static final String TAG = "RtpOpusReader";

private final RtpPayloadFormat payloadFormat;
ManishaJajoo marked this conversation as resolved.
Show resolved Hide resolved
private static final long MEDIA_CLOCK_FREQUENCY = 48_000;

private @MonotonicNonNull TrackOutput trackOutput;
private long firstReceivedTimestamp;
private long startTimeOffsetUs;
private int previousSequenceNumber;
private boolean foundOpusIDHeader;
private boolean foundOpusCommentHeader;

public RtpOpusReader(RtpPayloadFormat payloadFormat) {
this.payloadFormat = payloadFormat;
this.firstReceivedTimestamp = C.INDEX_UNSET;
this.previousSequenceNumber = C.INDEX_UNSET;
this.foundOpusIDHeader = false;
this.foundOpusCommentHeader = false;
}

// RtpPayloadReader implementation.

@Override
public void createTracks(ExtractorOutput extractorOutput, int trackId) {
trackOutput = extractorOutput.track(trackId, C.TRACK_TYPE_AUDIO);
trackOutput.format(payloadFormat.format);
}

@Override
public void onReceivingFirstPacket(long timestamp, int sequenceNumber) {
this.firstReceivedTimestamp = timestamp;
}

@Override
public void consume(
ParsableByteArray data, long timestamp, int sequenceNumber, boolean rtpMarker) {
checkStateNotNull(trackOutput);

/* RFC7845 Section 3.
* +---------+ +----------------+ +--------------------+ +-----
* |ID Header| | Comment Header | |Audio Data Packet 1 | | ...
* +---------+ +----------------+ +--------------------+ +-----
*/
if (!foundOpusIDHeader) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for not getting back to this sooner, I thought we have merged it.

All the logic here to find Opus ID and comment headers imply that the header and comment header is ONE PACKET long. But that is not what is specified in the RFC.

According to the RFC, the ID header is one PAGE long, and the comment header can span across many pages. And one page can span multiple RTP packets. Also, "Audio data packets might span page boundaries. The first audio data page could have the 'continued packet' flag set"

Although this is working right now, it's not semantically correct. So please try to make it compliant with the RFC by supporting multiple packet pages. There are examples of one access unit spans multiple RTP packets, like in the H264/265 reader. It's probably hard to test, but given you have some tests going on, please add some test based on your interpretation to the RFC.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sorry for not getting back to this sooner, I thought we have merged it.

All the logic here to find Opus ID and comment headers imply that the header and comment header is ONE PACKET long. But that is not what is specified in the RFC.

According to the RFC, the ID header is one PAGE long, and the comment header can span across many pages. And one page can span multiple RTP packets. Also, "Audio data packets might span page boundaries. The first audio data page could have the 'continued packet' flag set"

Although this is working right now, it's not semantically correct. So please try to make it compliant with the RFC by supporting multiple packet pages. There are examples of one access unit spans multiple RTP packets, like in the H264/265 reader. It's probably hard to test, but given you have some tests going on, please add some test based on your interpretation to the RFC.

checkForOpusIdHeader(data);
List<byte[]> initializationData = OpusUtil.buildInitializationData(data.getData());
Format.Builder formatBuilder = payloadFormat.format.buildUpon();
formatBuilder.setInitializationData(initializationData);
trackOutput.format(formatBuilder.build());
foundOpusIDHeader = true;
} else if (!foundOpusCommentHeader) {
claincly marked this conversation as resolved.
Show resolved Hide resolved
// Comment Header RFC7845 Section 5.2.
int sampleSize = data.limit();
checkArgument(sampleSize >= 8 , "Comment Header has insufficient data");
String header = data.readString(8);
checkArgument(header.equals("OpusTags"), "Comment Header should follow ID Header");
foundOpusCommentHeader = true;
} else {
// Check that this packet is in the sequence of the previous packet.
int expectedSequenceNumber = RtpPacket.getNextSequenceNumber(previousSequenceNumber);
if (sequenceNumber != expectedSequenceNumber) {
Log.w(
TAG,
Util.formatInvariant(
"Received RTP packet with unexpected sequence number. Expected: %d; received: %d.",
expectedSequenceNumber, sequenceNumber));
}

// sending opus data.
int size = data.bytesLeft();
trackOutput.sampleData(data, size);
long timeUs =
toSampleTimeUs(startTimeOffsetUs, timestamp, firstReceivedTimestamp);
trackOutput.sampleMetadata(
timeUs, C.BUFFER_FLAG_KEY_FRAME, size, /* offset*/ 0, /* cryptoData*/ null);
}
previousSequenceNumber = sequenceNumber;
}

@Override
public void seek(long nextRtpTimestamp, long timeUs) {
firstReceivedTimestamp = nextRtpTimestamp;
startTimeOffsetUs = timeUs;
}

// Internal methods.

private static void checkForOpusIdHeader(ParsableByteArray data) {
int currPosition = data.getPosition();
int sampleSize = data.limit();
checkArgument(sampleSize > 18, "ID Header has insufficient data");
String header = data.readString(8);
// Identification header RFC7845 Section 5.1.
checkArgument(header.equals("OpusHead"), "ID Header missing");
checkArgument(data.readUnsignedByte() == 1, "version number must always be 1");
data.setPosition(currPosition);
}

/** Returns the correct sample time from RTP timestamp, accounting for the OPUS sampling rate. */
private static long toSampleTimeUs(
long startTimeOffsetUs, long rtpTimestamp, long firstReceivedRtpTimestamp) {
return startTimeOffsetUs
+ Util.scaleLargeTimestamp(
rtpTimestamp - firstReceivedRtpTimestamp,
/* multiplier= */ C.MICROS_PER_SECOND,
/* divisor= */ MEDIA_CLOCK_FREQUENCY);
}
}
Loading