fix: use a better estimate of frame rate for cases with very short fi…

…rst sample durations (#838) Use the second sample in mp4 and webm formats. #835 had issues with merging due to golden file conflicts. Because we cannot make dependent pull requests, this is a replica of #835. --------- Signed-off-by: Cosmin Stejerean <cstejerean@meta.com> Co-authored-by: Cosmin Stejerean <cstejerean@meta.com>
shaka-project · Feb 28, 2024 · 5644041 · 5644041
1 parent 4d22e99
commit 5644041
Show file tree

Hide file tree

Showing 19 changed files with 44 additions and 27 deletions.
diff --git a/packager/app/test/testdata/audio-video-with-two-trick-play/output.mpd b/packager/app/test/testdata/audio-video-with-two-trick-play/output.mpd
@@ -19,9 +19,9 @@
         </SegmentBase>
       </Representation>
     </AdaptationSet>
-    <AdaptationSet id="2" contentType="video" width="640" height="360" maxFrameRate="30000/30030" par="16:9">
+    <AdaptationSet id="2" contentType="video" width="640" height="360" maxFrameRate="30000/22022" par="16:9">
       <EssentialProperty schemeIdUri="http://dashif.org/guidelines/trickmode" value="1"/>
-      <Representation id="2" bandwidth="211545" codecs="avc1.64001e" mimeType="video/mp4" sar="1:1" frameRate="30000/60060" maxPlayoutRate="60" codingDependency="false">
+      <Representation id="2" bandwidth="211545" codecs="avc1.64001e" mimeType="video/mp4" sar="1:1" frameRate="30000/22022" maxPlayoutRate="60" codingDependency="false">
         <BaseURL>bear-640x360-video-trick_play_factor_2.mp4</BaseURL>
         <SegmentBase indexRange="870-925" timescale="30000">
           <Initialization range="0-869"/>

diff --git a/packager/app/test/testdata/ec3-and-hls-single-segment-mp4-encrypted/output.m3u8 b/packager/app/test/testdata/ec3-and-hls-single-segment-mp4-encrypted/output.m3u8
@@ -5,7 +5,7 @@
 
 #EXT-X-MEDIA:TYPE=AUDIO,URI="bear-640x360-ec3-audio.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="2"
 
-#EXT-X-STREAM-INF:BANDWIDTH=1174214,AVERAGE-BANDWIDTH=1061802,CODECS="avc1.64001e,ec-3",RESOLUTION=640x360,FRAME-RATE=9.990,AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
+#EXT-X-STREAM-INF:BANDWIDTH=1174214,AVERAGE-BANDWIDTH=1061802,CODECS="avc1.64001e,ec-3",RESOLUTION=640x360,FRAME-RATE=29.970,AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
 bear-640x360-ec3-video.m3u8
 
 #EXT-X-I-FRAME-STREAM-INF:BANDWIDTH=218705,AVERAGE-BANDWIDTH=159315,CODECS="avc1.64001e",RESOLUTION=640x360,CLOSED-CAPTIONS=NONE,URI="bear-640x360-ec3-video-iframe.m3u8"
diff --git a/packager/app/test/testdata/encryption-and-two-trick-plays/output.mpd b/packager/app/test/testdata/encryption-and-two-trick-plays/output.mpd
@@ -27,7 +27,7 @@
         </SegmentBase>
       </Representation>
     </AdaptationSet>
-    <AdaptationSet id="2" contentType="video" width="640" height="360" maxFrameRate="30000/30030" par="16:9">
+    <AdaptationSet id="2" contentType="video" width="640" height="360" maxFrameRate="30000/22022" par="16:9">
       <ContentProtection value="cenc" schemeIdUri="urn:mpeg:dash:mp4protection:2011" cenc:default_KID="31323334-3536-3738-3930-313233343536"/>
       <ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b">
         <cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
@@ -39,7 +39,7 @@
           <Initialization range="0-1137"/>
         </SegmentBase>
       </Representation>
-      <Representation id="3" bandwidth="212297" codecs="avc1.64001e" mimeType="video/mp4" sar="1:1" frameRate="30000/60060" maxPlayoutRate="60" codingDependency="false">
+      <Representation id="3" bandwidth="212297" codecs="avc1.64001e" mimeType="video/mp4" sar="1:1" frameRate="30000/22022" maxPlayoutRate="60" codingDependency="false">
         <BaseURL>bear-640x360-video-trick_play_factor_2.mp4</BaseURL>
         <SegmentBase indexRange="1138-1193" timescale="30000">
           <Initialization range="0-1137"/>

diff --git a/packager/app/test/testdata/live-profile-with-webm/output.m3u8 b/packager/app/test/testdata/live-profile-with-webm/output.m3u8
@@ -5,5 +5,5 @@
 
 #EXT-X-MEDIA:TYPE=AUDIO,URI="stream_0.m3u8",GROUP-ID="default-audio-group",NAME="stream_0",DEFAULT=NO,AUTOSELECT=YES,CHANNELS="2"
 
-#EXT-X-STREAM-INF:BANDWIDTH=556353,AVERAGE-BANDWIDTH=412719,CODECS="vp08.00.10.08.01.02.02.02.00,vorbis",RESOLUTION=640x360,FRAME-RATE=30.303,AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
+#EXT-X-STREAM-INF:BANDWIDTH=556353,AVERAGE-BANDWIDTH=412719,CODECS="vp08.00.10.08.01.02.02.02.00,vorbis",RESOLUTION=640x360,FRAME-RATE=29.412,AUDIO="default-audio-group",CLOSED-CAPTIONS=NONE
 stream_1.m3u8
diff --git a/packager/app/test/testdata/live-profile-with-webm/output.mpd b/packager/app/test/testdata/live-profile-with-webm/output.mpd
@@ -14,7 +14,7 @@
         </SegmentTemplate>
       </Representation>
     </AdaptationSet>
-    <AdaptationSet id="1" contentType="video" width="640" height="360" frameRate="1000000/33000" segmentAlignment="true" par="16:9">
+    <AdaptationSet id="1" contentType="video" width="640" height="360" frameRate="1000000/34000" segmentAlignment="true" par="16:9">
       <Representation id="1" bandwidth="472384" codecs="vp8" mimeType="video/webm" sar="1:1">
         <SegmentTemplate timescale="1000000" initialization="bear-640x360-video-init.webm" media="bear-640x360-video-$Number$.webm" startNumber="1">
           <SegmentTimeline>

diff --git a/packager/app/test/testdata/opus-vp9-mp4-with-encryption/output.mpd b/packager/app/test/testdata/opus-vp9-mp4-with-encryption/output.mpd
@@ -15,7 +15,7 @@
         </SegmentBase>
       </Representation>
     </AdaptationSet>
-    <AdaptationSet id="1" contentType="video" width="320" height="240" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
+    <AdaptationSet id="1" contentType="video" width="320" height="240" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
       <ContentProtection value="cenc" schemeIdUri="urn:mpeg:dash:mp4protection:2011" cenc:default_KID="31323334-3536-3738-3930-313233343536"/>
       <ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b">
         <cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>

diff --git a/packager/app/test/testdata/vp8-mp4-with-encryption/output.mpd b/packager/app/test/testdata/vp8-mp4-with-encryption/output.mpd
@@ -2,7 +2,7 @@
 <!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
 <MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" xmlns:cenc="urn:mpeg:cenc:2013" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.736S">
   <Period id="0">
-    <AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
+    <AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
       <ContentProtection value="cenc" schemeIdUri="urn:mpeg:dash:mp4protection:2011" cenc:default_KID="31323334-3536-3738-3930-313233343536"/>
       <ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b">
         <cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>

diff --git a/packager/app/test/testdata/vp8-webm/output.mpd b/packager/app/test/testdata/vp8-webm/output.mpd
@@ -2,7 +2,7 @@
 <!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
 <MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.736S">
   <Period id="0">
-    <AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
+    <AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
       <Representation id="0" bandwidth="472384" codecs="vp8" mimeType="video/webm" sar="1:1">
         <BaseURL>bear-640x360-video.webm</BaseURL>
         <SegmentBase indexRange="296-345" timescale="1000000" presentationTimeOffset="26000">

diff --git a/packager/app/test/testdata/vp9-webm-with-blockgroup/output.mpd b/packager/app/test/testdata/vp9-webm-with-blockgroup/output.mpd
@@ -2,7 +2,7 @@
 <!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
 <MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.736S">
   <Period id="0">
-    <AdaptationSet id="0" contentType="video" width="320" height="240" frameRate="1000000/33000" subsegmentAlignment="true" par="4:3">
+    <AdaptationSet id="0" contentType="video" width="320" height="240" frameRate="1000000/34000" subsegmentAlignment="true" par="4:3">
       <Representation id="0" bandwidth="196039" codecs="vp09.00.20.08.01.02.02.02.00" mimeType="video/webm" sar="1:1">
         <BaseURL>bear-vp9-blockgroup-video.webm</BaseURL>
         <SegmentBase indexRange="309-327" timescale="1000000">

diff --git a/packager/app/test/testdata/vp9-webm/output.mpd b/packager/app/test/testdata/vp9-webm/output.mpd
@@ -11,7 +11,7 @@
         </SegmentBase>
       </Representation>
     </AdaptationSet>
-    <AdaptationSet id="1" contentType="video" width="320" height="240" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
+    <AdaptationSet id="1" contentType="video" width="320" height="240" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
       <Representation id="1" bandwidth="225727" codecs="vp09.00.20.08.01.02.02.02.00" mimeType="video/webm" sar="427:320">
         <BaseURL>bear-320x240-vp9-opus-video.webm</BaseURL>
         <SegmentBase indexRange="309-357" timescale="1000000" presentationTimeOffset="37000">

diff --git a/packager/app/test/testdata/webm-subsample-encryption/output.mpd b/packager/app/test/testdata/webm-subsample-encryption/output.mpd
@@ -2,7 +2,7 @@
 <!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
 <MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" xmlns:cenc="urn:mpeg:cenc:2013" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.769S">
   <Period id="0">
-    <AdaptationSet id="0" contentType="video" width="320" height="180" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
+    <AdaptationSet id="0" contentType="video" width="320" height="180" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
       <ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b" cenc:default_KID="31323334-3536-3738-3930-313233343536">
         <cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
       </ContentProtection>

diff --git a/packager/app/test/testdata/webm-vp9-full-sample-encryption/output.mpd b/packager/app/test/testdata/webm-vp9-full-sample-encryption/output.mpd
@@ -2,7 +2,7 @@
 <!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
 <MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" xmlns:cenc="urn:mpeg:cenc:2013" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.769S">
   <Period id="0">
-    <AdaptationSet id="0" contentType="video" width="320" height="180" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
+    <AdaptationSet id="0" contentType="video" width="320" height="180" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
       <ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b" cenc:default_KID="31323334-3536-3738-3930-313233343536">
         <cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
       </ContentProtection>

diff --git a/packager/app/test/testdata/webm-with-encryption/output.mpd b/packager/app/test/testdata/webm-with-encryption/output.mpd
@@ -2,7 +2,7 @@
 <!--Generated with https://github.com/shaka-project/shaka-packager version <tag>-<hash>-<test>-->
 <MPD xmlns="urn:mpeg:dash:schema:mpd:2011" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 DASH-MPD.xsd" xmlns:cenc="urn:mpeg:cenc:2013" profiles="urn:mpeg:dash:profile:isoff-on-demand:2011" minBufferTime="PT2S" type="static" mediaPresentationDuration="PT2.736S">
   <Period id="0">
-    <AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="1000000/33000" subsegmentAlignment="true" par="16:9">
+    <AdaptationSet id="0" contentType="video" width="640" height="360" frameRate="1000000/34000" subsegmentAlignment="true" par="16:9">
       <ContentProtection schemeIdUri="urn:uuid:1077efec-c0b2-4d02-ace3-3c1e52e2fb4b" cenc:default_KID="31323334-3536-3738-3930-313233343536">
         <cenc:pssh>AAAANHBzc2gBAAAAEHfv7MCyTQKs4zweUuL7SwAAAAExMjM0NTY3ODkwMTIzNDU2AAAAAA==</cenc:pssh>
       </ContentProtection>

diff --git a/packager/media/formats/mp2t/ts_muxer.cc b/packager/media/formats/mp2t/ts_muxer.cc
@@ -49,6 +49,9 @@ Status TsMuxer::Finalize() {
 
 Status TsMuxer::AddMediaSample(size_t stream_id, const MediaSample& sample) {
   DCHECK_EQ(stream_id, 0u);
+
+  // The duration of the first sample may have been adjusted, so use
+  // the duration of the second sample instead.
   if (num_samples_ < 2) {
     sample_durations_[num_samples_] =
         sample.duration() * kTsTimescale / streams().front()->time_scale();

diff --git a/packager/media/formats/mp2t/ts_muxer.h b/packager/media/formats/mp2t/ts_muxer.h
@@ -38,8 +38,8 @@ class TsMuxer : public Muxer {
   void FireOnMediaEndEvent();
 
   std::unique_ptr<TsSegmenter> segmenter_;
-  int64_t sample_durations_[2];
-  int64_t num_samples_ = 0;
+  int64_t sample_durations_[2] = {0, 0};
+  size_t num_samples_ = 0;
 
   // Used in multi-segment mode for segment template.
   uint64_t segment_number_ = 0;

diff --git a/packager/media/formats/mp4/segmenter.cc b/packager/media/formats/mp4/segmenter.cc
@@ -141,8 +141,12 @@ Status Segmenter::AddSample(size_t stream_id, const MediaSample& sample) {
   if (!status.ok())
     return status;
 
-  if (sample_duration_ == 0)
-    sample_duration_ = sample.duration();
+  // The duration of the first sample may have been adjusted, so use
+  // the duration of the second sample instead.
+  if (num_samples_ < 2) {
+    sample_durations_[num_samples_] = sample.duration();
+    num_samples_++;
+  }
   stream_durations_[stream_id] += sample.duration();
   return Status::OK;
 }

diff --git a/packager/media/formats/mp4/segmenter.h b/packager/media/formats/mp4/segmenter.h
@@ -99,7 +99,9 @@ class Segmenter {
 
   /// @return The sample duration in the timescale of the media.
   ///         Returns 0 if no samples are added yet.
-  int32_t sample_duration() const { return sample_duration_; }
+  int64_t sample_duration() const {
+    return sample_durations_[num_samples_ < 2 ? 0 : 1];
+  }
 
  protected:
   /// Update segmentation progress using ProgressListener.
@@ -147,7 +149,8 @@ class Segmenter {
   ProgressListener* progress_listener_ = nullptr;
   uint64_t progress_target_ = 0u;
   uint64_t accumulated_progress_ = 0u;
-  int32_t sample_duration_ = 0;
+  int64_t sample_durations_[2] = {0, 0};
+  size_t num_samples_ = 0;
   std::vector<uint64_t> stream_durations_;
   std::vector<KeyFrameInfo> key_frame_infos_;
 

diff --git a/packager/media/formats/webm/segmenter.cc b/packager/media/formats/webm/segmenter.cc
@@ -159,11 +159,15 @@ Status Segmenter::Finalize() {
 Status Segmenter::AddSample(const MediaSample& source_sample) {
   std::shared_ptr<MediaSample> sample(source_sample.Clone());
 
-  if (sample_duration_ == 0) {
-    first_timestamp_ = sample->pts();
-    sample_duration_ = sample->duration();
-    if (muxer_listener_)
-      muxer_listener_->OnSampleDurationReady(sample_duration_);
+  // The duration of the first sample may have been adjusted, so use
+  // the duration of the second sample instead.
+  if (num_samples_ < 2) {
+    sample_durations_[num_samples_] = sample->duration();
+    if (num_samples_ == 0)
+      first_timestamp_ = sample->pts();
+    else if (muxer_listener_)
+      muxer_listener_->OnSampleDurationReady(sample_durations_[num_samples_]);
+    num_samples_++;
   }
 
   UpdateProgress(sample->duration());

diff --git a/packager/media/formats/webm/segmenter.h b/packager/media/formats/webm/segmenter.h
@@ -139,8 +139,11 @@ class Segmenter {
   ProgressListener* progress_listener_ = nullptr;
   uint64_t progress_target_ = 0;
   uint64_t accumulated_progress_ = 0;
+
   int64_t first_timestamp_ = 0;
-  int64_t sample_duration_ = 0;
+  int64_t sample_durations_[2] = {0, 0};
+  size_t num_samples_ = 0;
+
   // The position (in bytes) of the start of the Segment payload in the init
   // file.  This is also the size of the header before the SeekHead.
   uint64_t segment_payload_pos_ = 0;