[vda/vtb] limit m_max_ref_frames to 2-5 because some encoders just cannot get it right

2 files changed, 10 insertions, 5 deletions

diff --git a/xbmc/cores/dvdplayer/DVDCodecs/Video/DVDVideoCodecVDA.cpp b/xbmc/cores/dvdplayer/DVDCodecs/Video/DVDVideoCodecVDA.cpp
index 3bc3fbbda3..3fdf172830 100644
--- a/xbmc/cores/dvdplayer/DVDCodecs/Video/DVDVideoCodecVDA.cpp
+++ b/xbmc/cores/dvdplayer/DVDCodecs/Video/DVDVideoCodecVDA.cpp
@@ -678,7 +678,6 @@ bool CDVDVideoCodecVDA::Open(CDVDStreamInfo &hints, CDVDCodecOptions &options)
     switch (hints.codec)
     {
       case CODEC_ID_H264:
-        // TODO: need to quality h264 encoding (profile, level and number of reference frame)
         // source must be H.264 with valid avcC atom data in extradata
         if (extrasize < 7 || extradata == NULL)
         {
@@ -770,6 +769,8 @@ bool CDVDVideoCodecVDA::Open(CDVDStreamInfo &hints, CDVDCodecOptions &options)
         CFRelease(avcCData);
         return false;
       }
+      if (m_max_ref_frames == 0)
+        m_max_ref_frames = 2;
     }
 
     // input stream is qualified, now we can load dlls.
@@ -864,6 +865,7 @@ bool CDVDVideoCodecVDA::Open(CDVDStreamInfo &hints, CDVDCodecOptions &options)
     memset(m_videobuffer.data[2], 0, iChromaPixels);
 
     m_DropPictures = false;
+    m_max_ref_frames = std::min(m_max_ref_frames, 5);
     m_sort_time_offset = (CurrentHostCounter() * 1000.0) / CurrentHostFrequency();
 
     return true;
@@ -981,7 +983,7 @@ int CDVDVideoCodecVDA::Decode(BYTE* pData, int iSize, double dts, double pts)
     }
   }
 
-  if (m_queue_depth < m_max_ref_frames)
+  if (!m_queue_depth || m_queue_depth < m_max_ref_frames)
   {
     return VC_BUFFER;
   }
diff --git a/xbmc/cores/dvdplayer/DVDCodecs/Video/DVDVideoCodecVideoToolBox.cpp b/xbmc/cores/dvdplayer/DVDCodecs/Video/DVDVideoCodecVideoToolBox.cpp
index 6ca58abd3f..782c1ce137 100644
--- a/xbmc/cores/dvdplayer/DVDCodecs/Video/DVDVideoCodecVideoToolBox.cpp
+++ b/xbmc/cores/dvdplayer/DVDCodecs/Video/DVDVideoCodecVideoToolBox.cpp
@@ -1135,7 +1135,7 @@ bool CDVDVideoCodecVideoToolBox::Open(CDVDStreamInfo &hints, CDVDCodecOptions &o
           m_fmt_desc = CreateFormatDescriptionFromCodecData(
             kVTFormatH264, width, height, extradata, extrasize, 'avcC');
 
-          CLog::Log(LOGNOTICE, "%s - using avcC atom of size(%d)", __FUNCTION__, extrasize);
+          CLog::Log(LOGNOTICE, "%s - using avcC atom of size(%d), ref_frames(%d)", __FUNCTION__, extrasize, m_max_ref_frames);
         }
         else
         {
@@ -1191,6 +1191,9 @@ bool CDVDVideoCodecVideoToolBox::Open(CDVDStreamInfo &hints, CDVDCodecOptions &o
       m_pFormatName = "";
       return false;
     }
+    if (m_max_ref_frames == 0)
+      m_max_ref_frames = 2;
+
     CreateVTSession(width, height, m_fmt_desc);
     if (m_vt_session == NULL)
     {
@@ -1219,7 +1222,7 @@ bool CDVDVideoCodecVideoToolBox::Open(CDVDStreamInfo &hints, CDVDCodecOptions &o
     m_videobuffer.iDisplayHeight = hints.height;
 
     m_DropPictures = false;
-    m_max_ref_frames = std::min(m_max_ref_frames, 4);
+    m_max_ref_frames = std::min(m_max_ref_frames, 5);
     m_sort_time_offset = (CurrentHostCounter() * 1000.0) / CurrentHostFrequency();
 
     return true;
@@ -1367,7 +1370,7 @@ int CDVDVideoCodecVideoToolBox::Decode(BYTE* pData, int iSize, double dts, doubl
 
   // TODO: queue depth is related to the number of reference frames in encoded h.264.
   // so we need to buffer until we get N ref frames + 1.
-  if (m_queue_depth < m_max_ref_frames)
+  if (!m_queue_depth || m_queue_depth < m_max_ref_frames)
     return VC_BUFFER;
 
   return VC_PICTURE | VC_BUFFER;