aboutsummaryrefslogtreecommitdiff
path: root/youtube_dl/utils.py
diff options
context:
space:
mode:
authorPhilipp Hagemeister <phihag@phihag.de>2014-01-22 20:00:16 +0100
committerPhilipp Hagemeister <phihag@phihag.de>2014-01-22 20:00:16 +0100
commit65697b3bf3bf6eaeb91a34e5308a6d2239118071 (patch)
treee174cfbd658bd57da9d0fa990dc10949065db42b /youtube_dl/utils.py
parent50317b111dadccba73bcdd828d9997d1da78a5f1 (diff)
parentb7ab05908440915c6c5faa541abe00c62a88bc27 (diff)
Merge branch 'paged-lists'
Conflicts: test/test_utils.py youtube_dl/extractor/youtube.py
Diffstat (limited to 'youtube_dl/utils.py')
-rw-r--r--youtube_dl/utils.py44
1 files changed, 44 insertions, 0 deletions
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 3d2903986..ed5ee222f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -6,6 +6,7 @@ import datetime
import email.utils
import errno
import gzip
+import itertools
import io
import json
import locale
@@ -1164,3 +1165,46 @@ def check_executable(exe, args=[]):
except OSError:
return False
return exe
+
+
+class PagedList(object):
+ def __init__(self, pagefunc, pagesize):
+ self._pagefunc = pagefunc
+ self._pagesize = pagesize
+
+ def getslice(self, start=0, end=None):
+ res = []
+ for pagenum in itertools.count(start // self._pagesize):
+ firstid = pagenum * self._pagesize
+ nextfirstid = pagenum * self._pagesize + self._pagesize
+ if start >= nextfirstid:
+ continue
+
+ page_results = list(self._pagefunc(pagenum))
+
+ startv = (
+ start % self._pagesize
+ if firstid <= start < nextfirstid
+ else 0)
+
+ endv = (
+ ((end - 1) % self._pagesize) + 1
+ if (end is not None and firstid <= end <= nextfirstid)
+ else None)
+
+ if startv != 0 or endv is not None:
+ page_results = page_results[startv:endv]
+ res.extend(page_results)
+
+ # A little optimization - if current page is not "full", ie. does
+ # not contain page_size videos then we can assume that this page
+ # is the last one - there are no more ids on further pages -
+ # i.e. no need to query again.
+ if len(page_results) + startv < self._pagesize:
+ break
+
+ # If we got the whole page, but the next page is not interesting,
+ # break out early as well
+ if end == nextfirstid:
+ break
+ return res