aboutsummaryrefslogtreecommitdiff
path: root/youtube_dl/extractor/generic.py
diff options
context:
space:
mode:
authoranovicecodemonkey <anovicecodemonkey13435@mailinator.com>2014-06-01 01:16:35 +0930
committeranovicecodemonkey <anovicecodemonkey13435@mailinator.com>2014-06-01 01:16:35 +0930
commit37e3cbe22e0bfa6b98a6343be88e1c8c2c7ac41f (patch)
tree081cb0e3a1d949d4e264aa32ccee6fcfae82e0cc /youtube_dl/extractor/generic.py
parent610134730abfdaaa226de2092d8ad5d731d5b54b (diff)
Move duplicate check to generic.py
Diffstat (limited to 'youtube_dl/extractor/generic.py')
-rw-r--r--youtube_dl/extractor/generic.py24
1 files changed, 24 insertions, 0 deletions
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c1e533821..dfa8d6153 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -494,6 +494,14 @@ class GenericIE(InfoExtractor):
if matches:
urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube')
for tuppl in matches]
+ # First, ensure we have a duplicate free list of entries
+ seen = set()
+ new_list = []
+ theurl = tuple(url.items())
+ if theurl not in seen:
+ seen.add(theurl)
+ new_list.append(url)
+ urlrs = new_list
return self.playlist_result(
urlrs, playlist_id=video_id, playlist_title=video_title)
@@ -503,6 +511,14 @@ class GenericIE(InfoExtractor):
if matches:
urlrs = [self.url_result(unescapeHTML(tuppl[1]))
for tuppl in matches]
+ # First, ensure we have a duplicate free list of entries
+ seen = set()
+ new_list = []
+ theurl = tuple(url.items())
+ if theurl not in seen:
+ seen.add(theurl)
+ new_list.append(url)
+ urlrs = new_list
return self.playlist_result(
urlrs, playlist_id=video_id, playlist_title=video_title)
@@ -615,6 +631,14 @@ class GenericIE(InfoExtractor):
if matches:
urlrs = [self.url_result(unescapeHTML(eurl), 'FunnyOrDie')
for eurl in matches]
+ # First, ensure we have a duplicate free list of entries
+ seen = set()
+ new_list = []
+ theurl = tuple(url.items())
+ if theurl not in seen:
+ seen.add(theurl)
+ new_list.append(url)
+ urlrs = new_list
return self.playlist_result(
urlrs, playlist_id=video_id, playlist_title=video_title)