new version of pytube

Author: ndg63276

pytube/__init__.py

@@ -5,7 +5,7 @@
 Pytube: a very serious Python library for downloading YouTube Videos.
 """
 __title__ = 'pytube'
-__version__ = '9.2.2'
+__version__ = '9.2.3'
 __author__ = 'Nick Ficano'
 __license__ = 'MIT License'
 __copyright__ = 'Copyright 2017 Nick Ficano'

pytube/__init__.pyc

@@ -21,6 +21,7 @@
 from pytube import StreamQuery
 from pytube.compat import install_proxy
 from pytube.compat import parse_qsl
+from pytube.exceptions import VideoUnavailable
 from pytube.helpers import apply_mixin
 
 logger = logging.getLogger(__name__)
@@ -116,6 +117,7 @@ def init(self):
                 self.watch_html,
             )['args']
 
+        self.vid_descr = extract.get_vid_descr(self.watch_html)
         # https://github.com/nficano/pytube/issues/165
         stream_maps = ['url_encoded_fmt_stream_map']
         if 'adaptive_fmts' in self.player_config_args:
@@ -156,6 +158,8 @@ def prefetch(self):
 
         """
         self.watch_html = request.get(url=self.watch_url)
+        if '<img class="icon meh" src="/yts/img' not in self.watch_html:
+            raise VideoUnavailable('This video is unavailable.')
         self.embed_html = request.get(url=self.embed_url)
         self.age_restricted = extract.is_age_restricted(self.watch_html)
         self.vid_info_url = extract.video_info_url(
@@ -249,6 +253,42 @@ def title(self):
         """
         return self.player_config_args['title']
 
+    @property
+    def description(self):
+        """Get the video description.
+
+        :rtype: str
+
+        """
+        return self.vid_descr
+
+    @property
+    def rating(self):
+        """Get the video average rating.
+
+        :rtype: str
+
+        """
+        return self.player_config_args['avg_rating']
+
+    @property
+    def length(self):
+        """Get the video length in seconds.
+
+        :rtype: str
+
+        """
+        return self.player_config_args['length_seconds']
+
+    @property
+    def views(self):
+        """Get the number of the times the video has been viewed.
+
+        :rtype: str
+
+        """
+        return self.player_config_args['view_count']
+
     def register_on_progress_callback(self, func):
         """Register a download progress callback function post initialization.
 

pytube/__main__.py

@@ -35,7 +35,12 @@ def get_initial_function_name(js):
 
     """
     # c&&d.set("signature", EE(c));
-    pattern = r'"signature",\s?([a-zA-Z0-9$]+)\('
+    pattern = [
+        r'yt\.akamaized\.net/\)\s*\|\|\s*'
+        r'.*?\s*c\s*&&\s*d\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
+        r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
+        r'\bc\s*&&\s*d\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
+    ]
     logger.debug('finding initial function name')
     return regex_search(pattern, js, group=1)
 

pytube/__main__.pyc

@@ -4,12 +4,14 @@
 """Python 2/3 compatibility support."""
 import sys
 
+
 PY2 = sys.version_info[0] == 2
 PY3 = sys.version_info[0] == 3
 PY33 = sys.version_info[0:2] >= (3, 3)
 
-
 if PY2:
+    reload(sys)
+    sys.setdefaultencoding('utf8')
     import urllib2
     from urllib import urlencode
     from urllib2 import URLError

pytube/captions.pyc

@@ -66,26 +66,67 @@ def populate_video_urls(self):
             complete_url = base_url + video_id
             self.video_urls.append(complete_url)
 
-    def download_all(self, download_path=None):
+    def _path_num_prefix_generator(self, reverse=False):
+        """
+        This generator function generates number prefixes, for the items
+        in the playlist.
+        If the number of digits required to name a file,is less than is
+        required to name the last file,it prepends 0s.
+        So if you have a playlist of 100 videos it will number them like:
+        001, 002, 003 ect, up to 100.
+        It also adds a space after the number.
+        :return: prefix string generator : generator
+        """
+        digits = len(str(len(self.video_urls)))
+        if reverse:
+            start, stop, step = (len(self.video_urls), 0, -1)
+        else:
+            start, stop, step = (1, len(self.video_urls) + 1, 1)
+        return (str(i).zfill(digits) for i in range(start, stop, step))
+
+    def download_all(
+        self, download_path=None, prefix_number=True,
+        reverse_numbering=False,
+    ):
         """Download all the videos in the the playlist. Initially, download
         resolution is 720p (or highest available), later more option
         should be added to download resolution of choice
 
         TODO(nficano): Add option to download resolution of user's choice
+
+        :param download_path:
+            (optional) Output path for the playlist If one is not
+            specified, defaults to the current working directory.
+            This is passed along to the Stream objects.
+        :type download_path: str or None
+        :param prefix_number:
+            (optional) Automatically numbers playlists using the
+            _path_num_prefix_generator function.
+        :type prefix_number: bool
+        :param reverse_numbering:
+            (optional) Lets you number playlists in reverse, since some
+            playlists are ordered newest -> oldests.
+        :type reverse_numbering: bool
         """
 
         self.populate_video_urls()
-        logger.debug('total videos found: ', len(self.video_urls))
+        logger.debug('total videos found: %d', len(self.video_urls))
         logger.debug('starting download')
 
+        prefix_gen = self._path_num_prefix_generator(reverse_numbering)
+
         for link in self.video_urls:
             yt = YouTube(link)
-
             # TODO: this should not be hardcoded to a single user's preference
             dl_stream = yt.streams.filter(
                 progressive=True, subtype='mp4',
             ).order_by('resolution').desc().first()
 
             logger.debug('download path: %s', download_path)
-            dl_stream.download(download_path)
+            if prefix_number:
+                prefix = next(prefix_gen)
+                logger.debug('file prefix is: %s', prefix)
+                dl_stream.download(download_path, filename_prefix=prefix)
+            else:
+                dl_stream.download(download_path)
             logger.debug('download complete')

pytube/cipher.py

@@ -34,3 +34,11 @@ def __init__(self, msg, video_id=None):
 
 class RegexMatchError(ExtractError):
     """Regex pattern did not return any matches."""
+
+
+class LiveStreamError(ExtractError):
+    """Video is a live stream."""
+
+
+class VideoUnavailable(PytubeError):
+    """Video is unavailable."""

pytube/cipher.pyc

@@ -3,12 +3,32 @@
 import json
 from collections import OrderedDict
 
+from pytube.compat import HTMLParser
 from pytube.compat import quote
 from pytube.compat import urlencode
 from pytube.exceptions import RegexMatchError
 from pytube.helpers import regex_search
 
 
+class PytubeHTMLParser(HTMLParser):
+    in_vid_descr = False
+    vid_descr = ''
+
+    def handle_starttag(self, tag, attrs):
+        if tag == 'p':
+            for attr in attrs:
+                if attr[0] == 'id' and attr[1] == 'eow-description':
+                    self.in_vid_descr = True
+
+    def handle_endtag(self, tag):
+        if tag == 'p' and self.in_vid_descr:
+            self.in_vid_descr = False
+
+    def handle_data(self, data):
+        if self.in_vid_descr:
+            self.vid_descr += data
+
+
 def is_age_restricted(watch_html):
     """Check if content is age restricted.
 
@@ -173,3 +193,9 @@ def get_ytplayer_config(html, age_restricted=False):
         pattern = r';ytplayer\.config\s*=\s*({.*?});'
     yt_player_config = regex_search(pattern, html, group=1)
     return json.loads(yt_player_config)
+
+
+def get_vid_descr(html):
+    html_parser = PytubeHTMLParser()
+    html_parser.feed(html)
+    return html_parser.vid_descr

pytube/cli.pyc

@@ -31,29 +31,55 @@ def regex_search(pattern, string, groups=False, group=None, flags=0):
     :returns:
         Substring pattern matches.
     """
-    regex = re.compile(pattern, flags)
-    results = regex.search(string)
-    if not results:
-        raise RegexMatchError(
-            'regex pattern ({pattern}) had zero matches'
-            .format(pattern=pattern),
-        )
+    if type(pattern) == list:
+        for p in pattern:
+            regex = re.compile(p, flags)
+            results = regex.search(string)
+            if not results:
+                raise RegexMatchError(
+                    'regex pattern ({pattern}) had zero matches'
+                    .format(pattern=p),
+                )
+            else:
+                logger.debug(
+                    'finished regex search: %s',
+                    pprint.pformat(
+                        {
+                            'pattern': p,
+                            'results': results.group(0),
+                        }, indent=2,
+                    ),
+                )
+                if groups:
+                    return results.groups()
+                elif group is not None:
+                    return results.group(group)
+                else:
+                    return results
     else:
-        logger.debug(
-            'finished regex search: %s',
-            pprint.pformat(
-                {
-                    'pattern': pattern,
-                    'results': results.group(0),
-                }, indent=2,
-            ),
-        )
-        if groups:
-            return results.groups()
-        elif group is not None:
-            return results.group(group)
+        regex = re.compile(pattern, flags)
+        results = regex.search(string)
+        if not results:
+            raise RegexMatchError(
+                'regex pattern ({pattern}) had zero matches'
+                .format(pattern=pattern),
+            )
         else:
-            return results
+            logger.debug(
+                'finished regex search: %s',
+                pprint.pformat(
+                    {
+                        'pattern': pattern,
+                        'results': results.group(0),
+                    }, indent=2,
+                ),
+            )
+            if groups:
+                return results.groups()
+            elif group is not None:
+                return results.group(group)
+            else:
+                return results
 
 
 def apply_mixin(dct, key, func, *args, **kwargs):

pytube/compat.py

@@ -2,12 +2,14 @@
 """Applies in-place data mutations."""
 from __future__ import absolute_import
 
+import json
 import logging
 import pprint
 
 from pytube import cipher
 from pytube.compat import parse_qsl
 from pytube.compat import unquote
+from pytube.exceptions import LiveStreamError
 
 
 logger = logging.getLogger(__name__)
@@ -27,8 +29,14 @@ def apply_signature(config_args, fmt, js):
 
     """
     stream_manifest = config_args[fmt]
+    live_stream = json.loads(config_args['player_response']).get(
+        'playabilityStatus', {},
+    ).get('liveStreamability')
     for i, stream in enumerate(stream_manifest):
-        url = stream['url']
+        if 'url' in stream:
+            url = stream['url']
+        elif live_stream:
+            raise LiveStreamError('Video is currently being streamed live')
 
         if 'signature=' in url:
             # For certain videos, YouTube will just provide them pre-signed, in

pytube/compat.pyc

@@ -162,9 +162,23 @@ def order_by(self, attribute_name):
         :param str attribute_name:
             The name of the attribute to sort by.
         """
+        integer_attr_repr = {}
+        for stream in self.fmt_streams:
+            attr = getattr(stream, attribute_name)
+            if attr is None:
+                break
+            num = [char for char in attr if char.isdigit()]
+            integer_attr_repr[attr] = int(''.join(num)) if num else None
+
+        # if every attribute has an integer representation
+        if integer_attr_repr and all(integer_attr_repr.values()):
+            def key(s): return integer_attr_repr[getattr(s, attribute_name)]
+        else:
+            def key(s): return getattr(s, attribute_name)
+
         fmt_streams = sorted(
             self.fmt_streams,
-            key=lambda s: getattr(s, attribute_name),
+            key=key,
         )
         return StreamQuery(fmt_streams)
 
@@ -187,7 +201,7 @@ def asc(self):
     def get_by_itag(self, itag):
         """Get the corresponding :class:`Stream <Stream>` for a given itag.
 
-        :param str itag:
+        :param str int itag:
             YouTube format identifier code.
         :rtype: :class:`Stream <Stream>` or None
         :returns:
@@ -196,7 +210,7 @@ def get_by_itag(self, itag):
 
         """
         try:
-            return self.itag_index[itag]
+            return self.itag_index[int(itag)]
         except KeyError:
             pass