Update to use requests 1.0+

Update to require feedparser 5.1.3+

git-svn-id: http://svn.flexget.com/trunk@3274 3942dd89-8c5d-46d7-aeed-044bccf3e60c

flexget/plugins/input/trakt_list.py

@@ -103,7 +103,7 @@ def on_task_input(self, task, config):
 
         result = task.requests.get(url, data=json.dumps(auth))
         try:
-            data = task.requests.post(url, data=json.dumps(auth)).json
+            data = task.requests.post(url, data=json.dumps(auth)).json()
         except RequestException as e:
             raise PluginError('Could not retrieve list from trakt (%s)' % e.message)
 

flexget/plugins/services/pogcal_acquired.py

@@ -13,7 +13,7 @@
 
 log = logging.getLogger('pogcal_acquired')
 Base = versioned_base('pogcal_acquired', 0)
-session = requests.Session(config={'max_retries': 2})
+session = requests.Session(max_retries=2)
 
 
 class PogcalShow(Base):

flexget/utils/imdb.py

@@ -10,7 +10,8 @@
 log = logging.getLogger('utils.imdb')
 # IMDb delivers a version of the page which is unparsable to unknown (and some known) user agents, such as requests'
 # Spoof the old urllib user agent to keep results consistent
-requests = Session(headers={'User-Agent': 'Python-urllib/2.6'})
+requests = Session()
+requests.headers.update({'User-Agent': 'Python-urllib/2.6'})
 # give imdb a little break between requests (see: http://flexget.com/ticket/129#comment:1)
 requests.set_domain_delay('imdb.com', '3 seconds')
 

flexget/utils/requests.py

@@ -44,20 +44,36 @@ def set_unresponsive(url):
     unresponsive_hosts[host] = datetime.now()
 
 
+class FileAdapter(requests.adapters.BaseAdapter):
+    """Handles file:// URIs by passing them to urllib2"""
+    def send(self, request):
+        url = request.url
+        try:
+            raw = urllib2.urlopen(url)
+        except IOError as e:
+            msg = 'IOError openening file %s: %s' % (url, e)
+            log.error(msg)
+            raise RequestException(msg)
+        resp = requests.Response()
+        resp.raw = raw
+        resp.status_code = 200
+        resp.headers = requests.structures.CaseInsensitiveDict(raw.headers)
+        return resp
+
+    def close(self):
+        pass
+
 class Session(requests.Session):
     """Subclass of requests Session class which defines some of our own defaults, records unresponsive sites,
     and raises errors by default."""
 
-    def __init__(self, **kwargs):
+    def __init__(self, timeout=None, max_retries=None):
         """Set some defaults for our session if not explicitly defined."""
-        kwargs.setdefault('timeout', 15)
-        kwargs.setdefault('config', {}).setdefault('max_retries', 1)
-        kwargs.setdefault('prefetch', False)
-        # TODO: This is a temporary fix for requests not properly handling deflate encoding, can be removed when
-        # next version of requests is released (>0.9) See #1412
-        kwargs.setdefault('headers', {}).setdefault('Accept-Encoding', ', '.join(('identity', 'compress', 'gzip')))
-        requests.Session.__init__(self, **kwargs)
-        self.cookiejar = None
+        requests.Session.__init__(self)
+        self.timeout = timeout
+        self.stream = True
+        self.mount('file://', FileAdapter())
+        self.adapters['http://'].max_retries = max_retries or 1
         # Stores min intervals between requests for certain sites
         self.domain_delay = {}
 
@@ -85,20 +101,6 @@ def request(self, method, url, *args, **kwargs):
         Also raises errors getting the content by default.
         """
 
-        # Handle file URI scheme using urllib2, wrap in requests response object
-        if url.startswith('file://'):
-            try:
-                raw = urllib2.urlopen(url)
-            except IOError as e:
-                msg = 'IOError openening file %s: %s' % (url, e)
-                log.error(msg)
-                raise RequestException(msg)
-            resp = requests.Response()
-            resp.raw = raw
-            resp.status_code = 200
-            resp.headers = requests.structures.CaseInsensitiveDict(raw.headers)
-            return resp
-
         # Raise Timeout right away if site is known to timeout
         if is_unresponsive(url):
             raise requests.Timeout('Requests to this site are known to timeout.')
@@ -117,10 +119,8 @@ def request(self, method, url, *args, **kwargs):
                 domain_dict['next_req'] = datetime.now() + domain_dict['delay']
                 break
 
-        # Pop our custom keyword argument before calling super method
-        config = kwargs.pop('config', {})
-        config['danger_mode'] = kwargs.pop('raise_status', True)
-        kwargs['config'] = config
+        kwargs.setdefault('timeout', self.timeout)
+        raise_status = kwargs.pop('raise_status', True)
 
         try:
             result = requests.Session.request(self, method, url, *args, **kwargs)
@@ -129,6 +129,9 @@ def request(self, method, url, *args, **kwargs):
             set_unresponsive(url)
             raise
 
+        if raise_status:
+            result.raise_for_status()
+
         return result
 
 

pavement.py

@@ -15,9 +15,9 @@
 sys.path.insert(0, '')
 
 options = environment.options
-install_requires = ['FeedParser>=5.1.2', 'SQLAlchemy >=0.7, <0.7.99', 'PyYAML', 'BeautifulSoup>=3.2, <3.3',
+install_requires = ['FeedParser>=5.1.3', 'SQLAlchemy >=0.7, <0.7.99', 'PyYAML', 'BeautifulSoup>=3.2, <3.3',
                     'beautifulsoup4>=4.1, <4.2', 'html5lib>=0.11', 'PyRSS2Gen', 'pynzb', 'progressbar', 'jinja2',
-                    'flask', 'cherrypy', 'requests>=0.14, <0.15', 'python-dateutil!=2.0']
+                    'flask', 'cherrypy', 'requests>=1.0, <1.99', 'python-dateutil!=2.0']
 if sys.version_info < (2, 7):
     # argparse is part of the standard library in python 2.7+
     install_requires.append('argparse')

rtd-requirements.txt

@@ -1,5 +1,5 @@
 paver
-FeedParser>=5.1.2
+FeedParser>=5.1.3
 SQLAlchemy >=0.7, <0.7.99
 PyYAML
 BeautifulSoup>=3.2, <3.3
@@ -11,5 +11,5 @@ progressbar
 jinja2
 flask
 cherrypy
-requests>=0.14, <0.15
+requests>=1.0, <1.99
 python-dateutil!=2.0