fetch.py

@@ -56,6 +56,8 @@ class FetchSession(object):
     def fetch(self, url, head=False):
         """Fetch URL, reusing connection if possible."""
         network_stats.set_category('scraper')
+        if isinstance(url, unicode):
+            url = url.encode('utf-8')
         host, port, ssl, uri = _parse_url(url)
 
         # Check if we can reuse existing connection
@@ -489,6 +491,8 @@ def fetch_contents(url, head=False, proxy=None):
 retry_messages = ('Engines cannot retrieve results', 'Rate limit exceeded')
 def _fetch_contents(url, head = False, proxy=None):
     network_stats.set_category('scraper')
+    if isinstance(url, unicode):
+        url = url.encode('utf-8')
     host, port, ssl, uri = _parse_url(url)
     headers=[
         'Accept-Language: en-US,en;q=0.8',