diff --git a/fetch.py b/fetch.py index fead0e5..d57da6d 100644 --- a/fetch.py +++ b/fetch.py @@ -56,6 +56,8 @@ class FetchSession(object): def fetch(self, url, head=False): """Fetch URL, reusing connection if possible.""" network_stats.set_category('scraper') + if isinstance(url, unicode): + url = url.encode('utf-8') host, port, ssl, uri = _parse_url(url) # Check if we can reuse existing connection @@ -489,6 +491,8 @@ def fetch_contents(url, head=False, proxy=None): retry_messages = ('Engines cannot retrieve results', 'Rate limit exceeded') def _fetch_contents(url, head = False, proxy=None): network_stats.set_category('scraper') + if isinstance(url, unicode): + url = url.encode('utf-8') host, port, ssl, uri = _parse_url(url) headers=[ 'Accept-Language: en-US,en;q=0.8',