#!/usr/bin/env python2 # -*- coding: utf-8 -*- """Logging and utility functions.""" import time import sys # Log levels: lower number = more verbose LOG_LEVELS = { 'debug': 0, 'info': 1, 'rate': 1, # rate limiting info, same as info 'scraper': 1, # scraper info, same as info 'stats': 1, # statistics, same as info 'warn': 2, 'error': 3, 'none': 99, # suppress all } # Failure categories for proxy errors FAIL_TIMEOUT = 'timeout' FAIL_REFUSED = 'refused' FAIL_AUTH = 'auth' FAIL_UNREACHABLE = 'unreachable' FAIL_DNS = 'dns' FAIL_SSL = 'ssl' FAIL_CLOSED = 'closed' FAIL_PROXY = 'proxy' FAIL_OTHER = 'other' # SSL errors - proxy is actively intercepting (still working for MITM proxies) SSL_ERRORS = frozenset({FAIL_SSL}) # Connection errors - proxy might be dead, need secondary verification CONN_ERRORS = frozenset({FAIL_TIMEOUT, FAIL_REFUSED, FAIL_UNREACHABLE, FAIL_CLOSED, FAIL_DNS}) # Levels that go to stderr STDERR_LEVELS = ('warn', 'error') # Default log level (info) _log_level = 1 def set_log_level(level): """Set minimum log level. Messages below this level are suppressed.""" global _log_level if isinstance(level, int): _log_level = level else: _log_level = LOG_LEVELS.get(level, 1) def get_log_level(): """Get current log level.""" return _log_level def _log(msg, level='info'): """Log a message if it meets the current log level threshold. Args: msg: Message to log level: Log level (debug, info, warn, error) """ msg_level = LOG_LEVELS.get(level, 1) if msg_level < _log_level: return output = sys.stderr if level in STDERR_LEVELS else sys.stdout print >> output, '\r%s/%s\t%s' % (timestamp(), level, msg) def timestamp(): """Return formatted timestamp.""" return time.strftime('%H:%M:%S', time.gmtime()) def categorize_error(exc): """Categorize a RocksockException into failure type. Args: exc: RocksockException instance Returns: One of FAIL_* constants """ # Import here to avoid circular dependency import rocksock if not hasattr(exc, 'get_error'): return FAIL_OTHER error = exc.get_error() errortype = exc.get_errortype() # DNS resolution failures if errortype == rocksock.RS_ET_GAI: return FAIL_DNS # SSL errors if errortype == rocksock.RS_ET_SSL: return FAIL_SSL if error in (rocksock.RS_E_SSL_GENERIC, rocksock.RS_E_SSL_CERTIFICATE_ERROR): return FAIL_SSL # Timeout errors if error in (rocksock.RS_E_HIT_TIMEOUT, rocksock.RS_E_HIT_READTIMEOUT, rocksock.RS_E_HIT_WRITETIMEOUT, rocksock.RS_E_HIT_CONNECTTIMEOUT): return FAIL_TIMEOUT # Connection refused if error in (rocksock.RS_E_TARGET_CONN_REFUSED, rocksock.RS_E_TARGETPROXY_CONN_REFUSED): return FAIL_REFUSED # Authentication failures if error in (rocksock.RS_E_PROXY_AUTH_FAILED, rocksock.RS_E_SOCKS4_NOAUTH): return FAIL_AUTH # Unreachable if error in (rocksock.RS_E_TARGET_NET_UNREACHABLE, rocksock.RS_E_TARGETPROXY_NET_UNREACHABLE, rocksock.RS_E_TARGET_HOST_UNREACHABLE, rocksock.RS_E_TARGETPROXY_HOST_UNREACHABLE, rocksock.RS_E_TARGET_TTL_EXPIRED, rocksock.RS_E_TARGETPROXY_TTL_EXPIRED): return FAIL_UNREACHABLE # Connection closed if error == rocksock.RS_E_REMOTE_DISCONNECTED: return FAIL_CLOSED # Proxy-specific errors if error in (rocksock.RS_E_PROXY_UNEXPECTED_RESPONSE, rocksock.RS_E_TARGETPROXY_CONNECT_FAILED, rocksock.RS_E_PROXY_GENERAL_FAILURE, rocksock.RS_E_PROXY_COMMAND_NOT_SUPPORTED, rocksock.RS_E_PROXY_ADDRESSTYPE_NOT_SUPPORTED): return FAIL_PROXY return FAIL_OTHER