diff --git a/http2.py b/http2.py index 5520b37..0b26b00 100644 --- a/http2.py +++ b/http2.py @@ -32,6 +32,11 @@ def _parse_url(url): ssl = False url = url[7:] port = 80 + elif url_l.startswith('//'): + # can happen with a redirect + ssl = False + url = url[2:] + port = -1 elif url_l.startswith('/'): # can happen with a redirect url = url[1:] @@ -45,7 +50,12 @@ def _parse_url(url): return "", 0, False, url port_index = -1 + fixed_amazon_redirect = False for i in range(len(url)): + if url[i] == '?': + if not fixed_amazon_redirect: + url = url.replace('?','/?',True) + fixed_amazon_redirect = True if url[i] == ':': host = url[:i] port_index = i+1 @@ -78,12 +88,14 @@ class RsHttp(): def __init__(self, host, port=80, ssl=False, follow_redirects=False, \ auto_set_cookies=False, keep_alive=False, timeout=60, \ user_agent=None, proxies=None, max_tries=10, log_errors=True, \ + verify_cert=False, **kwargs): self.host = host self.port = port self.use_ssl = ssl self.debugreq = False self.follow_redirects = follow_redirects + self.redirect_counter = 0 self.auto_set_cookies = auto_set_cookies self.keep_alive = keep_alive self.timeout = timeout @@ -93,6 +105,7 @@ class RsHttp(): self.max_tries = max_tries self.log_errors = log_errors self.last_rs_exception = None + self.verify_cert=verify_cert self.headers = [] def get_last_rocksock_exception(self): @@ -166,6 +179,15 @@ class RsHttp(): data = urllib.urlencode(values) return self._make_post_request_raw(url, data, extras) + def _try_gunzip(self, data): + try: + res = zlib.decompress(data, 16+zlib.MAX_WBITS) + return 0, res + except zlib.error as e: + if 'incomplete' in e.message: + return -1, '' + return -2, '' + def _get_response(self): def parse_header_fields(line): if not ':' in line: return line.rstrip(' '), "" @@ -181,7 +203,9 @@ class RsHttp(): s = '' res = '' #'HTTP/1.1 302 Found\r\n' - l = self.conn.recvline().strip() + l = '' + while not l.startswith('HTTP/'): + l = self.conn.recvline().strip() s = l + '\n' foo, code, msg = _parse_errorcode(l) while True: @@ -227,7 +251,13 @@ class RsHttp(): if len(res) != 0: if unzip == 'gzip': - res = zlib.decompress(res, 16+zlib.MAX_WBITS) + ec, extr = self._try_gunzip(res) + while ec == -1: + res += self.conn.recv(-1) + ec, extr = self._try_gunzip(res) + if ec == -2: + raise zlib.error + res = extr elif unzip == 'deflate': try: res = zlib.decompress(res) @@ -247,7 +277,7 @@ class RsHttp(): while tries < self.max_tries: tries += 1 try: - self.conn = Rocksock(host=self.host, port=self.port, proxies=self.proxies, ssl=self.use_ssl, timeout=self.timeout) + self.conn = Rocksock(host=self.host, port=self.port, proxies=self.proxies, ssl=self.use_ssl, timeout=self.timeout, verifycert=self.verify_cert) self.conn.connect() return True except RocksockException as e: @@ -314,15 +344,23 @@ class RsHttp(): hdr, res, redirect = self._send_and_recv(req) if redirect != '' and self.follow_redirects: + MAX_REDIRECTS = 16 + self.redirect_counter += 1 + if self.redirect_counter > MAX_REDIRECTS: + return '', '' + host, port, use_ssl, url = _parse_url(redirect) if port != 0: self.host = host - self.port = port - self.use_ssl = use_ssl + if port != -1: # -1: use existing port/ssl + self.port = port + self.use_ssl = use_ssl self.conn.disconnect() self.conn = None self.reconnect() return self.get(url, extras) + else: + self.redirect_counter = 0 return hdr, res @@ -371,6 +409,12 @@ class RsHttp(): # http.add_header("Referer: http://bbc.com") self.headers.append(s) + def add_headers(self, lines): + # copy a multi-line header chunk verbatim into each request: + for line in lines.split('\n'): + line = line.rstrip('\r') + if len(line): self.headers.append(line) + def set_cookie(self, c): if c.lower().startswith('set-cookie: '): c = c[len('Set-Cookie: '):] diff --git a/rocksock.py b/rocksock.py index 5483857..0ed0ffa 100644 --- a/rocksock.py +++ b/rocksock.py @@ -1,3 +1,23 @@ +# rocksock socket library routines for python. +# Copyright (C) 2018-2020 rofl0r + +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. + +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. + +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + +# you can find the full license text at +# https://www.gnu.org/licenses/old-licenses/lgpl-2.1.en.html + import socket, ssl, select, copy, errno # rs_proxyType @@ -45,7 +65,7 @@ RS_E_REMOTE_DISCONNECTED = 24 RS_E_NO_PROXYSTORAGE = 25 RS_E_HOSTNAME_TOO_LONG = 26 RS_E_INVALID_PROXY_URL = 27 - +RS_E_SSL_CERTIFICATE_ERROR = 28 class RocksockException(Exception): def __init__(self, error, failedproxy=None, errortype=RS_ET_OWN, *args, **kwargs): @@ -66,10 +86,10 @@ class RocksockException(Exception): def reraise(self): import sys ei = sys.exc_info() - raise ei[0], ei[1], ei[2] + raise(ei[0], ei[1], ei[2]) # import traceback, sys # traceback.print_exc(file=sys.stderr) -# raise self +# raise(self) def get_errormessage(self): errordict = { @@ -100,7 +120,8 @@ class RocksockException(Exception): RS_E_REMOTE_DISCONNECTED : "remote socket closed connection", RS_E_NO_PROXYSTORAGE : "no proxy storage assigned", RS_E_HOSTNAME_TOO_LONG : "hostname exceeds 255 chars", - RS_E_INVALID_PROXY_URL : "invalid proxy URL string" + RS_E_INVALID_PROXY_URL : "invalid proxy URL string", + RS_E_SSL_CERTIFICATE_ERROR : "certificate check error" } if self.errortype == RS_ET_SYS: if self.error in errno.errorcode: @@ -111,7 +132,7 @@ class RocksockException(Exception): msg = "GAI: " + self.failedproxy elif self.errortype == RS_ET_SSL: msg = errordict[self.error] - if self.error == RS_E_SSL_GENERIC and self.failedproxy != None: + if (self.error == RS_E_SSL_GENERIC or self.error == RS_E_SSL_CERTIFICATE_ERROR) and self.failedproxy != None: msg += ': ' + self.failedproxy #failedproxy is repurposed for SSL exceptions else: #RS_ET_OWN msg = errordict[self.error] + " (proxy %d)"%self.failedproxy @@ -121,7 +142,7 @@ class RocksockException(Exception): class RocksockHostinfo(): def __init__(self, host, port): if port < 0 or port > 65535: - raise RocksockException(RS_E_INVALID_PROXY_URL, failedproxy=-1) + raise(RocksockException(RS_E_INVALID_PROXY_URL, failedproxy=-1)) self.host = host self.port = port @@ -151,7 +172,7 @@ def resolve(hostinfo, want_v4=True): except socket.gaierror as e: eno, str = e.args - raise RocksockException(eno, str, errortype=RS_ET_GAI) + raise(RocksockException(eno, str, errortype=RS_ET_GAI)) return None, None @@ -164,7 +185,7 @@ class RocksockProxy(): 'http' : RS_PT_HTTP } self.type = typemap[type] if type in typemap else type if not self.type in [RS_PT_NONE, RS_PT_SOCKS4, RS_PT_SOCKS5, RS_PT_HTTP]: - raise ValueError('Invalid proxy type') + raise(ValueError('Invalid proxy type')) self.username = username self.password = password self.hostinfo = RocksockHostinfo(host, port) @@ -193,7 +214,10 @@ class Rocksock(): if 'ssl' in kwargs and kwargs['ssl'] == True: self.sslcontext = ssl.create_default_context() self.sslcontext.check_hostname = False - if not verifycert: self.sslcontext.verify_mode = ssl.CERT_NONE + self.sslcontext.verify_mode = ssl.CERT_NONE + if verifycert: + self.sslcontext.verify_mode = ssl.CERT_OPTIONAL + self.sslcontext.check_hostname = True else: self.sslcontext = None self.proxychain = [] @@ -225,21 +249,21 @@ class Rocksock(): try: x = af+1 except TypeError: - raise RocksockException(-3, "unexpected problem resolving DNS, try again", failedproxy=self._failed_proxy(0), errortype=RS_ET_GAI) -# print "GOT A WEIRD AF" -# print af -# raise RocksockException(-6666, af, errortype=RS_ET_GAI) + raise(RocksockException(-3, "unexpected problem resolving DNS, try again", failedproxy=self._failed_proxy(0), errortype=RS_ET_GAI)) +# print("GOT A WEIRD AF") +# print(af) +# raise(RocksockException(-6666, af, errortype=RS_ET_GAI)) self.sock = socket.socket(af, socket.SOCK_STREAM) self.sock.settimeout(None if self.timeout == 0 else self.timeout) try: self.sock.connect((sa[0], sa[1])) except socket.timeout: - raise RocksockException(RS_E_HIT_TIMEOUT, failedproxy=self._failed_proxy(0)) + raise(RocksockException(RS_E_HIT_TIMEOUT, failedproxy=self._failed_proxy(0))) except socket.error as e: - raise self._translate_socket_error(e, 0) + raise(self._translate_socket_error(e, 0)) - for pnum in xrange(1, len(self.proxychain)): + for pnum in range(1, len(self.proxychain)): curr = self.proxychain[pnum] prev = self.proxychain[pnum-1] self._connect_step(pnum) @@ -247,14 +271,17 @@ class Rocksock(): if self.sslcontext: try: self.sock = self.sslcontext.wrap_socket(self.sock, server_hostname=self.proxychain[len(self.proxychain)-1].hostinfo.host) + except ssl.CertificateError as e: + reason = self._get_ssl_exception_reason(e) + raise(RocksockException(RS_E_SSL_CERTIFICATE_ERROR, failedproxy=reason, errortype=RS_ET_SSL)) except ssl.SSLError as e: reason = self._get_ssl_exception_reason(e) #if hasattr(e, 'library'): subsystem = e.library - raise RocksockException(RS_E_SSL_GENERIC, failedproxy=reason, errortype=RS_ET_SSL) + raise(RocksockException(RS_E_SSL_GENERIC, failedproxy=reason, errortype=RS_ET_SSL)) except socket.error as e: - raise self._translate_socket_error(e, -1) + raise(self._translate_socket_error(e, -1)) except Exception as e: - raise e + raise(e) """ while True: try: @@ -281,11 +308,11 @@ class Rocksock(): def send(self, buf, pnum=-1): if self.sock is None: - raise RocksockException(RS_E_NO_SOCKET, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_NO_SOCKET, failedproxy=self._failed_proxy(pnum))) try: return self.sock.sendall(buf) except socket.error as e: - raise self._translate_socket_error(e, pnum) + raise(self._translate_socket_error(e, pnum)) def _get_ssl_exception_reason(self, e): s = '' @@ -302,17 +329,17 @@ class Rocksock(): if n >= 1024*1024: n = 1024*1024 chunk = self.sock.recv(n) except socket.timeout: - raise RocksockException(RS_E_HIT_TIMEOUT, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_HIT_TIMEOUT, failedproxy=self._failed_proxy(pnum))) except socket.error as e: - raise self._translate_socket_error(e, pnum) + raise(self._translate_socket_error(e, pnum)) except ssl.SSLError as e: s = self._get_ssl_exception_reason(e) if s == 'The read operation timed out': - raise RocksockException(RS_E_HIT_READTIMEOUT, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_HIT_READTIMEOUT, failedproxy=self._failed_proxy(pnum))) else: - raise RocksockException(RS_E_SSL_GENERIC, failedproxy=s, errortype=RS_ET_SSL) + raise(RocksockException(RS_E_SSL_GENERIC, failedproxy=s, errortype=RS_ET_SSL)) if len(chunk) == 0: - raise RocksockException(RS_E_REMOTE_DISCONNECTED, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_REMOTE_DISCONNECTED, failedproxy=self._failed_proxy(pnum))) data += chunk if count == -1: break else: count -= len(chunk) @@ -322,12 +349,9 @@ class Rocksock(): s = '' c = '\0' while c != '\n': - try: c = self.recv(1) if c == '': return s s += c - except Exception as e: - raise e return s def recvuntil(self, until): @@ -358,7 +382,7 @@ class Rocksock(): buf += '\0\0\0\x01' else: af, sa = resolve(dest.hostinfo, True) - if af != socket.AF_INET: raise RocksockException(RS_E_SOCKS4_NO_IP6, failedproxy=-1) + if af != socket.AF_INET: raise(RocksockException(RS_E_SOCKS4_NO_IP6, failedproxy=-1)) buf += self._ip_to_bytes(sa[0]) buf += '\0' if v4a: buf += dest.hostinfo.host + '\0' @@ -368,16 +392,16 @@ class Rocksock(): self.send(header) res = self.recv(8, pnum=pnum) if len(res) < 8 or ord(res[0]) != 0: - raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, failedproxy=self._failed_proxy(pnum))) ch = ord(res[1]) if ch == 0x5a: pass elif ch == 0x5b: - raise RocksockException(RS_E_TARGETPROXY_CONNECT_FAILED, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_TARGETPROXY_CONNECT_FAILED, failedproxy=self._failed_proxy(pnum))) elif ch == 0x5c or ch == 0x5d: return RocksockException(RS_E_PROXY_AUTH_FAILED, failedproxy=self._failed_proxy(pnum)) else: - raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, failedproxy=self._failed_proxy(pnum))) def _setup_socks5_header(self, proxy): buf = '\x05' @@ -391,9 +415,9 @@ class Rocksock(): self.send(header) res = self.recv(2, pnum=pnum) if len(res) != 2 or res[0] != '\x05': - raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, failedproxy=self._failed_proxy(pnum))) if res[1] == '\xff': - raise RocksockException(RS_E_PROXY_AUTH_FAILED, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_PROXY_AUTH_FAILED, failedproxy=self._failed_proxy(pnum))) if ord(res[1]) == 2: px = self.proxychain[pnum-1] @@ -402,8 +426,8 @@ class Rocksock(): self.send(pkt) res = self.recv(2, pnum=pnum) if len(res) < 2 or res[1] != '\0': - raise RocksockException(RS_E_PROXY_AUTH_FAILED, failedproxy=self._failed_proxy(pnum)) - else: raise RocksockException(RS_E_PROXY_AUTH_FAILED, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_PROXY_AUTH_FAILED, failedproxy=self._failed_proxy(pnum))) + else: raise(RocksockException(RS_E_PROXY_AUTH_FAILED, failedproxy=self._failed_proxy(pnum))) dst = self.proxychain[pnum] numeric = isnumericipv4(dst.hostinfo.host) if numeric: @@ -415,18 +439,18 @@ class Rocksock(): self.send(pkt) res = self.recv(pnum=pnum) if len(res) < 2 or res[0] != '\x05': - raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, failedproxy=self._failed_proxy(pnum))) ch = ord(res[1]) if ch == 0: pass - elif ch == 1: raise RocksockException(RS_E_PROXY_GENERAL_FAILURE, failedproxy=self._failed_proxy(pnum)) - elif ch == 2: raise RocksockException(RS_E_PROXY_AUTH_FAILED, failedproxy=self._failed_proxy(pnum)) - elif ch == 3: raise RocksockException(RS_E_TARGETPROXY_NET_UNREACHABLE, failedproxy=self._failed_proxy(pnum)) - elif ch == 4: raise RocksockException(RS_E_TARGETPROXY_HOST_UNREACHABLE, failedproxy=self._failed_proxy(pnum)) - elif ch == 5: raise RocksockException(RS_E_TARGETPROXY_CONN_REFUSED, failedproxy=self._failed_proxy(pnum)) - elif ch == 6: raise RocksockException(RS_E_TARGETPROXY_TTL_EXPIRED, failedproxy=self._failed_proxy(pnum)) - elif ch == 7: raise RocksockException(RS_E_PROXY_COMMAND_NOT_SUPPORTED, failedproxy=self._failed_proxy(pnum)) - elif ch == 8: raise RocksockException(RS_E_PROXY_ADDRESSTYPE_NOT_SUPPORTED, failedproxy=self._failed_proxy(pnum)) - else: raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, failedproxy=self._failed_proxy(pnum)) + elif ch == 1: raise(RocksockException(RS_E_PROXY_GENERAL_FAILURE, failedproxy=self._failed_proxy(pnum))) + elif ch == 2: raise(RocksockException(RS_E_PROXY_AUTH_FAILED, failedproxy=self._failed_proxy(pnum))) + elif ch == 3: raise(RocksockException(RS_E_TARGETPROXY_NET_UNREACHABLE, failedproxy=self._failed_proxy(pnum))) + elif ch == 4: raise(RocksockException(RS_E_TARGETPROXY_HOST_UNREACHABLE, failedproxy=self._failed_proxy(pnum))) + elif ch == 5: raise(RocksockException(RS_E_TARGETPROXY_CONN_REFUSED, failedproxy=self._failed_proxy(pnum))) + elif ch == 6: raise(RocksockException(RS_E_TARGETPROXY_TTL_EXPIRED, failedproxy=self._failed_proxy(pnum))) + elif ch == 7: raise(RocksockException(RS_E_PROXY_COMMAND_NOT_SUPPORTED, failedproxy=self._failed_proxy(pnum))) + elif ch == 8: raise(RocksockException(RS_E_PROXY_ADDRESSTYPE_NOT_SUPPORTED, failedproxy=self._failed_proxy(pnum))) + else: raise(RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, failedproxy=self._failed_proxy(pnum))) def _connect_step(self, pnum): @@ -440,7 +464,7 @@ class Rocksock(): if e.get_error() == RS_E_TARGETPROXY_CONNECT_FAILED: s4 = self._setup_socks4_header(False, curr) self._connect_socks4(s4a, pnum) - else: raise e + else: raise(e) elif prev.type == RS_PT_SOCKS5: s5 = self._setup_socks5_header(prev) self._connect_socks5(s5, pnum) @@ -449,9 +473,9 @@ class Rocksock(): self.send("CONNECT %s:%d HTTP/1.1\r\n\r\n"%(dest.hostinfo.host, dest.hostinfo.port)) resp = self.recv(pnum=pnum) if len(resp) <12: - raise RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_PROXY_UNEXPECTED_RESPONSE, failedproxy=self._failed_proxy(pnum))) if resp[9] != '2': - raise RocksockException(RS_E_TARGETPROXY_CONNECT_FAILED, failedproxy=self._failed_proxy(pnum)) + raise(RocksockException(RS_E_TARGETPROXY_CONNECT_FAILED, failedproxy=self._failed_proxy(pnum))) if __name__ == '__main__': @@ -466,12 +490,12 @@ if __name__ == '__main__': try: rs.connect() except RocksockException as e: - print e.get_errormessage() + print(e.get_errormessage()) e.reraise() rs.send('GET / HTTP/1.0\r\n\r\n') - print rs.recvline() + print(rs.recvline()) rs.disconnect() rs.connect() rs.send('GET / HTTP/1.0\r\n\r\n') - print rs.recvline() + print(rs.recvline())