Example webproxy.pyΒΆ

  1#!/usr/bin/env python
  2"""A web application that retrieves other websites for you.
  3
  4To start serving the application on port 8088, type
  5
  6  python webproxy.py
  7
  8To start the server on some other interface/port, use
  9
 10  python -m gevent.wsgi -p 8000 -i 0.0.0.0 webproxy.py
 11
 12"""
 13from __future__ import print_function
 14from gevent import monkey; monkey.patch_all()
 15import sys
 16import re
 17import traceback
 18
 19try:
 20    from cgi import escape
 21except ImportError:
 22    # Python 3.8 removed this API
 23    from html import escape
 24
 25try:
 26    import urllib2
 27    from urlparse import urlparse
 28    from urllib import unquote
 29except ImportError:
 30    # pylint:disable=import-error,no-name-in-module
 31    from urllib import request as urllib2
 32    from urllib.parse import urlparse
 33    from urllib.parse import unquote
 34
 35LISTEN = ('127.0.0.1', 8088)
 36
 37
 38def _as_bytes(s):
 39    if not isinstance(s, bytes): # Py3
 40        s = s.encode('utf-8')
 41    return s
 42
 43
 44def _as_str(s):
 45    if not isinstance(s, str): # Py3
 46        s = s.decode('latin-1')
 47    return s
 48
 49
 50def application(env, start_response):
 51    proxy_url = 'http://%s/' % env['HTTP_HOST']
 52    method = env['REQUEST_METHOD']
 53    path = env['PATH_INFO']
 54    if env['QUERY_STRING']:
 55        path += '?' + env['QUERY_STRING']
 56    path = path.lstrip('/')
 57
 58    if (method, path) == ('GET', ''):
 59        start_response('200 OK', [('Content-Type', 'text/html')])
 60        return [FORM]
 61
 62    if method == 'GET':
 63        return proxy(path, start_response, proxy_url)
 64
 65    if (method, path) == ('POST', ''):
 66        key, value = env['wsgi.input'].read().strip().split(b'=')
 67        assert key == b'url', repr(key)
 68        value = _as_str(value)
 69        start_response('302 Found', [('Location', _as_str(join(proxy_url, unquote(value))))])
 70    elif method == 'POST':
 71        start_response('404 Not Found', [])
 72    else:
 73        start_response('501 Not Implemented', [])
 74    return []
 75
 76
 77def proxy(path, start_response, proxy_url):
 78    # pylint:disable=too-many-locals
 79    if '://' not in path:
 80        path = 'http://' + path
 81
 82    try:
 83        try:
 84            response = urllib2.urlopen(path)
 85        except urllib2.HTTPError as ex:
 86            response = ex
 87        print('%s: %s %s' % (path, response.code, response.msg))
 88        # Beginning in Python 3.8, headers aren't guaranteed to arrive in
 89        # lowercase; we must do so ourself.
 90        headers = [(k, v) for (k, v) in response.headers.items() if k.lower() not in DROP_HEADERS]
 91        scheme, netloc, path, _params, _query, _fragment = urlparse(path)
 92        host = (scheme or 'http') + '://' + netloc
 93    except Exception as ex: # pylint:disable=broad-except
 94        sys.stderr.write('error while reading %s:\n' % path)
 95        traceback.print_exc()
 96        tb = traceback.format_exc()
 97        start_response('502 Bad Gateway', [('Content-Type', 'text/html')])
 98        # pylint:disable=deprecated-method
 99        error_str = escape(str(ex) or ex.__class__.__name__ or 'Error')
100        error_str = '<h1>%s</h1><h2>%s</h2><pre>%s</pre>' % (error_str, escape(path), escape(tb))
101        return [_as_bytes(error_str)]
102    else:
103        print("Returning", headers)
104        start_response('%s %s' % (response.code, response.msg), headers)
105        data = response.read()
106        data = fix_links(data, proxy_url, host)
107        return [data]
108
109
110def join(url1, *rest):
111    if not rest:
112        return url1
113    url2, rest = rest[0], rest[1:]
114    url1 = _as_bytes(url1)
115    url2 = _as_bytes(url2)
116    if url1.endswith(b'/'):
117        if url2.startswith(b'/'):
118            return join(url1 + url2[1:], *rest)
119        return join(url1 + url2, *rest)
120
121    if url2.startswith(b'/'):
122        return join(url1 + url2, *rest)
123
124    return join(url1 + b'/' + url2, *rest)
125
126
127def fix_links(data, proxy_url, host_url):
128    """
129    >>> fix_links("><img src=images/hp0.gif width=158", 'http://127.0.0.1:8088', 'www.google.com')
130    '><img src="http://127.0.0.1:8088/www.google.com/images/hp0.gif" width=158'
131    """
132    def fix_link_cb(m):
133        url = m.group('url')
134        if b'://' in url:
135            result = m.group('before') + b'"' + join(proxy_url, url) + b'"'
136        else:
137            result = m.group('before') + b'"' + join(proxy_url, host_url, url) + b'"'
138        #print('replaced %r -> %r' % (m.group(0), result))
139        return result
140    data = _link_re_1.sub(fix_link_cb, data)
141    data = _link_re_2.sub(fix_link_cb, data)
142    return data
143
144_link_re_1 = re.compile(br'''(?P<before>(href|src|action)\s*=\s*)(?P<quote>['"])(?P<url>[^#].*?)(?P=quote)''')
145_link_re_2 = re.compile(br'''(?P<before>(href|src|action)\s*=\s*)(?P<url>[^'"#>][^ >]*)''')
146
147# The lowercase names of headers that we will *NOT* forward.
148DROP_HEADERS = {
149    'transfer-encoding',
150    'set-cookie'
151}
152
153FORM = b"""<html><head>
154<title>Web Proxy - gevent example</title></head><body>
155<table width=60% height=100% align=center>
156<tr height=30%><td align=center valign=bottom>Type in URL you want to visit and press Enter</td></tr>
157<tr><td align=center valign=top>
158<form action=/ method=post>
159<input size=80 name=url value="http://www.gevent.org"/>
160</form>
161</td></tr>
162</table></body></table>
163"""
164
165if __name__ == '__main__':
166    from gevent.pywsgi import WSGIServer
167    print('Serving on %s...' % (LISTEN,))
168    WSGIServer(LISTEN, application).serve_forever()

Current source

Next page: Example webpy.py

Navigation