mirror of
https://github.com/chubin/wttr.in
synced 2024-12-25 19:23:14 +00:00
bin/proxy.py clean up
This commit is contained in:
Igor Chubin
parent
bd10befae4
commit
5353cc3a17
1 changed files with 96 additions and 65 deletions
161
bin/proxy.py
161
bin/proxy.py
|
|
@ -1,30 +1,39 @@
|
|||
#vim: encoding=utf-8
|
||||
#vim: fileencoding=utf-8
|
||||
|
||||
import gevent
|
||||
from gevent.wsgi import WSGIServer
|
||||
from gevent.queue import Queue
|
||||
"""
|
||||
|
||||
The proxy server acts as a backend for the wttr.in service.
|
||||
|
||||
It caches the answers and handles various data sources transforming their
|
||||
answers into format supported by the wttr.in service.
|
||||
|
||||
"""
|
||||
|
||||
from gevent.pywsgi import WSGIServer
|
||||
from gevent.monkey import patch_all
|
||||
from gevent.subprocess import Popen, PIPE, STDOUT
|
||||
patch_all()
|
||||
|
||||
# pylint: disable=wrong-import-position,wrong-import-order
|
||||
import sys
|
||||
import os
|
||||
import time
|
||||
import json
|
||||
import glob
|
||||
|
||||
from flask import Flask, request, render_template, send_from_directory, Response
|
||||
app = Flask(__name__)
|
||||
|
||||
import requests
|
||||
# Disable InsecureRequestWarning "Unverified HTTPS request is being..."
|
||||
from requests.packages.urllib3.exceptions import InsecureRequestWarning, InsecurePlatformWarning, SNIMissingWarning
|
||||
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
|
||||
requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
|
||||
requests.packages.urllib3.disable_warnings(SNIMissingWarning)
|
||||
|
||||
import cyrtranslit
|
||||
|
||||
CACHEDIR = "api-cache"
|
||||
from flask import Flask, request
|
||||
APP = Flask(__name__)
|
||||
|
||||
|
||||
MYDIR = os.path.abspath(
|
||||
os.path.dirname(os.path.dirname('__file__')))
|
||||
sys.path.append("%s/lib/" % MYDIR)
|
||||
|
||||
from globals import PROXY_CACHEDIR, PROXY_HOST, PROXY_PORT
|
||||
from translations import PROXY_LANGS
|
||||
# pylint: enable=wrong-import-position
|
||||
|
||||
|
||||
|
||||
def load_translations():
|
||||
|
|
@ -33,8 +42,7 @@ def load_translations():
|
|||
"""
|
||||
translations = {}
|
||||
|
||||
langs = ['az', 'bs', 'ca', 'cy', 'eo', 'he', 'hr', 'hy', 'id', 'is', 'it', 'ja', 'kk', 'lv', 'mk', 'nb', 'nn', 'sl', 'uz']
|
||||
for f_name in langs:
|
||||
for f_name in PROXY_LANGS:
|
||||
f_name = 'share/translations/%s.txt' % f_name
|
||||
translation = {}
|
||||
lang = f_name.split('/')[-1].split('.', 1)[0]
|
||||
|
|
@ -55,116 +63,139 @@ def load_translations():
|
|||
TRANSLATIONS = load_translations()
|
||||
|
||||
|
||||
def find_srv_for_query( path, query ):
|
||||
def _find_srv_for_query(path, query): # pylint: disable=unused-argument
|
||||
return 'http://api.worldweatheronline.com/'
|
||||
|
||||
def load_content_and_headers( path, query ):
|
||||
timestamp = time.strftime( "%Y%m%d%H", time.localtime() )
|
||||
p = os.path.join( CACHEDIR, timestamp, path, query )
|
||||
def _load_content_and_headers(path, query):
|
||||
timestamp = time.strftime("%Y%m%d%H", time.localtime())
|
||||
cache_file = os.path.join(PROXY_CACHEDIR, timestamp, path, query)
|
||||
try:
|
||||
return open( p, 'r' ).read(), json.loads( open( p+".headers", 'r' ).read() )
|
||||
except Exception, e:
|
||||
return (open(cache_file, 'r').read(),
|
||||
json.loads(open(cache_file+".headers", 'r').read()))
|
||||
except IOError:
|
||||
return None, None
|
||||
|
||||
def save_content_and_headers( path, query, content, headers ):
|
||||
timestamp = time.strftime( "%Y%m%d%H", time.localtime() )
|
||||
p = os.path.join( CACHEDIR, timestamp, path, query )
|
||||
d = os.path.dirname( p )
|
||||
if not os.path.exists( d ): os.makedirs( d )
|
||||
open( p+".headers", 'w' ).write( json.dumps( headers ) )
|
||||
open( p, 'w' ).write( content )
|
||||
def _save_content_and_headers(path, query, content, headers):
|
||||
timestamp = time.strftime("%Y%m%d%H", time.localtime())
|
||||
cache_file = os.path.join(PROXY_CACHEDIR, timestamp, path, query)
|
||||
cache_dir = os.path.dirname(cache_file)
|
||||
if not os.path.exists(cache_dir):
|
||||
os.makedirs(cache_dir)
|
||||
open(cache_file + ".headers", 'w').write(json.dumps(headers))
|
||||
open(cache_file, 'w').write(content)
|
||||
|
||||
def translate(text, lang):
|
||||
"""
|
||||
Translate `text` into `lang`
|
||||
"""
|
||||
translated = TRANSLATIONS.get(lang, {}).get(text, text)
|
||||
if text.encode('utf-8') == translated:
|
||||
print "%s: %s" % (lang, text)
|
||||
return translated
|
||||
|
||||
def cyr(to_translate):
|
||||
"""
|
||||
Transliterate `to_translate` from latin into cyrillic
|
||||
"""
|
||||
return cyrtranslit.to_cyrillic(to_translate)
|
||||
|
||||
def patch_greek(original):
|
||||
def _patch_greek(original):
|
||||
return original.decode('utf-8').replace(u"Ηλιόλουστη/ο", u"Ηλιόλουστη").encode('utf-8')
|
||||
|
||||
def add_translations(content, lang):
|
||||
"""
|
||||
Add `lang` translation to `content` (JSON)
|
||||
returned by the data source
|
||||
"""
|
||||
languages_to_translate = TRANSLATIONS.keys()
|
||||
#print type(content['data']) #['current_condition'][0]['lang_xx'] = [{'value':'XXX'}]
|
||||
try:
|
||||
d = json.loads(content)
|
||||
except Exception as e:
|
||||
d = json.loads(content) # pylint: disable=invalid-name
|
||||
except ValueError as exception:
|
||||
print "---"
|
||||
print e
|
||||
print exception
|
||||
print "---"
|
||||
|
||||
try:
|
||||
weather_condition = d['data']['current_condition'][0]['weatherDesc'][0]['value']
|
||||
if lang in languages_to_translate:
|
||||
d['data']['current_condition'][0]['lang_%s' % lang] = [{'value':translate(weather_condition, lang)}]
|
||||
print translate(weather_condition, lang)
|
||||
d['data']['current_condition'][0]['lang_%s' % lang] = \
|
||||
[{'value': translate(weather_condition, lang)}]
|
||||
elif lang == 'sr':
|
||||
d['data']['current_condition'][0]['lang_%s' % lang] = [{'value':cyr(d['data']['current_condition'][0]['lang_%s' % lang][0]['value'].encode('utf-8'))}]
|
||||
d['data']['current_condition'][0]['lang_%s' % lang] = \
|
||||
[{'value': cyr(
|
||||
d['data']['current_condition'][0]['lang_%s' % lang][0]['value']\
|
||||
.encode('utf-8'))}]
|
||||
elif lang == 'el':
|
||||
d['data']['current_condition'][0]['lang_%s' % lang] = [{'value':patch_greek(d['data']['current_condition'][0]['lang_%s' % lang][0]['value'].encode('utf-8'))}]
|
||||
d['data']['current_condition'][0]['lang_%s' % lang] = \
|
||||
[{'value': _patch_greek(
|
||||
d['data']['current_condition'][0]['lang_%s' % lang][0]['value']\
|
||||
.encode('utf-8'))}]
|
||||
elif lang == 'sr-lat':
|
||||
d['data']['current_condition'][0]['lang_%s' % lang] = [{'value':d['data']['current_condition'][0]['lang_sr'][0]['value'].encode('utf-8')}]
|
||||
d['data']['current_condition'][0]['lang_%s' % lang] = \
|
||||
[{'value':d['data']['current_condition'][0]['lang_sr'][0]['value']\
|
||||
.encode('utf-8')}]
|
||||
|
||||
fixed_weather = []
|
||||
for w in d['data']['weather']:
|
||||
for w in d['data']['weather']: # pylint: disable=invalid-name
|
||||
fixed_hourly = []
|
||||
for h in w['hourly']:
|
||||
for h in w['hourly']: # pylint: disable=invalid-name
|
||||
weather_condition = h['weatherDesc'][0]['value']
|
||||
if lang in languages_to_translate:
|
||||
h['lang_%s' % lang] = [{'value':translate(weather_condition, lang)}]
|
||||
h['lang_%s' % lang] = \
|
||||
[{'value': translate(weather_condition, lang)}]
|
||||
elif lang == 'sr':
|
||||
h['lang_%s' % lang] = [{'value':cyr(h['lang_%s' % lang][0]['value'].encode('utf-8'))}]
|
||||
h['lang_%s' % lang] = \
|
||||
[{'value': cyr(h['lang_%s' % lang][0]['value'].encode('utf-8'))}]
|
||||
elif lang == 'el':
|
||||
h['lang_%s' % lang] = [{'value':patch_greek(h['lang_%s' % lang][0]['value'].encode('utf-8'))}]
|
||||
h['lang_%s' % lang] = \
|
||||
[{'value': _patch_greek(h['lang_%s' % lang][0]['value'].encode('utf-8'))}]
|
||||
elif lang == 'sr-lat':
|
||||
h['lang_%s' % lang] = [{'value':h['lang_sr'][0]['value'].encode('utf-8')}]
|
||||
h['lang_%s' % lang] = \
|
||||
[{'value': h['lang_sr'][0]['value'].encode('utf-8')}]
|
||||
fixed_hourly.append(h)
|
||||
w['hourly'] = fixed_hourly
|
||||
fixed_weather.append(w)
|
||||
d['data']['weather'] = fixed_weather
|
||||
d['data']['weather'] = fixed_weather
|
||||
|
||||
content = json.dumps(d)
|
||||
except Exception as e:
|
||||
print e
|
||||
except (IndexError, ValueError) as exception:
|
||||
print exception
|
||||
return content
|
||||
|
||||
@app.route("/<path:path>")
|
||||
@APP.route("/<path:path>")
|
||||
def proxy(path):
|
||||
"""
|
||||
Main proxy function. Handles incoming HTTP queries.
|
||||
"""
|
||||
|
||||
lang = request.args.get('lang', 'en')
|
||||
query_string = request.query_string
|
||||
query_string = query_string.replace('sr-lat', 'sr')
|
||||
content, headers = load_content_and_headers(path,query_string)
|
||||
content, headers = _load_content_and_headers(path, query_string)
|
||||
|
||||
if content is None:
|
||||
srv = find_srv_for_query(path, query_string)
|
||||
srv = _find_srv_for_query(path, query_string)
|
||||
url = '%s/%s?%s' % (srv, path, query_string)
|
||||
print url
|
||||
|
||||
attempts = 5
|
||||
while attempts:
|
||||
r = requests.get(url, timeout=10)
|
||||
response = requests.get(url, timeout=10)
|
||||
try:
|
||||
json.loads(r.content)
|
||||
json.loads(response.content)
|
||||
break
|
||||
except:
|
||||
except ValueError:
|
||||
attempts -= 1
|
||||
|
||||
headers = {}
|
||||
headers['Content-Type'] = r.headers['content-type']
|
||||
content = add_translations(r.content, lang)
|
||||
try:
|
||||
save_content_and_headers( path, query_string, content, headers )
|
||||
except Exception, e:
|
||||
print e
|
||||
headers['Content-Type'] = response.headers['content-type']
|
||||
content = add_translations(response.content, lang)
|
||||
_save_content_and_headers(path, query_string, content, headers)
|
||||
|
||||
return content, 200, headers
|
||||
|
||||
if __name__ == "__main__":
|
||||
#app.run(host='0.0.0.0', port=5001, debug=False)
|
||||
#app.debug = True
|
||||
server = WSGIServer(("", 5001), app)
|
||||
server.serve_forever()
|
||||
|
||||
SERVER = WSGIServer((PROXY_HOST, PROXY_PORT), APP)
|
||||
SERVER.serve_forever()
|
||||
|
|
|
|||
Loading…
Reference in a new issue