Mirrors/wttr.in
2
0
Fork 0
mirror of https://github.com/chubin/wttr.in synced 2025-01-13 20:49:03 +00:00
Code Issues Projects Releases Packages Wiki Activity

bin/proxy.py clean up

Browse source
This commit is contained in:
Igor Chubin 2018-10-23 20:04:25 +02:00
parent bd10befae4
commit 5353cc3a17
1 changed files with 96 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

161
bin/proxy.py
View file

@ -1,30 +1,39 @@
#vim: encoding=utf-8 #vim: fileencoding=utf-8
import gevent """
from gevent.wsgi import WSGIServer
from gevent.queue import Queue The proxy server acts as a backend for the wttr.in service.
It caches the answers and handles various data sources transforming their
answers into format supported by the wttr.in service.
"""
from gevent.pywsgi import WSGIServer
from gevent.monkey import patch_all from gevent.monkey import patch_all
from gevent.subprocess import Popen, PIPE, STDOUT
patch_all() patch_all()
# pylint: disable=wrong-import-position,wrong-import-order
import sys
import os import os
import time import time
import json import json
import glob
from flask import Flask, request, render_template, send_from_directory, Response
app = Flask(__name__)
import requests import requests
# Disable InsecureRequestWarning "Unverified HTTPS request is being..."
from requests.packages.urllib3.exceptions import InsecureRequestWarning, InsecurePlatformWarning, SNIMissingWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
requests.packages.urllib3.disable_warnings(SNIMissingWarning)
import cyrtranslit import cyrtranslit
CACHEDIR = "api-cache" from flask import Flask, request
APP = Flask(__name__)
MYDIR = os.path.abspath(
os.path.dirname(os.path.dirname('__file__')))
sys.path.append("%s/lib/" % MYDIR)
from globals import PROXY_CACHEDIR, PROXY_HOST, PROXY_PORT
from translations import PROXY_LANGS
# pylint: enable=wrong-import-position
def load_translations(): def load_translations():
@ -33,8 +42,7 @@ def load_translations():
""" """
translations = {} translations = {}
langs = ['az', 'bs', 'ca', 'cy', 'eo', 'he', 'hr', 'hy', 'id', 'is', 'it', 'ja', 'kk', 'lv', 'mk', 'nb', 'nn', 'sl', 'uz'] for f_name in PROXY_LANGS:
for f_name in langs:
f_name = 'share/translations/%s.txt' % f_name f_name = 'share/translations/%s.txt' % f_name
translation = {} translation = {}
lang = f_name.split('/')[-1].split('.', 1)[0] lang = f_name.split('/')[-1].split('.', 1)[0]
@ -55,116 +63,139 @@ def load_translations():
TRANSLATIONS = load_translations() TRANSLATIONS = load_translations()
def find_srv_for_query( path, query ): def _find_srv_for_query(path, query): # pylint: disable=unused-argument
return 'http://api.worldweatheronline.com/' return 'http://api.worldweatheronline.com/'
def load_content_and_headers( path, query ): def _load_content_and_headers(path, query):
timestamp = time.strftime( "%Y%m%d%H", time.localtime() ) timestamp = time.strftime("%Y%m%d%H", time.localtime())
p = os.path.join( CACHEDIR, timestamp, path, query ) cache_file = os.path.join(PROXY_CACHEDIR, timestamp, path, query)
try: try:
return open( p, 'r' ).read(), json.loads( open( p+".headers", 'r' ).read() ) return (open(cache_file, 'r').read(),
except Exception, e: json.loads(open(cache_file+".headers", 'r').read()))
except IOError:
return None, None return None, None
def save_content_and_headers( path, query, content, headers ): def _save_content_and_headers(path, query, content, headers):
timestamp = time.strftime( "%Y%m%d%H", time.localtime() ) timestamp = time.strftime("%Y%m%d%H", time.localtime())
p = os.path.join( CACHEDIR, timestamp, path, query ) cache_file = os.path.join(PROXY_CACHEDIR, timestamp, path, query)
d = os.path.dirname( p ) cache_dir = os.path.dirname(cache_file)
if not os.path.exists( d ): os.makedirs( d ) if not os.path.exists(cache_dir):
open( p+".headers", 'w' ).write( json.dumps( headers ) ) os.makedirs(cache_dir)
open( p, 'w' ).write( content ) open(cache_file + ".headers", 'w').write(json.dumps(headers))
open(cache_file, 'w').write(content)
def translate(text, lang): def translate(text, lang):
"""
Translate `text` into `lang`
"""
translated = TRANSLATIONS.get(lang, {}).get(text, text) translated = TRANSLATIONS.get(lang, {}).get(text, text)
if text.encode('utf-8') == translated: if text.encode('utf-8') == translated:
print "%s: %s" % (lang, text) print "%s: %s" % (lang, text)
return translated return translated
def cyr(to_translate): def cyr(to_translate):
"""
Transliterate `to_translate` from latin into cyrillic
"""
return cyrtranslit.to_cyrillic(to_translate) return cyrtranslit.to_cyrillic(to_translate)
def patch_greek(original): def _patch_greek(original):
return original.decode('utf-8').replace(u"Ηλιόλουστη/ο", u"Ηλιόλουστη").encode('utf-8') return original.decode('utf-8').replace(u"Ηλιόλουστη/ο", u"Ηλιόλουστη").encode('utf-8')
def add_translations(content, lang): def add_translations(content, lang):
"""
Add `lang` translation to `content` (JSON)
returned by the data source
"""
languages_to_translate = TRANSLATIONS.keys() languages_to_translate = TRANSLATIONS.keys()
#print type(content['data']) #['current_condition'][0]['lang_xx'] = [{'value':'XXX'}]
try: try:
d = json.loads(content) d = json.loads(content) # pylint: disable=invalid-name
except Exception as e: except ValueError as exception:
print "---" print "---"
print e print exception
print "---" print "---"
try: try:
weather_condition = d['data']['current_condition'][0]['weatherDesc'][0]['value'] weather_condition = d['data']['current_condition'][0]['weatherDesc'][0]['value']
if lang in languages_to_translate: if lang in languages_to_translate:
d['data']['current_condition'][0]['lang_%s' % lang] = [{'value':translate(weather_condition, lang)}] d['data']['current_condition'][0]['lang_%s' % lang] = \
print translate(weather_condition, lang) [{'value': translate(weather_condition, lang)}]
elif lang == 'sr': elif lang == 'sr':
d['data']['current_condition'][0]['lang_%s' % lang] = [{'value':cyr(d['data']['current_condition'][0]['lang_%s' % lang][0]['value'].encode('utf-8'))}] d['data']['current_condition'][0]['lang_%s' % lang] = \
[{'value': cyr(
d['data']['current_condition'][0]['lang_%s' % lang][0]['value']\
.encode('utf-8'))}]
elif lang == 'el': elif lang == 'el':
d['data']['current_condition'][0]['lang_%s' % lang] = [{'value':patch_greek(d['data']['current_condition'][0]['lang_%s' % lang][0]['value'].encode('utf-8'))}] d['data']['current_condition'][0]['lang_%s' % lang] = \
[{'value': _patch_greek(
d['data']['current_condition'][0]['lang_%s' % lang][0]['value']\
.encode('utf-8'))}]
elif lang == 'sr-lat': elif lang == 'sr-lat':
d['data']['current_condition'][0]['lang_%s' % lang] = [{'value':d['data']['current_condition'][0]['lang_sr'][0]['value'].encode('utf-8')}] d['data']['current_condition'][0]['lang_%s' % lang] = \
[{'value':d['data']['current_condition'][0]['lang_sr'][0]['value']\
.encode('utf-8')}]
fixed_weather = [] fixed_weather = []
for w in d['data']['weather']: for w in d['data']['weather']: # pylint: disable=invalid-name
fixed_hourly = [] fixed_hourly = []
for h in w['hourly']: for h in w['hourly']: # pylint: disable=invalid-name
weather_condition = h['weatherDesc'][0]['value'] weather_condition = h['weatherDesc'][0]['value']
if lang in languages_to_translate: if lang in languages_to_translate:
h['lang_%s' % lang] = [{'value':translate(weather_condition, lang)}] h['lang_%s' % lang] = \
[{'value': translate(weather_condition, lang)}]
elif lang == 'sr': elif lang == 'sr':
h['lang_%s' % lang] = [{'value':cyr(h['lang_%s' % lang][0]['value'].encode('utf-8'))}] h['lang_%s' % lang] = \
[{'value': cyr(h['lang_%s' % lang][0]['value'].encode('utf-8'))}]
elif lang == 'el': elif lang == 'el':
h['lang_%s' % lang] = [{'value':patch_greek(h['lang_%s' % lang][0]['value'].encode('utf-8'))}] h['lang_%s' % lang] = \
[{'value': _patch_greek(h['lang_%s' % lang][0]['value'].encode('utf-8'))}]
elif lang == 'sr-lat': elif lang == 'sr-lat':
h['lang_%s' % lang] = [{'value':h['lang_sr'][0]['value'].encode('utf-8')}] h['lang_%s' % lang] = \
[{'value': h['lang_sr'][0]['value'].encode('utf-8')}]
fixed_hourly.append(h) fixed_hourly.append(h)
w['hourly'] = fixed_hourly w['hourly'] = fixed_hourly
fixed_weather.append(w) fixed_weather.append(w)
d['data']['weather'] = fixed_weather d['data']['weather'] = fixed_weather
content = json.dumps(d) content = json.dumps(d)
except Exception as e: except (IndexError, ValueError) as exception:
print e print exception
return content return content
@app.route("/<path:path>") @APP.route("/<path:path>")
def proxy(path): def proxy(path):
"""
Main proxy function. Handles incoming HTTP queries.
"""
lang = request.args.get('lang', 'en') lang = request.args.get('lang', 'en')
query_string = request.query_string query_string = request.query_string
query_string = query_string.replace('sr-lat', 'sr') query_string = query_string.replace('sr-lat', 'sr')
content, headers = load_content_and_headers(path,query_string) content, headers = _load_content_and_headers(path, query_string)
if content is None: if content is None:
srv = find_srv_for_query(path, query_string) srv = _find_srv_for_query(path, query_string)
url = '%s/%s?%s' % (srv, path, query_string) url = '%s/%s?%s' % (srv, path, query_string)
print url print url
attempts = 5 attempts = 5
while attempts: while attempts:
r = requests.get(url, timeout=10) response = requests.get(url, timeout=10)
try: try:
json.loads(r.content) json.loads(response.content)
break break
except: except ValueError:
attempts -= 1 attempts -= 1
headers = {} headers = {}
headers['Content-Type'] = r.headers['content-type'] headers['Content-Type'] = response.headers['content-type']
content = add_translations(r.content, lang) content = add_translations(response.content, lang)
try: _save_content_and_headers(path, query_string, content, headers)
save_content_and_headers( path, query_string, content, headers )
except Exception, e:
print e
return content, 200, headers return content, 200, headers
if __name__ == "__main__": if __name__ == "__main__":
#app.run(host='0.0.0.0', port=5001, debug=False) #app.run(host='0.0.0.0', port=5001, debug=False)
#app.debug = True #app.debug = True
server = WSGIServer(("", 5001), app) SERVER = WSGIServer((PROXY_HOST, PROXY_PORT), APP)
server.serve_forever() SERVER.serve_forever()