From 5353cc3a17faf4ba0de8db7795f6ff42d3c3c569 Mon Sep 17 00:00:00 2001 From: Igor Chubin Date: Tue, 23 Oct 2018 20:04:25 +0200 Subject: [PATCH] bin/proxy.py clean up --- bin/proxy.py | 161 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 96 insertions(+), 65 deletions(-) diff --git a/bin/proxy.py b/bin/proxy.py index 01286ec..bfc9fe5 100644 --- a/bin/proxy.py +++ b/bin/proxy.py @@ -1,30 +1,39 @@ -#vim: encoding=utf-8 +#vim: fileencoding=utf-8 -import gevent -from gevent.wsgi import WSGIServer -from gevent.queue import Queue +""" + +The proxy server acts as a backend for the wttr.in service. + +It caches the answers and handles various data sources transforming their +answers into format supported by the wttr.in service. + +""" + +from gevent.pywsgi import WSGIServer from gevent.monkey import patch_all -from gevent.subprocess import Popen, PIPE, STDOUT patch_all() +# pylint: disable=wrong-import-position,wrong-import-order +import sys import os import time import json -import glob - -from flask import Flask, request, render_template, send_from_directory, Response -app = Flask(__name__) import requests -# Disable InsecureRequestWarning "Unverified HTTPS request is being..." -from requests.packages.urllib3.exceptions import InsecureRequestWarning, InsecurePlatformWarning, SNIMissingWarning -requests.packages.urllib3.disable_warnings(InsecureRequestWarning) -requests.packages.urllib3.disable_warnings(InsecurePlatformWarning) -requests.packages.urllib3.disable_warnings(SNIMissingWarning) - import cyrtranslit -CACHEDIR = "api-cache" +from flask import Flask, request +APP = Flask(__name__) + + +MYDIR = os.path.abspath( + os.path.dirname(os.path.dirname('__file__'))) +sys.path.append("%s/lib/" % MYDIR) + +from globals import PROXY_CACHEDIR, PROXY_HOST, PROXY_PORT +from translations import PROXY_LANGS +# pylint: enable=wrong-import-position + def load_translations(): @@ -33,8 +42,7 @@ def load_translations(): """ translations = {} - langs = ['az', 'bs', 'ca', 'cy', 'eo', 'he', 'hr', 'hy', 'id', 'is', 'it', 'ja', 'kk', 'lv', 'mk', 'nb', 'nn', 'sl', 'uz'] - for f_name in langs: + for f_name in PROXY_LANGS: f_name = 'share/translations/%s.txt' % f_name translation = {} lang = f_name.split('/')[-1].split('.', 1)[0] @@ -55,116 +63,139 @@ def load_translations(): TRANSLATIONS = load_translations() -def find_srv_for_query( path, query ): +def _find_srv_for_query(path, query): # pylint: disable=unused-argument return 'http://api.worldweatheronline.com/' -def load_content_and_headers( path, query ): - timestamp = time.strftime( "%Y%m%d%H", time.localtime() ) - p = os.path.join( CACHEDIR, timestamp, path, query ) +def _load_content_and_headers(path, query): + timestamp = time.strftime("%Y%m%d%H", time.localtime()) + cache_file = os.path.join(PROXY_CACHEDIR, timestamp, path, query) try: - return open( p, 'r' ).read(), json.loads( open( p+".headers", 'r' ).read() ) - except Exception, e: + return (open(cache_file, 'r').read(), + json.loads(open(cache_file+".headers", 'r').read())) + except IOError: return None, None -def save_content_and_headers( path, query, content, headers ): - timestamp = time.strftime( "%Y%m%d%H", time.localtime() ) - p = os.path.join( CACHEDIR, timestamp, path, query ) - d = os.path.dirname( p ) - if not os.path.exists( d ): os.makedirs( d ) - open( p+".headers", 'w' ).write( json.dumps( headers ) ) - open( p, 'w' ).write( content ) +def _save_content_and_headers(path, query, content, headers): + timestamp = time.strftime("%Y%m%d%H", time.localtime()) + cache_file = os.path.join(PROXY_CACHEDIR, timestamp, path, query) + cache_dir = os.path.dirname(cache_file) + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + open(cache_file + ".headers", 'w').write(json.dumps(headers)) + open(cache_file, 'w').write(content) def translate(text, lang): + """ + Translate `text` into `lang` + """ translated = TRANSLATIONS.get(lang, {}).get(text, text) if text.encode('utf-8') == translated: print "%s: %s" % (lang, text) return translated def cyr(to_translate): + """ + Transliterate `to_translate` from latin into cyrillic + """ return cyrtranslit.to_cyrillic(to_translate) -def patch_greek(original): +def _patch_greek(original): return original.decode('utf-8').replace(u"Ηλιόλουστη/ο", u"Ηλιόλουστη").encode('utf-8') def add_translations(content, lang): + """ + Add `lang` translation to `content` (JSON) + returned by the data source + """ languages_to_translate = TRANSLATIONS.keys() - #print type(content['data']) #['current_condition'][0]['lang_xx'] = [{'value':'XXX'}] try: - d = json.loads(content) - except Exception as e: + d = json.loads(content) # pylint: disable=invalid-name + except ValueError as exception: print "---" - print e + print exception print "---" try: weather_condition = d['data']['current_condition'][0]['weatherDesc'][0]['value'] if lang in languages_to_translate: - d['data']['current_condition'][0]['lang_%s' % lang] = [{'value':translate(weather_condition, lang)}] - print translate(weather_condition, lang) + d['data']['current_condition'][0]['lang_%s' % lang] = \ + [{'value': translate(weather_condition, lang)}] elif lang == 'sr': - d['data']['current_condition'][0]['lang_%s' % lang] = [{'value':cyr(d['data']['current_condition'][0]['lang_%s' % lang][0]['value'].encode('utf-8'))}] + d['data']['current_condition'][0]['lang_%s' % lang] = \ + [{'value': cyr( + d['data']['current_condition'][0]['lang_%s' % lang][0]['value']\ + .encode('utf-8'))}] elif lang == 'el': - d['data']['current_condition'][0]['lang_%s' % lang] = [{'value':patch_greek(d['data']['current_condition'][0]['lang_%s' % lang][0]['value'].encode('utf-8'))}] + d['data']['current_condition'][0]['lang_%s' % lang] = \ + [{'value': _patch_greek( + d['data']['current_condition'][0]['lang_%s' % lang][0]['value']\ + .encode('utf-8'))}] elif lang == 'sr-lat': - d['data']['current_condition'][0]['lang_%s' % lang] = [{'value':d['data']['current_condition'][0]['lang_sr'][0]['value'].encode('utf-8')}] + d['data']['current_condition'][0]['lang_%s' % lang] = \ + [{'value':d['data']['current_condition'][0]['lang_sr'][0]['value']\ + .encode('utf-8')}] fixed_weather = [] - for w in d['data']['weather']: + for w in d['data']['weather']: # pylint: disable=invalid-name fixed_hourly = [] - for h in w['hourly']: + for h in w['hourly']: # pylint: disable=invalid-name weather_condition = h['weatherDesc'][0]['value'] if lang in languages_to_translate: - h['lang_%s' % lang] = [{'value':translate(weather_condition, lang)}] + h['lang_%s' % lang] = \ + [{'value': translate(weather_condition, lang)}] elif lang == 'sr': - h['lang_%s' % lang] = [{'value':cyr(h['lang_%s' % lang][0]['value'].encode('utf-8'))}] + h['lang_%s' % lang] = \ + [{'value': cyr(h['lang_%s' % lang][0]['value'].encode('utf-8'))}] elif lang == 'el': - h['lang_%s' % lang] = [{'value':patch_greek(h['lang_%s' % lang][0]['value'].encode('utf-8'))}] + h['lang_%s' % lang] = \ + [{'value': _patch_greek(h['lang_%s' % lang][0]['value'].encode('utf-8'))}] elif lang == 'sr-lat': - h['lang_%s' % lang] = [{'value':h['lang_sr'][0]['value'].encode('utf-8')}] + h['lang_%s' % lang] = \ + [{'value': h['lang_sr'][0]['value'].encode('utf-8')}] fixed_hourly.append(h) w['hourly'] = fixed_hourly fixed_weather.append(w) - d['data']['weather'] = fixed_weather + d['data']['weather'] = fixed_weather content = json.dumps(d) - except Exception as e: - print e + except (IndexError, ValueError) as exception: + print exception return content -@app.route("/") +@APP.route("/") def proxy(path): + """ + Main proxy function. Handles incoming HTTP queries. + """ + lang = request.args.get('lang', 'en') query_string = request.query_string query_string = query_string.replace('sr-lat', 'sr') - content, headers = load_content_and_headers(path,query_string) + content, headers = _load_content_and_headers(path, query_string) if content is None: - srv = find_srv_for_query(path, query_string) + srv = _find_srv_for_query(path, query_string) url = '%s/%s?%s' % (srv, path, query_string) print url attempts = 5 while attempts: - r = requests.get(url, timeout=10) + response = requests.get(url, timeout=10) try: - json.loads(r.content) + json.loads(response.content) break - except: + except ValueError: attempts -= 1 headers = {} - headers['Content-Type'] = r.headers['content-type'] - content = add_translations(r.content, lang) - try: - save_content_and_headers( path, query_string, content, headers ) - except Exception, e: - print e + headers['Content-Type'] = response.headers['content-type'] + content = add_translations(response.content, lang) + _save_content_and_headers(path, query_string, content, headers) return content, 200, headers if __name__ == "__main__": #app.run(host='0.0.0.0', port=5001, debug=False) #app.debug = True - server = WSGIServer(("", 5001), app) - server.serve_forever() - + SERVER = WSGIServer((PROXY_HOST, PROXY_PORT), APP) + SERVER.serve_forever()