From 71f4785d13aa51b2858f182cf2dd25bb93996661 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Fri, 21 Apr 2017 18:06:19 -0700 Subject: [PATCH 1/4] Initial Redislite implementation with limits --- .gitignore | 2 ++ titanembeds/app.py | 1 + titanembeds/database/__init__.py | 1 + titanembeds/database/custom_redislite.py | 25 ++++++++++++++++++++++++ 4 files changed, 29 insertions(+) create mode 100644 titanembeds/database/custom_redislite.py diff --git a/.gitignore b/.gitignore index 09526cb..591a53a 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,5 @@ ENV/ # Project specifc config.py +redislite.db +redislite.db.settings diff --git a/titanembeds/app.py b/titanembeds/app.py index 6d5b2d2..90a506c 100644 --- a/titanembeds/app.py +++ b/titanembeds/app.py @@ -15,6 +15,7 @@ app.config['SQLALCHEMY_DATABASE_URI'] = config['database-uri'] app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False # Suppress the warning/no need this on for now. app.config['RATELIMIT_HEADERS_ENABLED'] = True app.config['SQLALCHEMY_POOL_RECYCLE'] = 250 +app.config['RATELIMIT_STORAGE_URL'] = 'redislite://redislite.db' app.secret_key = config['app-secret'] db.init_app(app) diff --git a/titanembeds/database/__init__.py b/titanembeds/database/__init__.py index 1fb2b67..28951e6 100644 --- a/titanembeds/database/__init__.py +++ b/titanembeds/database/__init__.py @@ -5,3 +5,4 @@ from guilds import Guilds from unauthenticated_users import UnauthenticatedUsers from unauthenticated_bans import UnauthenticatedBans from authenticated_users import AuthenticatedUsers +from custom_redislite import LimitsRedisLite diff --git a/titanembeds/database/custom_redislite.py b/titanembeds/database/custom_redislite.py new file mode 100644 index 0000000..e79849c --- /dev/null +++ b/titanembeds/database/custom_redislite.py @@ -0,0 +1,25 @@ +import urlparse +from limits.storage import Storage +from redislite import Redis + +class LimitsRedisLite(Storage): # For Python Limits + STORAGE_SCHEME = "redislite" + def __init__(self, uri, **options): + self.redis_instance = Redis(urlparse.urlparse(uri).netloc) + + def check(self): + return True + + def get_expiry(self, key): + return self.redis_instance.ttl(key) + + def incr(self, key, expiry, elastic_expiry=False): + if self.redis_instance.exists(key): + self.redis_instance.set(key, int(self.redis_instance.get(key))+1) + else: + self.redis_instance.set(key, 1) + self.redis_instance.expire(key, expiry) + return + + def get(self, key): + return int(self.redis_instance.get(key)) From 52af78ade36b5f0b312d14412796b20ce058bea6 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Fri, 21 Apr 2017 21:18:14 -0700 Subject: [PATCH 2/4] Implement global caching for all workers --- .gitignore | 2 ++ titanembeds/app.py | 3 +-- titanembeds/blueprints/api/api.py | 2 +- titanembeds/discordrest.py | 12 +++++------- titanembeds/oauth.py | 2 +- titanembeds/utils.py | 14 +++++++++++--- tmp/.gitinclude | 0 7 files changed, 21 insertions(+), 14 deletions(-) create mode 100644 tmp/.gitinclude diff --git a/.gitignore b/.gitignore index 591a53a..1eb9de4 100644 --- a/.gitignore +++ b/.gitignore @@ -92,3 +92,5 @@ ENV/ config.py redislite.db redislite.db.settings +tmp/* +!tmp/.gitinclude diff --git a/titanembeds/app.py b/titanembeds/app.py index 90a506c..96976c5 100644 --- a/titanembeds/app.py +++ b/titanembeds/app.py @@ -2,7 +2,7 @@ from config import config from database import db from flask import Flask, render_template, request, session, url_for, redirect, jsonify from flask_sslify import SSLify -from titanembeds.utils import cache, rate_limiter +from titanembeds.utils import rate_limiter, cache import blueprints.api import blueprints.user import blueprints.embed @@ -19,7 +19,6 @@ app.config['RATELIMIT_STORAGE_URL'] = 'redislite://redislite.db' app.secret_key = config['app-secret'] db.init_app(app) -cache.init_app(app, config={'CACHE_TYPE': 'simple'}) rate_limiter.init_app(app) sslify = SSLify(app, permanent=True) diff --git a/titanembeds/blueprints/api/api.py b/titanembeds/blueprints/api/api.py index 554b678..698a401 100644 --- a/titanembeds/blueprints/api/api.py +++ b/titanembeds/blueprints/api/api.py @@ -114,7 +114,7 @@ def format_post_content(message): message = "**<{}#{}>** {}".format(session['username'], session['discriminator'], message) # I would like to do a @ mention, but i am worried about notif spam return message -@cache.cached(timeout=60, key_prefix=make_guildchannels_cache_key) +@cache.cache(make_guildchannels_cache_key, expires=60) def get_guild_channels(guild_id): if user_unauthenticated(): member_roles = [guild_id] #equivilant to @everyone role diff --git a/titanembeds/discordrest.py b/titanembeds/discordrest.py index c3d8237..5db8f32 100644 --- a/titanembeds/discordrest.py +++ b/titanembeds/discordrest.py @@ -3,11 +3,9 @@ import sys import time import json from functools import partial -from cachetools import cached, TTLCache -from cachetools.keys import hashkey +from titanembeds.utils import cache _DISCORD_API_BASE = "https://discordapp.com/api/v6" -cache = TTLCache(200, 200) def json_or_text(response): text = response.text @@ -122,7 +120,7 @@ class DiscordREST: r = self.request("GET", _endpoint) return r - @cached(cache, key=partial(hashkey, 'get_guild_member')) + @cache.cache('get_guild_member', expire=200) def get_guild_member(self, guild_id, user_id): _endpoint = "/guilds/{guild_id}/members/{user_id}".format(guild_id=guild_id, user_id=user_id) r = self.request("GET", _endpoint) @@ -160,7 +158,7 @@ class DiscordREST: r = self.request("GET", _endpoint) return r - @cached(cache, key=partial(hashkey, 'list_all_guild_members')) + @cache.cache('list_all_guild_members', expire=200) def list_all_guild_members(self, guild_id): _endpoint = "/guilds/{guild_id}/members".format(guild_id=guild_id) count = 1 @@ -183,7 +181,7 @@ class DiscordREST: # User ##################### - @cached(cache, key=partial(hashkey, 'get_all_guilds')) + @cache.cache('get_all_guilds', expire=100) def get_all_guilds(self): _endpoint = "/users/@me/guilds" params = {} @@ -206,7 +204,7 @@ class DiscordREST: # Widget Handler ##################### - @cached(cache, key=partial(hashkey, 'get_widget')) + @cache.cache('get_widget', expire=200) def get_widget(self, guild_id): _endpoint = _DISCORD_API_BASE + "/servers/{guild_id}/widget.json".format(guild_id=guild_id) embed = self.get_guild_embed(guild_id) diff --git a/titanembeds/oauth.py b/titanembeds/oauth.py index ae925c0..9bb1332 100644 --- a/titanembeds/oauth.py +++ b/titanembeds/oauth.py @@ -42,7 +42,7 @@ def get_current_authenticated_user(): def user_has_permission(permission, index): return bool((int(permission) >> index) & 1) -@cache.cached(timeout=120, key_prefix=make_guilds_cache_key) +@cache.cache(make_guilds_cache_key, expire=120) def get_user_guilds(): req = discordrest_from_user("/users/@me/guilds") return req diff --git a/titanembeds/utils.py b/titanembeds/utils.py index cfe9c93..6184876 100644 --- a/titanembeds/utils.py +++ b/titanembeds/utils.py @@ -1,15 +1,23 @@ +from beaker.cache import CacheManager +from beaker.util import parse_cache_config_options from titanembeds.database import db, Guilds -from titanembeds.discordrest import DiscordREST from flask import request, session -from flask.ext.cache import Cache from flask_limiter import Limiter from config import config import random import string import hashlib +cache_opts = { + 'cache.type': 'file', + 'cache.data_dir': 'tmp/cachedata', + 'cache.lock_dir': 'tmp/cachelock' +} +cache = CacheManager(**parse_cache_config_options(cache_opts)) + +from titanembeds.discordrest import DiscordREST + discord_api = DiscordREST(config['bot-token']) -cache = Cache() def get_client_ipaddr(): if "X-Real-IP" in request.headers: # pythonanywhere specific diff --git a/tmp/.gitinclude b/tmp/.gitinclude new file mode 100644 index 0000000..e69de29 From 25e35b650d55e71b3af5b7725cdb8318a15137a2 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Fri, 21 Apr 2017 22:53:45 -0700 Subject: [PATCH 3/4] Set development process to threaded --- run.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/run.py b/run.py index 6df3c74..4cb2554 100644 --- a/run.py +++ b/run.py @@ -4,9 +4,9 @@ from titanembeds.app import app def init_debug(): import os from flask import jsonify, request - + os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1' # Testing oauthlib - + # Session viewer https://gist.github.com/babldev/502364a3f7c9bafaa6db def decode_flask_cookie(secret_key, cookie_str): import hashlib @@ -20,7 +20,7 @@ def init_debug(): } s = URLSafeTimedSerializer(secret_key, salt=salt, serializer=serializer, signer_kwargs=signer_kwargs) return s.loads(cookie_str) - + @app.route("/session") def session(): cookie = request.cookies.get('session') @@ -32,4 +32,4 @@ def init_debug(): if __name__ == "__main__": init_debug() - app.run(host="0.0.0.0",port=3000,debug=True) + app.run(host="0.0.0.0",port=3000,debug=True,processes=3) From 1b5ba2dffb1dd78e4169de846d3aa58a9056ce21 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Fri, 21 Apr 2017 22:58:24 -0700 Subject: [PATCH 4/4] Change requirements txt to reflect recent changes --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index ba4f3f6..d00a5a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ flask flask-sqlalchemy -cachetools -Flask-Cache flask_limiter requests_oauthlib mysql-python -Flask-SSLify \ No newline at end of file +Flask-SSLify +redislite +beaker