From b3a219e58d8cdd0bb536549401c1c4e276241cf9 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sun, 23 Apr 2017 20:02:04 -0700 Subject: [PATCH 1/4] inital keyval db --- titanembeds/database/__init__.py | 2 + titanembeds/database/keyvalue_properties.py | 50 +++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 titanembeds/database/keyvalue_properties.py diff --git a/titanembeds/database/__init__.py b/titanembeds/database/__init__.py index 28951e6..ee79d98 100644 --- a/titanembeds/database/__init__.py +++ b/titanembeds/database/__init__.py @@ -6,3 +6,5 @@ from unauthenticated_users import UnauthenticatedUsers from unauthenticated_bans import UnauthenticatedBans from authenticated_users import AuthenticatedUsers from custom_redislite import LimitsRedisLite + +from keyvalue_properties import set_keyvalproperty, get_keyvalproperty, getexpir_keyvalproperty, setexpir_keyvalproperty diff --git a/titanembeds/database/keyvalue_properties.py b/titanembeds/database/keyvalue_properties.py new file mode 100644 index 0000000..1b4fd84 --- /dev/null +++ b/titanembeds/database/keyvalue_properties.py @@ -0,0 +1,50 @@ +from titanembeds.database import db +from datetime import datetime + +def set_keyvalproperty(key, value, expiration=None): + q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) + if q.count() == 0: + db.session.add(KeyValueProperties(key=key, value=value, expiration=expiration)) + else: + firstobj = q.first() + firstobj.value = value + firstobj.expiration = expiration + db.session.commit() + +def get_keyvalproperty(key): + q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) + now = datetime.now() + if q.count() > 0 and (q.first().expiration is None or q.first().expiration > now): + return q.first().value + return None + +def getexpir_keyvalproperty(key): + q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) + now = datetime.now() + if q.count() > 0 and (q.first().expiration is not None and q.first().expiration > now): + return q.first().expiration + return None + +def setexpir_keyvalproperty(key, expiration=None): + q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) + if q.count() > 0: + if expiration: + q.first().expiration = datetime.now() + else: + q.first().expiration = None + db.session.commit() + +class KeyValueProperties(db.Model): + __tablename__ = "keyvalue_properties" + id = db.Column(db.Integer, primary_key=True) # Auto incremented id + key = db.Column(db.String(32)) # Property Key + value = db.Column(db.Text()) # Property value + expiration = db.Column(db.TIMESTAMP) # Suggested Expiration for value (None = no expire) in secs + + def __init__(self, key, value, expiration=None): + self.key = key + self.value = value + if expiration: + self.expiration = datetime.datetime.now() + datetime.timedelta(seconds = expiration) + else: + self.expiration = None From a212ca1052134c623e565a4e99b6e3c0cd785115 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sun, 23 Apr 2017 23:02:03 -0700 Subject: [PATCH 2/4] Some progress with moving over to sql keyval --- titanembeds/app.py | 8 +-- titanembeds/blueprints/embed/embed.py | 10 ++-- titanembeds/database/__init__.py | 5 +- titanembeds/database/keyvalue_properties.py | 56 +++++++++++++++++++-- titanembeds/discordrest.py | 10 ++-- titanembeds/utils.py | 2 +- 6 files changed, 65 insertions(+), 26 deletions(-) diff --git a/titanembeds/app.py b/titanembeds/app.py index 96976c5..b4bfe9e 100644 --- a/titanembeds/app.py +++ b/titanembeds/app.py @@ -15,7 +15,7 @@ app.config['SQLALCHEMY_DATABASE_URI'] = config['database-uri'] app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False # Suppress the warning/no need this on for now. app.config['RATELIMIT_HEADERS_ENABLED'] = True app.config['SQLALCHEMY_POOL_RECYCLE'] = 250 -app.config['RATELIMIT_STORAGE_URL'] = 'redislite://redislite.db' +app.config['RATELIMIT_STORAGE_URL'] = 'keyvalprops://' app.secret_key = config['app-secret'] db.init_app(app) @@ -29,9 +29,3 @@ app.register_blueprint(blueprints.embed.embed, url_prefix="/embed", template_fol @app.route("/") def index(): return render_template("index.html.j2") - -@app.route("/oldembed//") -def embed_get(guildid, channelid): - if 'username' not in session: - return redirect(url_for("get_set_username", guildid=guildid, channelid=channelid)) - return render_template("embed.html") diff --git a/titanembeds/blueprints/embed/embed.py b/titanembeds/blueprints/embed/embed.py index f6c7ca7..b3c7360 100644 --- a/titanembeds/blueprints/embed/embed.py +++ b/titanembeds/blueprints/embed/embed.py @@ -23,10 +23,10 @@ def get_logingreeting(): def guild_embed(guild_id): if check_guild_existance(guild_id): guild = discord_api.get_guild(guild_id)['content'] - return render_template("embed.html.j2", - login_greeting=get_logingreeting(), - guild_id=guild_id, guild=guild, - generate_guild_icon=generate_guild_icon_url, + return render_template("embed.html.j2", + login_greeting=get_logingreeting(), + guild_id=guild_id, guild=guild, + generate_guild_icon=generate_guild_icon_url, unauth_enabled=guild_query_unauth_users_bool(guild_id), client_id=config['client-id'] ) @@ -38,4 +38,4 @@ def signin_complete(): @embed.route("/login_discord") def login_discord(): - return redirect(url_for("user.login_authenticated", redirect=url_for("embed.signin_complete", _external=True))) \ No newline at end of file + return redirect(url_for("user.login_authenticated", redirect=url_for("embed.signin_complete", _external=True))) diff --git a/titanembeds/database/__init__.py b/titanembeds/database/__init__.py index ee79d98..85fdafa 100644 --- a/titanembeds/database/__init__.py +++ b/titanembeds/database/__init__.py @@ -1,10 +1,9 @@ from flask_sqlalchemy import SQLAlchemy + db = SQLAlchemy() from guilds import Guilds from unauthenticated_users import UnauthenticatedUsers from unauthenticated_bans import UnauthenticatedBans from authenticated_users import AuthenticatedUsers -from custom_redislite import LimitsRedisLite - -from keyvalue_properties import set_keyvalproperty, get_keyvalproperty, getexpir_keyvalproperty, setexpir_keyvalproperty +from keyvalue_properties import KeyValueProperties, set_keyvalproperty, get_keyvalproperty, getexpir_keyvalproperty, setexpir_keyvalproperty, ifexists_keyvalproperty, delete_keyvalproperty diff --git a/titanembeds/database/keyvalue_properties.py b/titanembeds/database/keyvalue_properties.py index 1b4fd84..4bb6eec 100644 --- a/titanembeds/database/keyvalue_properties.py +++ b/titanembeds/database/keyvalue_properties.py @@ -1,5 +1,7 @@ from titanembeds.database import db -from datetime import datetime +from datetime import datetime, timedelta +from limits.storage import Storage +import time def set_keyvalproperty(key, value, expiration=None): q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) @@ -22,8 +24,10 @@ def getexpir_keyvalproperty(key): q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) now = datetime.now() if q.count() > 0 and (q.first().expiration is not None and q.first().expiration > now): - return q.first().expiration - return None + print q.first().expiration + print datetime.now().strftime("%Y-%m-%d %H:%M:%S") + return int(q.first().expiration.strftime('%s')) + return 0 def setexpir_keyvalproperty(key, expiration=None): q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) @@ -34,10 +38,20 @@ def setexpir_keyvalproperty(key, expiration=None): q.first().expiration = None db.session.commit() +def ifexists_keyvalproperty(key): + q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) + return q.count() > 0 + +def delete_keyvalproperty(key): + q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key).first() + if q: + db.session.delete(q) + db.session.commit() + class KeyValueProperties(db.Model): __tablename__ = "keyvalue_properties" id = db.Column(db.Integer, primary_key=True) # Auto incremented id - key = db.Column(db.String(32)) # Property Key + key = db.Column(db.String(255)) # Property Key value = db.Column(db.Text()) # Property value expiration = db.Column(db.TIMESTAMP) # Suggested Expiration for value (None = no expire) in secs @@ -45,6 +59,38 @@ class KeyValueProperties(db.Model): self.key = key self.value = value if expiration: - self.expiration = datetime.datetime.now() + datetime.timedelta(seconds = expiration) + self.expiration = datetime.now() + timedelta(seconds = expiration) else: self.expiration = None + + +class LimitsKeyValueProperties(Storage): # For Python Limits + STORAGE_SCHEME = "keyvalprops" + def __init__(self, uri, **options): + pass + + def check(self): + return True + + def get_expiry(self, key): + return getexpir_keyvalproperty(key) + time.time() + + def incr(self, key, expiry, elastic_expiry=False): + if not ifexists_keyvalproperty(key): + set_keyvalproperty(key, 1, expiration=expiry) + else: + oldexp = getexpir_keyvalproperty(key) - time.time() + if oldexp <= 0: + delete_keyvalproperty(key) + return self.incr(key, expiry, elastic_expiry) + set_keyvalproperty(key, int(get_keyvalproperty(key))+1, expiration=int(round(oldexp))) + return int(self.get(key)) + + def get(self, key): + value = get_keyvalproperty(key) + if value: + return int(value) + return 0 + + def reset(self): + return False diff --git a/titanembeds/discordrest.py b/titanembeds/discordrest.py index 90af1c4..1343b31 100644 --- a/titanembeds/discordrest.py +++ b/titanembeds/discordrest.py @@ -4,7 +4,7 @@ import time import json from functools import partial from titanembeds.utils import cache -from redislite import Redis +from titanembeds.database import db, KeyValueProperties, get_keyvalproperty, set_keyvalproperty _DISCORD_API_BASE = "https://discordapp.com/api/v6" @@ -19,21 +19,21 @@ class DiscordREST: self.global_redis_prefix = "discordapiratelimit/" self.bot_token = bot_token self.user_agent = "TitanEmbeds (https://github.com/EndenDragon/Titan) Python/{} requests/{}".format(sys.version_info, requests.__version__) - self.rate_limit_bucket = Redis("redislite.db") if not self._bucket_contains("global_limited"): self._set_bucket("global_limited", False) self._set_bucket("global_limit_expire", 0) def _get_bucket(self, key): - value = self.rate_limit_bucket.get(self.global_redis_prefix + key) + value = get_keyvalproperty(self.global_redis_prefix + key) return value def _set_bucket(self, key, value): - return self.rate_limit_bucket.set(self.global_redis_prefix + key, value) + #set_keyvalproperty("", "") + return ""#set_keyvalproperty(self.global_redis_prefix + key, value) def _bucket_contains(self, key): - return self.rate_limit_bucket.exists(self.global_redis_prefix + key) + return ""#ifexists_keyvalproperty(self.global_redis_prefix + key) def request(self, verb, url, **kwargs): headers = { diff --git a/titanembeds/utils.py b/titanembeds/utils.py index 6184876..77a9c17 100644 --- a/titanembeds/utils.py +++ b/titanembeds/utils.py @@ -1,6 +1,6 @@ from beaker.cache import CacheManager from beaker.util import parse_cache_config_options -from titanembeds.database import db, Guilds +from titanembeds.database import db, Guilds, KeyValueProperties from flask import request, session from flask_limiter import Limiter from config import config From 9a698b72799f4b6b42ff2813d87f2cfa10bfb188 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Mon, 24 Apr 2017 11:55:09 -0700 Subject: [PATCH 3/4] Fixed discordrest issue of not initializing properly --- titanembeds/app.py | 6 +++++- titanembeds/discordrest.py | 9 +++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/titanembeds/app.py b/titanembeds/app.py index b4bfe9e..c26f719 100644 --- a/titanembeds/app.py +++ b/titanembeds/app.py @@ -2,7 +2,7 @@ from config import config from database import db from flask import Flask, render_template, request, session, url_for, redirect, jsonify from flask_sslify import SSLify -from titanembeds.utils import rate_limiter, cache +from titanembeds.utils import rate_limiter, cache, discord_api import blueprints.api import blueprints.user import blueprints.embed @@ -29,3 +29,7 @@ app.register_blueprint(blueprints.embed.embed, url_prefix="/embed", template_fol @app.route("/") def index(): return render_template("index.html.j2") + +@app.before_request +def before_request(): + discord_api.init_discordrest() diff --git a/titanembeds/discordrest.py b/titanembeds/discordrest.py index 1343b31..9e0b989 100644 --- a/titanembeds/discordrest.py +++ b/titanembeds/discordrest.py @@ -4,7 +4,8 @@ import time import json from functools import partial from titanembeds.utils import cache -from titanembeds.database import db, KeyValueProperties, get_keyvalproperty, set_keyvalproperty +from titanembeds.database import db, KeyValueProperties, get_keyvalproperty, set_keyvalproperty, ifexists_keyvalproperty +from flask import request _DISCORD_API_BASE = "https://discordapp.com/api/v6" @@ -20,6 +21,7 @@ class DiscordREST: self.bot_token = bot_token self.user_agent = "TitanEmbeds (https://github.com/EndenDragon/Titan) Python/{} requests/{}".format(sys.version_info, requests.__version__) + def init_discordrest(self): if not self._bucket_contains("global_limited"): self._set_bucket("global_limited", False) self._set_bucket("global_limit_expire", 0) @@ -29,11 +31,10 @@ class DiscordREST: return value def _set_bucket(self, key, value): - #set_keyvalproperty("", "") - return ""#set_keyvalproperty(self.global_redis_prefix + key, value) + return set_keyvalproperty(self.global_redis_prefix + key, value) def _bucket_contains(self, key): - return ""#ifexists_keyvalproperty(self.global_redis_prefix + key) + return ifexists_keyvalproperty(self.global_redis_prefix + key) def request(self, verb, url, **kwargs): headers = { From 8329165c7262ea24ac443cf6a1ae38c48a150b42 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Mon, 24 Apr 2017 19:57:00 -0700 Subject: [PATCH 4/4] Moved everything to database --- requirements.txt | 1 - titanembeds/app.py | 1 + titanembeds/database/keyvalue_properties.py | 8 +++++--- titanembeds/discordrest.py | 1 - titanembeds/utils.py | 6 +++--- 5 files changed, 9 insertions(+), 8 deletions(-) diff --git a/requirements.txt b/requirements.txt index d00a5a4..6039554 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,5 +4,4 @@ flask_limiter requests_oauthlib mysql-python Flask-SSLify -redislite beaker diff --git a/titanembeds/app.py b/titanembeds/app.py index c26f719..40f5b27 100644 --- a/titanembeds/app.py +++ b/titanembeds/app.py @@ -32,4 +32,5 @@ def index(): @app.before_request def before_request(): + db.create_all() discord_api.init_discordrest() diff --git a/titanembeds/database/keyvalue_properties.py b/titanembeds/database/keyvalue_properties.py index 4bb6eec..beed30a 100644 --- a/titanembeds/database/keyvalue_properties.py +++ b/titanembeds/database/keyvalue_properties.py @@ -8,9 +8,13 @@ def set_keyvalproperty(key, value, expiration=None): if q.count() == 0: db.session.add(KeyValueProperties(key=key, value=value, expiration=expiration)) else: + if expiration is not None: + converted_expr = datetime.fromtimestamp(time.time() + expiration) + else: + converted_expr = None firstobj = q.first() firstobj.value = value - firstobj.expiration = expiration + firstobj.expiration = converted_expr db.session.commit() def get_keyvalproperty(key): @@ -24,8 +28,6 @@ def getexpir_keyvalproperty(key): q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) now = datetime.now() if q.count() > 0 and (q.first().expiration is not None and q.first().expiration > now): - print q.first().expiration - print datetime.now().strftime("%Y-%m-%d %H:%M:%S") return int(q.first().expiration.strftime('%s')) return 0 diff --git a/titanembeds/discordrest.py b/titanembeds/discordrest.py index 9e0b989..35b7159 100644 --- a/titanembeds/discordrest.py +++ b/titanembeds/discordrest.py @@ -2,7 +2,6 @@ import requests import sys import time import json -from functools import partial from titanembeds.utils import cache from titanembeds.database import db, KeyValueProperties, get_keyvalproperty, set_keyvalproperty, ifexists_keyvalproperty from flask import request diff --git a/titanembeds/utils.py b/titanembeds/utils.py index 77a9c17..b6e87cf 100644 --- a/titanembeds/utils.py +++ b/titanembeds/utils.py @@ -9,9 +9,9 @@ import string import hashlib cache_opts = { - 'cache.type': 'file', - 'cache.data_dir': 'tmp/cachedata', - 'cache.lock_dir': 'tmp/cachelock' + 'cache.type': 'ext:database', + 'cache.lock_dir': 'tmp/cachelock', + 'cache.url': config["database-uri"], } cache = CacheManager(**parse_cache_config_options(cache_opts))