diff --git a/requirements.txt b/requirements.txt index d00a5a4..6039554 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,5 +4,4 @@ flask_limiter requests_oauthlib mysql-python Flask-SSLify -redislite beaker diff --git a/titanembeds/app.py b/titanembeds/app.py index 613c328..40f5b27 100644 --- a/titanembeds/app.py +++ b/titanembeds/app.py @@ -2,7 +2,7 @@ from config import config from database import db from flask import Flask, render_template, request, session, url_for, redirect, jsonify from flask_sslify import SSLify -from titanembeds.utils import rate_limiter, cache +from titanembeds.utils import rate_limiter, cache, discord_api import blueprints.api import blueprints.user import blueprints.embed @@ -15,7 +15,7 @@ app.config['SQLALCHEMY_DATABASE_URI'] = config['database-uri'] app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False # Suppress the warning/no need this on for now. app.config['RATELIMIT_HEADERS_ENABLED'] = True app.config['SQLALCHEMY_POOL_RECYCLE'] = 250 -app.config['RATELIMIT_STORAGE_URL'] = 'redislite://redislite.db' +app.config['RATELIMIT_STORAGE_URL'] = 'keyvalprops://' app.secret_key = config['app-secret'] db.init_app(app) @@ -29,3 +29,8 @@ app.register_blueprint(blueprints.embed.embed, url_prefix="/embed", template_fol @app.route("/") def index(): return render_template("index.html.j2") + +@app.before_request +def before_request(): + db.create_all() + discord_api.init_discordrest() diff --git a/titanembeds/blueprints/embed/embed.py b/titanembeds/blueprints/embed/embed.py index f6c7ca7..b3c7360 100644 --- a/titanembeds/blueprints/embed/embed.py +++ b/titanembeds/blueprints/embed/embed.py @@ -23,10 +23,10 @@ def get_logingreeting(): def guild_embed(guild_id): if check_guild_existance(guild_id): guild = discord_api.get_guild(guild_id)['content'] - return render_template("embed.html.j2", - login_greeting=get_logingreeting(), - guild_id=guild_id, guild=guild, - generate_guild_icon=generate_guild_icon_url, + return render_template("embed.html.j2", + login_greeting=get_logingreeting(), + guild_id=guild_id, guild=guild, + generate_guild_icon=generate_guild_icon_url, unauth_enabled=guild_query_unauth_users_bool(guild_id), client_id=config['client-id'] ) @@ -38,4 +38,4 @@ def signin_complete(): @embed.route("/login_discord") def login_discord(): - return redirect(url_for("user.login_authenticated", redirect=url_for("embed.signin_complete", _external=True))) \ No newline at end of file + return redirect(url_for("user.login_authenticated", redirect=url_for("embed.signin_complete", _external=True))) diff --git a/titanembeds/database/__init__.py b/titanembeds/database/__init__.py index 28951e6..85fdafa 100644 --- a/titanembeds/database/__init__.py +++ b/titanembeds/database/__init__.py @@ -1,8 +1,9 @@ from flask_sqlalchemy import SQLAlchemy + db = SQLAlchemy() from guilds import Guilds from unauthenticated_users import UnauthenticatedUsers from unauthenticated_bans import UnauthenticatedBans from authenticated_users import AuthenticatedUsers -from custom_redislite import LimitsRedisLite +from keyvalue_properties import KeyValueProperties, set_keyvalproperty, get_keyvalproperty, getexpir_keyvalproperty, setexpir_keyvalproperty, ifexists_keyvalproperty, delete_keyvalproperty diff --git a/titanembeds/database/keyvalue_properties.py b/titanembeds/database/keyvalue_properties.py new file mode 100644 index 0000000..beed30a --- /dev/null +++ b/titanembeds/database/keyvalue_properties.py @@ -0,0 +1,98 @@ +from titanembeds.database import db +from datetime import datetime, timedelta +from limits.storage import Storage +import time + +def set_keyvalproperty(key, value, expiration=None): + q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) + if q.count() == 0: + db.session.add(KeyValueProperties(key=key, value=value, expiration=expiration)) + else: + if expiration is not None: + converted_expr = datetime.fromtimestamp(time.time() + expiration) + else: + converted_expr = None + firstobj = q.first() + firstobj.value = value + firstobj.expiration = converted_expr + db.session.commit() + +def get_keyvalproperty(key): + q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) + now = datetime.now() + if q.count() > 0 and (q.first().expiration is None or q.first().expiration > now): + return q.first().value + return None + +def getexpir_keyvalproperty(key): + q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) + now = datetime.now() + if q.count() > 0 and (q.first().expiration is not None and q.first().expiration > now): + return int(q.first().expiration.strftime('%s')) + return 0 + +def setexpir_keyvalproperty(key, expiration=None): + q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) + if q.count() > 0: + if expiration: + q.first().expiration = datetime.now() + else: + q.first().expiration = None + db.session.commit() + +def ifexists_keyvalproperty(key): + q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) + return q.count() > 0 + +def delete_keyvalproperty(key): + q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key).first() + if q: + db.session.delete(q) + db.session.commit() + +class KeyValueProperties(db.Model): + __tablename__ = "keyvalue_properties" + id = db.Column(db.Integer, primary_key=True) # Auto incremented id + key = db.Column(db.String(255)) # Property Key + value = db.Column(db.Text()) # Property value + expiration = db.Column(db.TIMESTAMP) # Suggested Expiration for value (None = no expire) in secs + + def __init__(self, key, value, expiration=None): + self.key = key + self.value = value + if expiration: + self.expiration = datetime.now() + timedelta(seconds = expiration) + else: + self.expiration = None + + +class LimitsKeyValueProperties(Storage): # For Python Limits + STORAGE_SCHEME = "keyvalprops" + def __init__(self, uri, **options): + pass + + def check(self): + return True + + def get_expiry(self, key): + return getexpir_keyvalproperty(key) + time.time() + + def incr(self, key, expiry, elastic_expiry=False): + if not ifexists_keyvalproperty(key): + set_keyvalproperty(key, 1, expiration=expiry) + else: + oldexp = getexpir_keyvalproperty(key) - time.time() + if oldexp <= 0: + delete_keyvalproperty(key) + return self.incr(key, expiry, elastic_expiry) + set_keyvalproperty(key, int(get_keyvalproperty(key))+1, expiration=int(round(oldexp))) + return int(self.get(key)) + + def get(self, key): + value = get_keyvalproperty(key) + if value: + return int(value) + return 0 + + def reset(self): + return False diff --git a/titanembeds/discordrest.py b/titanembeds/discordrest.py index 90af1c4..35b7159 100644 --- a/titanembeds/discordrest.py +++ b/titanembeds/discordrest.py @@ -2,9 +2,9 @@ import requests import sys import time import json -from functools import partial from titanembeds.utils import cache -from redislite import Redis +from titanembeds.database import db, KeyValueProperties, get_keyvalproperty, set_keyvalproperty, ifexists_keyvalproperty +from flask import request _DISCORD_API_BASE = "https://discordapp.com/api/v6" @@ -19,21 +19,21 @@ class DiscordREST: self.global_redis_prefix = "discordapiratelimit/" self.bot_token = bot_token self.user_agent = "TitanEmbeds (https://github.com/EndenDragon/Titan) Python/{} requests/{}".format(sys.version_info, requests.__version__) - self.rate_limit_bucket = Redis("redislite.db") + def init_discordrest(self): if not self._bucket_contains("global_limited"): self._set_bucket("global_limited", False) self._set_bucket("global_limit_expire", 0) def _get_bucket(self, key): - value = self.rate_limit_bucket.get(self.global_redis_prefix + key) + value = get_keyvalproperty(self.global_redis_prefix + key) return value def _set_bucket(self, key, value): - return self.rate_limit_bucket.set(self.global_redis_prefix + key, value) + return set_keyvalproperty(self.global_redis_prefix + key, value) def _bucket_contains(self, key): - return self.rate_limit_bucket.exists(self.global_redis_prefix + key) + return ifexists_keyvalproperty(self.global_redis_prefix + key) def request(self, verb, url, **kwargs): headers = { diff --git a/titanembeds/utils.py b/titanembeds/utils.py index 6184876..b6e87cf 100644 --- a/titanembeds/utils.py +++ b/titanembeds/utils.py @@ -1,6 +1,6 @@ from beaker.cache import CacheManager from beaker.util import parse_cache_config_options -from titanembeds.database import db, Guilds +from titanembeds.database import db, Guilds, KeyValueProperties from flask import request, session from flask_limiter import Limiter from config import config @@ -9,9 +9,9 @@ import string import hashlib cache_opts = { - 'cache.type': 'file', - 'cache.data_dir': 'tmp/cachedata', - 'cache.lock_dir': 'tmp/cachelock' + 'cache.type': 'ext:database', + 'cache.lock_dir': 'tmp/cachelock', + 'cache.url': config["database-uri"], } cache = CacheManager(**parse_cache_config_options(cache_opts))