From 348580dcbbea6eec0806208ff1d43d4ef76efab8 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Fri, 29 Dec 2017 17:40:32 +0000 Subject: [PATCH] Remove keyvalue properties in favor of redis --- requirements.txt | 3 +- ...6894fa_remove_keyvalue_properties_table.py | 34 +++++++ webapp/titanembeds/app.py | 6 +- webapp/titanembeds/blueprints/api/api.py | 21 +--- webapp/titanembeds/database/__init__.py | 1 - .../database/keyvalue_properties.py | 98 ------------------- webapp/titanembeds/discordrest.py | 8 +- webapp/titanembeds/oauth.py | 9 +- webapp/titanembeds/static/js/admin_index.js | 30 ------ .../titanembeds/templates/admin_index.html.j2 | 10 -- webapp/titanembeds/utils.py | 3 + 11 files changed, 53 insertions(+), 170 deletions(-) create mode 100644 webapp/alembic/versions/d5dcee6894fa_remove_keyvalue_properties_table.py delete mode 100644 webapp/titanembeds/database/keyvalue_properties.py delete mode 100644 webapp/titanembeds/static/js/admin_index.js diff --git a/requirements.txt b/requirements.txt index 2848a12..2d6e3cd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ kombu redis aioredis Flask-Babel -patreon \ No newline at end of file +patreon +flask-redis \ No newline at end of file diff --git a/webapp/alembic/versions/d5dcee6894fa_remove_keyvalue_properties_table.py b/webapp/alembic/versions/d5dcee6894fa_remove_keyvalue_properties_table.py new file mode 100644 index 0000000..880ab74 --- /dev/null +++ b/webapp/alembic/versions/d5dcee6894fa_remove_keyvalue_properties_table.py @@ -0,0 +1,34 @@ +"""Remove keyvalue properties table + +Revision ID: d5dcee6894fa +Revises: 66971a97040e +Create Date: 2017-12-29 17:39:24.192424 + +""" + +# revision identifiers, used by Alembic. +revision = 'd5dcee6894fa' +down_revision = '66971a97040e' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('keyvalue_properties') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('keyvalue_properties', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('key', sa.VARCHAR(length=255), server_default=sa.text("''::character varying"), autoincrement=False, nullable=False), + sa.Column('value', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('expiration', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name='idx_25223_primary') + ) + # ### end Alembic commands ### diff --git a/webapp/titanembeds/app.py b/webapp/titanembeds/app.py index 5fd5f62..8564c41 100644 --- a/webapp/titanembeds/app.py +++ b/webapp/titanembeds/app.py @@ -2,7 +2,7 @@ from config import config from .database import db from flask import Flask, render_template, request, session, url_for, redirect, jsonify from flask_sslify import SSLify -from titanembeds.utils import rate_limiter, discord_api, socketio, babel +from titanembeds.utils import rate_limiter, discord_api, socketio, babel, redis_store from .blueprints import api, user, admin, embed, gateway import os from titanembeds.database import get_administrators_list @@ -29,8 +29,9 @@ app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False # Suppress the warning/no app.config['RATELIMIT_HEADERS_ENABLED'] = True app.config['SQLALCHEMY_POOL_RECYCLE'] = 250 app.config['SQLALCHEMY_POOL_SIZE'] = 100 -app.config['RATELIMIT_STORAGE_URL'] = 'keyvalprops://' +app.config['RATELIMIT_STORAGE_URL'] = config["redis-uri"] app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=3) +app.config['REDIS_URL'] = config["redis-uri"] app.secret_key = config['app-secret'] db.init_app(app) @@ -39,6 +40,7 @@ if config.get("enable-ssl", False): sslify = SSLify(app, permanent=True) socketio.init_app(app, message_queue=config["redis-uri"], path='gateway', async_mode=config.get("websockets-mode", None)) babel.init_app(app) +redis_store.init_app(app) app.register_blueprint(api.api, url_prefix="/api", template_folder="/templates") app.register_blueprint(admin.admin, url_prefix="/admin", template_folder="/templates") diff --git a/webapp/titanembeds/blueprints/api/api.py b/webapp/titanembeds/blueprints/api/api.py index 865052c..c6afe4e 100644 --- a/webapp/titanembeds/blueprints/api/api.py +++ b/webapp/titanembeds/blueprints/api/api.py @@ -1,4 +1,4 @@ -from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, KeyValueProperties, GuildMembers, Messages, get_channel_messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges +from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, Messages, get_channel_messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges from titanembeds.decorators import valid_session_required, discord_users_only from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild @@ -457,21 +457,4 @@ def user_info(guild_id, user_id): if gr["id"] == r: usr["roles"].append(gr) usr["badges"] = get_badges(user_id) - return jsonify(usr) - -def canCleanupDB(): - canclean = False - if request.form.get("secret", None) == config['app-secret']: - canclean = True - if 'user_id' in session: - if session['user_id'] in get_administrators_list(): - canclean = True - return canclean - -@api.route("/cleanup-db", methods=["DELETE"]) -def cleanup_keyval_db(): - if canCleanupDB(): - db.session.query(KeyValueProperties).filter(KeyValueProperties.expiration < datetime.datetime.now()).delete() - db.session.commit() - return ('', 204) - abort(401) + return jsonify(usr) \ No newline at end of file diff --git a/webapp/titanembeds/database/__init__.py b/webapp/titanembeds/database/__init__.py index c0833c5..221a24d 100644 --- a/webapp/titanembeds/database/__init__.py +++ b/webapp/titanembeds/database/__init__.py @@ -7,7 +7,6 @@ from .unauthenticated_users import UnauthenticatedUsers from .unauthenticated_bans import UnauthenticatedBans from .authenticated_users import AuthenticatedUsers from .guild_members import GuildMembers, list_all_guild_members, get_guild_member -from .keyvalue_properties import KeyValueProperties, set_keyvalproperty, get_keyvalproperty, getexpir_keyvalproperty, setexpir_keyvalproperty, ifexists_keyvalproperty, delete_keyvalproperty from .messages import Messages, get_channel_messages from .cosmetics import Cosmetics, set_badges, get_badges, add_badge, remove_badge from .user_css import UserCSS diff --git a/webapp/titanembeds/database/keyvalue_properties.py b/webapp/titanembeds/database/keyvalue_properties.py deleted file mode 100644 index 64db514..0000000 --- a/webapp/titanembeds/database/keyvalue_properties.py +++ /dev/null @@ -1,98 +0,0 @@ -from titanembeds.database import db -from datetime import datetime, timedelta -from limits.storage import Storage -import time - -def set_keyvalproperty(key, value, expiration=None): - q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) - if q.count() == 0: - db.session.add(KeyValueProperties(key=key, value=value, expiration=expiration)) - else: - if expiration is not None: - converted_expr = datetime.fromtimestamp(time.time() + expiration) - else: - converted_expr = None - firstobj = q.first() - firstobj.value = value - firstobj.expiration = converted_expr - db.session.commit() - -def get_keyvalproperty(key): - q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) - now = datetime.now() - if q.count() > 0 and (q.first().expiration is None or q.first().expiration.replace(tzinfo=None) > now.replace(tzinfo=None)): - return q.first().value - return None - -def getexpir_keyvalproperty(key): - q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) - now = datetime.now() - if q.count() > 0 and (q.first().expiration is not None and q.first().expiration.replace(tzinfo=None) > now.replace(tzinfo=None)): - return int(q.first().expiration.strftime('%s')) - return 0 - -def setexpir_keyvalproperty(key, expiration=None): - q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) - if q.count() > 0: - if expiration: - q.first().expiration = datetime.now() - else: - q.first().expiration = None - db.session.commit() - -def ifexists_keyvalproperty(key): - q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key) - return q.count() > 0 - -def delete_keyvalproperty(key): - q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key).first() - if q: - db.session.delete(q) - db.session.commit() - -class KeyValueProperties(db.Model): - __tablename__ = "keyvalue_properties" - id = db.Column(db.Integer, primary_key=True) # Auto incremented id - key = db.Column(db.String(255), nullable=False) # Property Key - value = db.Column(db.Text()) # Property value - expiration = db.Column(db.TIMESTAMP) # Suggested Expiration for value (None = no expire) in secs - - def __init__(self, key, value, expiration=None): - self.key = key - self.value = value - if expiration: - self.expiration = datetime.now() + timedelta(seconds = expiration) - else: - self.expiration = None - - -class LimitsKeyValueProperties(Storage): # For Python Limits - STORAGE_SCHEME = "keyvalprops" - def __init__(self, uri, **options): - pass - - def check(self): - return True - - def get_expiry(self, key): - return getexpir_keyvalproperty(key) + time.time() - - def incr(self, key, expiry, elastic_expiry=False): - if not ifexists_keyvalproperty(key): - set_keyvalproperty(key, 1, expiration=expiry) - else: - oldexp = getexpir_keyvalproperty(key) - time.time() - if oldexp <= 0: - delete_keyvalproperty(key) - return self.incr(key, expiry, elastic_expiry) - set_keyvalproperty(key, int(get_keyvalproperty(key))+1, expiration=int(round(oldexp))) - return int(self.get(key)) - - def get(self, key): - value = get_keyvalproperty(key) - if value: - return int(value) - return 0 - - def reset(self): - return False diff --git a/webapp/titanembeds/discordrest.py b/webapp/titanembeds/discordrest.py index c89d54f..cf31dae 100644 --- a/webapp/titanembeds/discordrest.py +++ b/webapp/titanembeds/discordrest.py @@ -2,7 +2,7 @@ import requests import sys import time import json -from titanembeds.database import db, KeyValueProperties, get_keyvalproperty, set_keyvalproperty, ifexists_keyvalproperty +from titanembeds.utils import redis_store from flask import request _DISCORD_API_BASE = "https://discordapp.com/api/v6" @@ -25,14 +25,14 @@ class DiscordREST: self._set_bucket("global_limit_expire", 0) def _get_bucket(self, key): - value = get_keyvalproperty(self.global_redis_prefix + key) + value = redis_store.get(self.global_redis_prefix + key).decode("utf-8") return value def _set_bucket(self, key, value): - return set_keyvalproperty(self.global_redis_prefix + key, value) + return redis_store.set(self.global_redis_prefix + key, value) def _bucket_contains(self, key): - return ifexists_keyvalproperty(self.global_redis_prefix + key) + return redis_store.exists(self.global_redis_prefix + key) def request(self, verb, url, **kwargs): headers = { diff --git a/webapp/titanembeds/oauth.py b/webapp/titanembeds/oauth.py index a3c3d19..aa83ce1 100644 --- a/webapp/titanembeds/oauth.py +++ b/webapp/titanembeds/oauth.py @@ -2,8 +2,7 @@ from config import config import json from requests_oauthlib import OAuth2Session from flask import session, abort, url_for -from titanembeds.database import get_keyvalproperty, set_keyvalproperty -from titanembeds.utils import make_user_cache_key +from titanembeds.utils import redis_store, make_user_cache_key authorize_url = "https://discordapp.com/api/oauth2/authorize" token_url = "https://discordapp.com/api/oauth2/token" @@ -45,14 +44,14 @@ def user_has_permission(permission, index): return bool((int(permission) >> index) & 1) def get_user_guilds(): - cache = get_keyvalproperty("OAUTH/USERGUILDS/"+str(make_user_cache_key())) + cache = redis_store.get("OAUTH/USERGUILDS/"+str(make_user_cache_key())) if cache: - return cache + return cache.decode("utf-8") req = discordrest_from_user("/users/@me/guilds") if req.status_code != 200: abort(req.status_code) req = json.dumps(req.json()) - set_keyvalproperty("OAUTH/USERGUILDS/"+str(make_user_cache_key()), req, 250) + redis_store.set("OAUTH/USERGUILDS/"+str(make_user_cache_key()), req, 250) return req def get_user_managed_servers(): diff --git a/webapp/titanembeds/static/js/admin_index.js b/webapp/titanembeds/static/js/admin_index.js deleted file mode 100644 index 1a1f11c..0000000 --- a/webapp/titanembeds/static/js/admin_index.js +++ /dev/null @@ -1,30 +0,0 @@ -/* global $ */ -/* global Materialize */ - -(function () { - function cleanup_database() { - var funct = $.ajax({ - method: "DELETE", - url: "/api/cleanup-db", - }); - return funct.promise(); - } - - $(function(){ - $("#db_cleanup_btn").click(run_cleanup_db); - }); - - function run_cleanup_db() { - $("#db_cleanup_btn").attr("disabled",true); - Materialize.toast('Please wait for the cleanup database task to finish...', 10000); - var cleanupdb = cleanup_database(); - cleanupdb.done(function () { - $("#db_cleanup_btn").attr("disabled",false); - Materialize.toast('Successfully cleaned up the database!', 10000); - }); - cleanupdb.fail(function () { - $("#db_cleanup_btn").attr("disabled",false); - Materialize.toast('Database cleanup failiure.', 10000); - }); - } -})(); \ No newline at end of file diff --git a/webapp/titanembeds/templates/admin_index.html.j2 b/webapp/titanembeds/templates/admin_index.html.j2 index 5201f37..310fbb6 100644 --- a/webapp/titanembeds/templates/admin_index.html.j2 +++ b/webapp/titanembeds/templates/admin_index.html.j2 @@ -27,15 +27,5 @@ Manage -
-
-

Run a Database Cleanup

-

Clears the keyval caches. (Hit once, and wait a minute)

- Run DB Cleanup Task -
-
-{% endblock %} -{% block script %} - {% endblock %} \ No newline at end of file diff --git a/webapp/titanembeds/utils.py b/webapp/titanembeds/utils.py index 4ec7bd8..1099782 100644 --- a/webapp/titanembeds/utils.py +++ b/webapp/titanembeds/utils.py @@ -3,6 +3,7 @@ from flask import request, session from flask_limiter import Limiter from flask_socketio import SocketIO from flask_babel import Babel +from flask_redis import FlaskRedis from config import config from sqlalchemy import and_ import random @@ -11,6 +12,8 @@ import hashlib import time import json +redis_store = FlaskRedis() + from titanembeds.discordrest import DiscordREST discord_api = DiscordREST(config['bot-token'])