Remove keyvalue properties in favor of redis

requirements.txt

@@ -11,4 +11,5 @@ kombu
 redis
 aioredis
 Flask-Babel
-patreon
+patreon
+flask-redis

webapp/alembic/versions/d5dcee6894fa_remove_keyvalue_properties_table.py

@@ -0,0 +1,34 @@
+"""Remove keyvalue properties table
+
+Revision ID: d5dcee6894fa
+Revises: 66971a97040e
+Create Date: 2017-12-29 17:39:24.192424
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'd5dcee6894fa'
+down_revision = '66971a97040e'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('keyvalue_properties')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('keyvalue_properties',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('key', sa.VARCHAR(length=255), server_default=sa.text("''::character varying"), autoincrement=False, nullable=False),
+    sa.Column('value', sa.TEXT(), autoincrement=False, nullable=True),
+    sa.Column('expiration', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
+    sa.PrimaryKeyConstraint('id', name='idx_25223_primary')
+    )
+    # ### end Alembic commands ###

webapp/titanembeds/app.py

@@ -2,7 +2,7 @@ from config import config
 from .database import db
 from flask import Flask, render_template, request, session, url_for, redirect, jsonify
 from flask_sslify import SSLify
-from titanembeds.utils import rate_limiter, discord_api, socketio, babel
+from titanembeds.utils import rate_limiter, discord_api, socketio, babel, redis_store
 from .blueprints import api, user, admin, embed, gateway
 import os
 from titanembeds.database import get_administrators_list
@@ -29,8 +29,9 @@ app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False  # Suppress the warning/no
 app.config['RATELIMIT_HEADERS_ENABLED'] = True
 app.config['SQLALCHEMY_POOL_RECYCLE'] = 250
 app.config['SQLALCHEMY_POOL_SIZE'] = 100
-app.config['RATELIMIT_STORAGE_URL'] = 'keyvalprops://'
+app.config['RATELIMIT_STORAGE_URL'] = config["redis-uri"]
 app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(days=3)
+app.config['REDIS_URL'] = config["redis-uri"]
 app.secret_key = config['app-secret']
 
 db.init_app(app)
@@ -39,6 +40,7 @@ if config.get("enable-ssl", False):
     sslify = SSLify(app, permanent=True)
 socketio.init_app(app, message_queue=config["redis-uri"], path='gateway', async_mode=config.get("websockets-mode", None))
 babel.init_app(app)
+redis_store.init_app(app)
 
 app.register_blueprint(api.api, url_prefix="/api", template_folder="/templates")
 app.register_blueprint(admin.admin, url_prefix="/admin", template_folder="/templates")

webapp/titanembeds/blueprints/api/api.py

@@ -1,4 +1,4 @@
-from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, KeyValueProperties, GuildMembers, Messages, get_channel_messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges
+from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, Messages, get_channel_messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges
 from titanembeds.decorators import valid_session_required, discord_users_only
 from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles
 from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild
@@ -457,21 +457,4 @@ def user_info(guild_id, user_id):
                 if gr["id"] == r:
                     usr["roles"].append(gr)
         usr["badges"] = get_badges(user_id)
-    return jsonify(usr)
-
-def canCleanupDB():
-    canclean = False
-    if request.form.get("secret", None) == config['app-secret']:
-        canclean = True
-    if 'user_id' in session:
-        if session['user_id'] in get_administrators_list():
-            canclean = True
-    return canclean
-    
-@api.route("/cleanup-db", methods=["DELETE"])
-def cleanup_keyval_db():
-    if canCleanupDB():
-        db.session.query(KeyValueProperties).filter(KeyValueProperties.expiration < datetime.datetime.now()).delete()
-        db.session.commit()
-        return ('', 204)
-    abort(401)
+    return jsonify(usr)

webapp/titanembeds/database/__init__.py

@@ -7,7 +7,6 @@ from .unauthenticated_users import UnauthenticatedUsers
 from .unauthenticated_bans import UnauthenticatedBans
 from .authenticated_users import AuthenticatedUsers
 from .guild_members import GuildMembers, list_all_guild_members, get_guild_member
-from .keyvalue_properties import KeyValueProperties, set_keyvalproperty, get_keyvalproperty, getexpir_keyvalproperty, setexpir_keyvalproperty, ifexists_keyvalproperty, delete_keyvalproperty
 from .messages import Messages, get_channel_messages
 from .cosmetics import Cosmetics, set_badges, get_badges, add_badge, remove_badge
 from .user_css import UserCSS

webapp/titanembeds/database/keyvalue_properties.py

@@ -1,98 +0,0 @@
-from titanembeds.database import db
-from datetime import datetime, timedelta
-from limits.storage import Storage
-import time
-
-def set_keyvalproperty(key, value, expiration=None):
-    q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key)
-    if q.count() == 0:
-        db.session.add(KeyValueProperties(key=key, value=value, expiration=expiration))
-    else:
-        if expiration is not None:
-            converted_expr = datetime.fromtimestamp(time.time() + expiration)
-        else:
-            converted_expr = None
-        firstobj = q.first()
-        firstobj.value = value
-        firstobj.expiration = converted_expr
-    db.session.commit()
-
-def get_keyvalproperty(key):
-    q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key)
-    now = datetime.now()
-    if q.count() > 0 and (q.first().expiration is None or q.first().expiration.replace(tzinfo=None) > now.replace(tzinfo=None)):
-        return q.first().value
-    return None
-
-def getexpir_keyvalproperty(key):
-    q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key)
-    now = datetime.now()
-    if q.count() > 0 and (q.first().expiration is not None and q.first().expiration.replace(tzinfo=None) > now.replace(tzinfo=None)):
-        return int(q.first().expiration.strftime('%s'))
-    return 0
-
-def setexpir_keyvalproperty(key, expiration=None):
-    q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key)
-    if q.count() > 0:
-        if expiration:
-            q.first().expiration = datetime.now()
-        else:
-            q.first().expiration = None
-        db.session.commit()
-
-def ifexists_keyvalproperty(key):
-    q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key)
-    return q.count() > 0
-
-def delete_keyvalproperty(key):
-    q = db.session.query(KeyValueProperties).filter(KeyValueProperties.key == key).first()
-    if q:
-        db.session.delete(q)
-        db.session.commit()
-
-class KeyValueProperties(db.Model):
-    __tablename__ = "keyvalue_properties"
-    id = db.Column(db.Integer, primary_key=True)                    # Auto incremented id
-    key = db.Column(db.String(255), nullable=False)                 # Property Key
-    value = db.Column(db.Text())                                    # Property value
-    expiration = db.Column(db.TIMESTAMP)                            # Suggested Expiration for value (None = no expire) in secs
-
-    def __init__(self, key, value, expiration=None):
-        self.key = key
-        self.value = value
-        if expiration:
-            self.expiration = datetime.now() + timedelta(seconds = expiration)
-        else:
-            self.expiration = None
-
-
-class LimitsKeyValueProperties(Storage): # For Python Limits
-    STORAGE_SCHEME = "keyvalprops"
-    def __init__(self, uri, **options):
-        pass
-
-    def check(self):
-        return True
-
-    def get_expiry(self, key):
-        return getexpir_keyvalproperty(key) + time.time()
-
-    def incr(self, key, expiry, elastic_expiry=False):
-        if not ifexists_keyvalproperty(key):
-            set_keyvalproperty(key, 1, expiration=expiry)
-        else:
-            oldexp = getexpir_keyvalproperty(key) - time.time()
-            if oldexp <= 0:
-                delete_keyvalproperty(key)
-                return self.incr(key, expiry, elastic_expiry)
-            set_keyvalproperty(key, int(get_keyvalproperty(key))+1, expiration=int(round(oldexp)))
-        return int(self.get(key))
-
-    def get(self, key):
-        value = get_keyvalproperty(key)
-        if value:
-            return int(value)
-        return 0
-
-    def reset(self):
-        return False

webapp/titanembeds/discordrest.py

@@ -2,7 +2,7 @@ import requests
 import sys
 import time
 import json
-from titanembeds.database import db, KeyValueProperties, get_keyvalproperty, set_keyvalproperty, ifexists_keyvalproperty
+from titanembeds.utils import redis_store
 from flask import request
 
 _DISCORD_API_BASE = "https://discordapp.com/api/v6"
@@ -25,14 +25,14 @@ class DiscordREST:
             self._set_bucket("global_limit_expire", 0)
 
     def _get_bucket(self, key):
-        value = get_keyvalproperty(self.global_redis_prefix + key)
+        value = redis_store.get(self.global_redis_prefix + key).decode("utf-8")
         return value
 
     def _set_bucket(self, key, value):
-        return set_keyvalproperty(self.global_redis_prefix + key, value)
+        return redis_store.set(self.global_redis_prefix + key, value)
 
     def _bucket_contains(self, key):
-        return ifexists_keyvalproperty(self.global_redis_prefix + key)
+        return redis_store.exists(self.global_redis_prefix + key)
 
     def request(self, verb, url, **kwargs):
         headers = {

webapp/titanembeds/oauth.py

@@ -2,8 +2,7 @@ from config import config
 import json
 from requests_oauthlib import OAuth2Session
 from flask import session, abort, url_for
-from titanembeds.database import get_keyvalproperty, set_keyvalproperty
-from titanembeds.utils import make_user_cache_key
+from titanembeds.utils import redis_store, make_user_cache_key
 
 authorize_url = "https://discordapp.com/api/oauth2/authorize"
 token_url = "https://discordapp.com/api/oauth2/token"
@@ -45,14 +44,14 @@ def user_has_permission(permission, index):
     return bool((int(permission) >> index) & 1)
 
 def get_user_guilds():
-    cache = get_keyvalproperty("OAUTH/USERGUILDS/"+str(make_user_cache_key()))
+    cache = redis_store.get("OAUTH/USERGUILDS/"+str(make_user_cache_key()))
     if cache:
-        return cache
+        return cache.decode("utf-8")
     req = discordrest_from_user("/users/@me/guilds")
     if req.status_code != 200:
         abort(req.status_code)
     req = json.dumps(req.json())
-    set_keyvalproperty("OAUTH/USERGUILDS/"+str(make_user_cache_key()), req, 250)
+    redis_store.set("OAUTH/USERGUILDS/"+str(make_user_cache_key()), req, 250)
     return req
 
 def get_user_managed_servers():

webapp/titanembeds/static/js/admin_index.js

@@ -1,30 +0,0 @@
-/* global $ */
-/* global Materialize */
-
-(function () {
-    function cleanup_database() {
-        var funct = $.ajax({
-            method: "DELETE",
-            url: "/api/cleanup-db",
-        });
-        return funct.promise();
-    }
-    
-    $(function(){
-        $("#db_cleanup_btn").click(run_cleanup_db);
-    });
-    
-    function run_cleanup_db() {
-        $("#db_cleanup_btn").attr("disabled",true);
-        Materialize.toast('Please wait for the cleanup database task to finish...', 10000);
-        var cleanupdb = cleanup_database();
-        cleanupdb.done(function () {
-            $("#db_cleanup_btn").attr("disabled",false);
-            Materialize.toast('Successfully cleaned up the database!', 10000);
-        });
-        cleanupdb.fail(function () {
-            $("#db_cleanup_btn").attr("disabled",false);
-            Materialize.toast('Database cleanup failiure.', 10000);
-        });
-    }
-})();

webapp/titanembeds/templates/admin_index.html.j2

@@ -27,15 +27,5 @@
       <a class="waves-effect waves-light btn" href="{{ url_for('admin.manage_titan_tokens') }}">Manage</a>
     </div>
   </div>
-  <div class="col s12">
-    <div class="card-panel indigo lighten-5 z-depth-3 hoverable black-text">
-      <h4>Run a Database Cleanup</h4>
-      <p class="flow-text">Clears the keyval caches. (Hit once, and wait a minute)</p>
-      <a class="waves-effect waves-light btn" id="db_cleanup_btn">Run DB Cleanup Task</a>
-    </div>
-  </div>
 </div>
-{% endblock %}
-{% block script %}
-<script type="text/javascript" src="{{ url_for('static', filename='js/admin_index.js') }}"></script>
 {% endblock %}

webapp/titanembeds/utils.py

@@ -3,6 +3,7 @@ from flask import request, session
 from flask_limiter import Limiter
 from flask_socketio import SocketIO
 from flask_babel import Babel
+from flask_redis import FlaskRedis
 from config import config
 from sqlalchemy import and_
 import random
@@ -11,6 +12,8 @@ import hashlib
 import time
 import json
 
+redis_store = FlaskRedis()
+
 from titanembeds.discordrest import DiscordREST
 
 discord_api = DiscordREST(config['bot-token'])