Use redis to store discord api related objects, so far messages only

Keeping the table for now in case the redis method does not work as hoped

webapp/titanembeds/blueprints/api/api.py

@@ -1,6 +1,6 @@
-from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, Messages, get_channel_messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges, DiscordBotsOrgTransactions
+from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, Messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges, DiscordBotsOrgTransactions
 from titanembeds.decorators import valid_session_required, discord_users_only, abort_if_guild_disabled
-from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles, get_online_embed_user_keys, redis_store
+from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles, get_online_embed_user_keys, redis_store, redisqueue
 from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild
 import titanembeds.constants as constants
 from flask import Blueprint, abort, jsonify, session, request, url_for
@@ -214,7 +214,7 @@ def get_all_users(guild_id):
 def fetch():
     guild_id = request.args.get("guild_id")
     channel_id = request.args.get('channel_id')
-    after_snowflake = request.args.get('after', None, type=int)
+    after_snowflake = request.args.get('after', 0, type=int)
     if user_unauthenticated():
         key = session['user_keys'][guild_id]
     else:
@@ -233,7 +233,7 @@ def fetch():
         if not chan.get("read") or chan["channel"]["type"] != "text":
             status_code = 401
         else:
-            messages = get_channel_messages(guild_id, channel_id, after_snowflake)
+            messages = redisqueue.get_channel_messages(guild_id, channel_id, after_snowflake)
             status_code = 200
     response = jsonify(messages=messages, status=status)
     response.status_code = status_code
@@ -245,7 +245,7 @@ def fetch():
 def fetch_visitor():
     guild_id = request.args.get("guild_id")
     channel_id = request.args.get('channel_id')
-    after_snowflake = request.args.get('after', None, type=int)
+    after_snowflake = request.args.get('after', 0, type=int)
     if not guild_accepts_visitors(guild_id):
         abort(403)
     messages = {}
@@ -255,7 +255,7 @@ def fetch_visitor():
     if not chan.get("read") or chan["channel"]["type"] != "text":
         status_code = 401
     else:
-        messages = get_channel_messages(guild_id, channel_id, after_snowflake)
+        messages = redisqueue.get_channel_messages(guild_id, channel_id, after_snowflake)
         status_code = 200
     response = jsonify(messages=messages)
     response.status_code = status_code

webapp/titanembeds/discordrest.py

@@ -27,7 +27,7 @@ class DiscordREST:
     def _get_bucket(self, key):
         value = redis_store.get(self.global_redis_prefix + key)
         if value:
-            value = value.decode("utf-8")
+            value = value
         return value
 
     def _set_bucket(self, key, value):

webapp/titanembeds/oauth.py

@@ -47,7 +47,7 @@ def user_has_permission(permission, index):
 def get_user_guilds():
     cache = redis_store.get("OAUTH/USERGUILDS/"+str(make_user_cache_key()))
     if cache:
-        return cache.decode("utf-8")
+        return cache
     req = discordrest_from_user("/users/@me/guilds")
     if req.status_code != 200:
         if hasattr(request, "sid"):

webapp/titanembeds/redisqueue.py

@@ -0,0 +1,88 @@
+from titanembeds.utils import redis_store
+from titanembeds.database import get_guild_member
+import json
+import time
+
+class RedisQueue:
+    def __init__(self):
+        pass # Nothing really to initialize
+    
+    def get(self, key, resource, params, *, data_type="str"):
+        key = "Queue" + key
+        data = self._get(key, data_type)
+        payload = {
+            "key": key,
+            "resource": resource,
+            "params": params
+        }
+        loop_count = 0
+        while not data and loop_count < 10:
+            if loop_count % 5 == 0:
+                redis_store.publish("discord-api-req", json.dumps(payload))
+            time.sleep(0.5)
+            data = self._get(key, data_type)
+            loop_count += 1
+        redis_store.expire(key, 60 * 5)
+        if data == None:
+            return None
+        if data_type == "set":
+            data = list(data)
+            data_parsed = []
+            for d in data:
+                data_parsed.append(json.loads(d))
+            return data_parsed
+        return json.loads(data)
+    
+    def _get(self, key, data_type):
+        if data_type == "set":
+            return redis_store.smembers(key)
+        else:
+            return redis_store.get(key)
+    
+    def get_channel_messages(self, guild_id, channel_id, after_snowflake=0):
+        key = "/channels/{}/messages".format(channel_id)
+        q = self.get(key, "get_channel_messages", {"channel_id": channel_id}, data_type="set")
+        msgs = []
+        snowflakes = []
+        guild_members = {}
+        for x in q:
+            if x["id"] in snowflakes or int(x["id"]) <= int(after_snowflake):
+                continue
+            snowflakes.append(x["id"])
+            message = {
+                "attachments": x["attachments"],
+                "timestamp": x["timestamp"],
+                "id": x["id"],
+                "edited_timestamp": x["edited_timestamp"],
+                "author": x["author"],
+                "content": x["content"],
+                "channel_id": str(x["channel_id"]),
+                "mentions": x["mentions"],
+                "embeds": x["embeds"],
+            }
+            if message["author"]["id"] not in guild_members:
+                member = get_guild_member(guild_id, message["author"]["id"])
+                guild_members[message["author"]["id"]] = member
+            else:
+                member = guild_members[message["author"]["id"]]
+            message["author"]["nickname"] = None
+            if member:
+                message["author"]["nickname"] = member.nickname
+                message["author"]["avatar"] = member.avatar
+                message["author"]["discriminator"] = member.discriminator
+                message["author"]["username"] = member.username
+            for mention in message["mentions"]:
+                if mention["id"] not in guild_members:
+                    author = get_guild_member(guild_id, mention["id"])
+                    guild_members[mention["id"]] = author
+                else:
+                    author = guild_members[mention["id"]]
+                mention["nickname"] = None
+                if author:
+                    mention["nickname"] = author.nickname
+                    mention["avatar"] = author.avatar
+                    mention["username"] = author.username
+                    mention["discriminator"] = author.discriminator
+            msgs.append(message)
+        sorted_msgs = sorted(msgs, key=lambda k: k['id'], reverse=True) 
+        return sorted_msgs

webapp/titanembeds/utils.py

@@ -13,11 +13,13 @@ import hashlib
 import time
 import json
 
-redis_store = FlaskRedis()
+redis_store = FlaskRedis(charset="utf-8", decode_responses=True)
 
 from titanembeds.discordrest import DiscordREST
+from titanembeds.redisqueue import RedisQueue
 
 discord_api = DiscordREST(config['bot-token'])
+redisqueue = RedisQueue()
 
 def get_client_ipaddr():
     if request.headers.getlist("X-Forwarded-For"):
@@ -181,7 +183,6 @@ def get_online_embed_user_keys(guild_id="*", user_type=None):
         usrs[utype] = []
         keys = redis_store.keys("MemberPresence/{}/{}/*".format(guild_id, utype))
         for key in keys:
-            key = str(key, "utf-8")
             client_key = key.split("/")[-1]
             usrs[utype].append(client_key)
     return usrs