Remove guild members table for storing in redis

discordbot/titanembeds/bot.py

@@ -105,23 +105,6 @@ class Titan(discord.AutoShardedClient):
 
     async def on_guild_join(self, guild):
         await self.database.update_guild(guild)
-        for member in guild.members:
-            await self.database.update_guild_member(member, True, False)
-        if guild.me.guild_permissions.ban_members:
-            banned = await guild.bans()
-            for ban in banned:
-                member = discord.Member(user={
-                    "username": ban.name,
-                    "id": ban.id,
-                    "discriminator": ban.discriminator,
-                    "avatar": ban.avatar,
-                    "bot": ban.bot
-                })
-                await self.database.update_guild_member(member, False, True)
-        for channel in list(guild.channels):
-            chanperm = channel.permissions_for(channel.guild.me)
-            if not chanperm.read_messages or not chanperm.read_message_history or not isinstance(channel, discord.channel.TextChannel):
-                continue
         await self.postStats()
 
     async def on_guild_remove(self, guild):
@@ -163,24 +146,21 @@ class Titan(discord.AutoShardedClient):
         await self.socketio.on_channel_update(channelafter)
 
     async def on_member_join(self, member):
-        await self.database.update_guild_member(member, active=True, banned=False)
+        await self.redisqueue.add_member(member)
         await self.socketio.on_guild_member_add(member)
 
     async def on_member_remove(self, member):
-        await self.database.update_guild_member(member, active=False, banned=False)
+        await self.redisqueue.remove_member(member)
         await self.socketio.on_guild_member_remove(member)
 
     async def on_member_update(self, memberbefore, memberafter):
-        await self.database.update_guild_member(memberafter)
+        await self.redisqueue.update_member(memberafter)
         await self.socketio.on_guild_member_update(memberafter)
 
     async def on_member_ban(self, guild, user):
         if self.user.id == user.id:
             return
-        await self.database.update_guild_member(user, active=False, banned=True, guild=guild)
-
-    async def on_member_unban(self, guild, user):
-        await self.database.unban_server_user(user, guild)
+        await self.redisqueue.ban_member(guild, user)
     
     async def on_guild_emojis_update(self, guild, before, after):
         if len(after) == 0:

discordbot/titanembeds/database/__init__.py

@@ -6,7 +6,6 @@ import datetime
 db = Gino()
 
 from titanembeds.database.guilds import Guilds
-from titanembeds.database.guild_members import GuildMembers
 from titanembeds.database.unauthenticated_users import UnauthenticatedUsers
 from titanembeds.database.unauthenticated_bans import UnauthenticatedBans
 
@@ -71,82 +70,6 @@ class DatabaseInterface(object):
             await Messages.delete.where(Messages.guild_id == int(guild.id)).gino.status()
             await gui.delete()
                 
-    async def update_guild_member(self, member, active=True, banned=False, guild=None):
-        if guild:
-            dbmember = await GuildMembers.query \
-                .where(GuildMembers.guild_id == int(guild.id)) \
-                .where(GuildMembers.user_id == int(member.id)) \
-                .order_by(GuildMembers.id).gino.all()
-        else:
-            dbmember = await GuildMembers.query \
-                .where(GuildMembers.guild_id == int(member.guild.id)) \
-                .where(GuildMembers.user_id == int(member.id)) \
-                .order_by(GuildMembers.id).gino.all()
-        if not dbmember:
-            await GuildMembers.create(
-                guild_id = int(member.guild.id),
-                user_id = int(member.id),
-                username = member.name,
-                discriminator = int(member.discriminator),
-                nickname = member.nick,
-                avatar = member.avatar,
-                active = active,
-                banned = banned,
-                roles = json.dumps(list_role_ids(member.roles))
-            )
-        else:
-            if len(dbmember) > 1:
-                for mem in dbmember[1:]:
-                    await mem.delete()
-            dbmember = dbmember[0]
-            if dbmember.banned != banned or dbmember.active != active or dbmember.username != member.name or dbmember.discriminator != int(member.discriminator) or dbmember.nickname != member.nick or dbmember.avatar != member.avatar or set(json.loads(dbmember.roles)) != set(list_role_ids(member.roles)):
-                await dbmember.update(
-                    banned = banned,
-                    active = active,
-                    username = member.name,
-                    discriminator = int(member.discriminator),
-                    nickname = member.nick,
-                    avatar = member.avatar,
-                    roles = json.dumps(list_role_ids(member.roles))
-                ).apply()
-    
-    async def unban_server_user(self, user, server):
-        await GuildMembers.update.values(banned = False) \
-            .where(GuildMembers.guild_id == int(server.id)) \
-            .where(GuildMembers.user_id == int(user.id)) \
-            .gino.status()
-
-
-    async def flag_unactive_guild_members(self, guild_id, guild_members):
-        async with db.transaction():
-            async for member in GuildMembers.query \
-                .where(GuildMembers.guild_id == int(guild_id)) \
-                .where(GuildMembers.active == True).gino.iterate():
-                dismember = discord.utils.get(guild_members, id=member.user_id)
-                if not dismember:
-                    await member.update(active = False).apply()
-
-    async def flag_unactive_bans(self, guild_id, guildbans):
-        for usr in guildbans:
-            dbusr = await GuildMembers.query \
-                .where(GuildMembers.guild_id == int(guild_id)) \
-                .where(GuildMembers.user_id == int(usr.id)) \
-                .where(GuildMembers.active == False).gino.first()
-            if dbusr:
-                dbusr.update(banned=True).apply()
-            else:
-                await GuildMembers.create(
-                    guild_id = int(guild_id),
-                    user_id = int(usr.id),
-                    username = usr.name,
-                    discriminator = int(usr.discriminator),
-                    nickname = None,
-                    avatar = usr.avatar,
-                    active = False,
-                    banned = True,
-                    roles = "[]"
-                )
-                
     async def ban_unauth_user_by_query(self, guild_id, placer_id, username, discriminator):
         dbuser = None
         if discriminator:

discordbot/titanembeds/database/guild_members.py

@@ -1,14 +0,0 @@
-from titanembeds.database import db
-
-class GuildMembers(db.Model):
-    __tablename__ = "guild_members"
-    id = db.Column(db.Integer, primary_key=True)    # Auto incremented id
-    guild_id = db.Column(db.BigInteger)            # Discord guild id
-    user_id = db.Column(db.BigInteger)             # Discord user id
-    username = db.Column(db.String(255))            # Name
-    discriminator = db.Column(db.Integer)           # User discriminator
-    nickname = db.Column(db.String(255))            # User nickname
-    avatar = db.Column(db.String(255))              # The avatar str of the user
-    active = db.Column(db.Boolean())                # If the user is a member of the guild
-    banned = db.Column(db.Boolean())                # If the user is banned in the guild
-    roles = db.Column(db.Text())                    # Member roles

discordbot/titanembeds/redisqueue.py

@@ -1,4 +1,4 @@
-from titanembeds.utils import get_formatted_message
+from titanembeds.utils import get_formatted_message, get_formatted_user
 from urllib.parse import urlparse
 import asyncio_redis
 import json
@@ -77,6 +77,15 @@ class RedisQueue:
                     break
         return (unformatted_item, formatted_item)
     
+    async def enforce_expiring_key(self, key):
+        ttl = await self.connection.ttl(key)
+        newttl = 0
+        if ttl == -1:
+            newttl = 60 * 5 # 5 minutes
+        if ttl >= 0:
+            newttl = ttl
+        await self.connection.expire(key, newttl)
+    
     async def on_get_channel_messages(self, key, params):
         channel = self.bot.get_channel(int(params["channel_id"]))
         if not channel or not isinstance(channel, discord.channel.TextChannel):
@@ -86,7 +95,7 @@ class RedisQueue:
         async for message in channel.history(limit=50):
             formatted = get_formatted_message(message)
             messages.append(json.dumps(formatted))
-        await self.connection.sadd(key, messages)
+        await self.connection.sadd(key, [""] + messages)
     
     async def push_message(self, message):
         if message.guild:
@@ -108,3 +117,64 @@ class RedisQueue:
     async def update_message(self, message):
         await self.delete_message(message)
         await self.push_message(message)
+
+    async def on_get_guild_member(self, key, params):
+        member = self.bot.get_guild(int(params["guild_id"])).get_member(int(params["user_id"]))
+        if not member:
+            await self.connection.set(key, "")
+            return
+        user = get_formatted_user(member)
+        await self.enforce_expiring_key(key)
+        await self.connection.set(key, json.dumps(user))
+    
+    async def on_get_guild_member_named(self, key, params):
+        guild = self.bot.get_guild(int(params["guild_id"]))
+        query = params["query"]
+        result = None
+        members = guild.members
+        if len(query) > 5 and query[-5] == '#':
+            potential_discriminator = query[-4:]
+            result = discord.utils.get(members, name=query[:-5], discriminator=potential_discriminator)
+            if not result:
+                result = discord.utils.get(members, nick=query[:-5], discriminator=potential_discriminator)
+        if not result:
+            result = ""
+        else:
+            result_id = result.id
+            result = json.dumps({"user_id": result_id})
+            get_guild_member_key = "Queue/guilds/{}/members/{}".format(guild.id, result_id)
+            get_guild_member_param = {"guild_id": guild.id, "user_id": result_id}
+            await self.on_get_guild_member(get_guild_member_key, get_guild_member_param)
+        await self.connection.set(key, result)
+    
+    async def on_list_guild_members(self, key, params):
+        guild = self.bot.get_guild(int(params["guild_id"]))
+        members = guild.members
+        member_ids = []
+        for member in members:
+            member_ids.append(json.dumps({"user_id": member.id}))
+            get_guild_member_key = "Queue/guilds/{}/members/{}".format(guild.id, member.id)
+            get_guild_member_param = {"guild_id": guild.id, "user_id": member.id}
+            await self.on_get_guild_member(get_guild_member_key, get_guild_member_param)
+        await self.connection.sadd(key, member_ids)
+    
+    async def add_member(self, member):
+        key = "Queue/guilds/{}/members".format(member.guild.id)
+        exists = await self.connection.exists(key)
+        if exists:
+            await self.connection.sadd(key, [json.dumps({"user_id": member.id})])
+    
+    async def remove_member(self, member, guild=None):
+        if not guild:
+            guild = member.guild
+        guild_member_key = "Queue/guilds/{}/members/{}".format(guild.id, member.id)
+        list_members_key = "Queue/guilds/{}/members".format(guild.id)
+        await self.connection.srem(list_members_key, [json.dumps({"user_id": member.id})])
+        await self.connection.delete([guild_member_key])
+    
+    async def update_member(self, member):
+        await self.remove_member(member)
+        await self.add_member(member)
+
+    async def ban_member(self, guild, user):
+        await self.remove_member(user, guild)

discordbot/titanembeds/utils.py

@@ -50,6 +50,8 @@ def get_formatted_user(user):
         "status": str(user.status),
         "username": user.name,
         "nick": None,
+        "bot": user.bot,
+        "roles": []
     }
     if userobj["color"] == "000000":
         userobj["color"] = None
@@ -63,13 +65,13 @@ def get_formatted_user(user):
         }
     roles = sorted(user.roles, key=lambda k: k.position, reverse=True)
     for role in roles:
-        if role.hoist:
+        userobj["roles"].append(role.id)
+        if role.hoist and userobj["hoist-role"] == None:
             userobj["hoist-role"] = {
                 "id": str(role.id),
                 "name": role.name,
                 "position": role.position,
             }
-            break
     return userobj
 
 def get_message_author(message):

webapp/alembic/versions/1d2c2dc41e86_removed_guild_members_table.py

@@ -0,0 +1,40 @@
+"""Removed guild members table
+
+Revision ID: 1d2c2dc41e86
+Revises: 52271b243ba2
+Create Date: 2018-07-17 23:45:18.803232
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '1d2c2dc41e86'
+down_revision = '52271b243ba2'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_table('guild_members')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table('guild_members',
+    sa.Column('id', sa.INTEGER(), nullable=False),
+    sa.Column('guild_id', sa.BIGINT(), autoincrement=False, nullable=False),
+    sa.Column('user_id', sa.BIGINT(), autoincrement=False, nullable=False),
+    sa.Column('username', sa.VARCHAR(length=255), autoincrement=False, nullable=False),
+    sa.Column('discriminator', sa.INTEGER(), autoincrement=False, nullable=False),
+    sa.Column('nickname', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
+    sa.Column('avatar', sa.VARCHAR(length=255), autoincrement=False, nullable=True),
+    sa.Column('active', sa.BOOLEAN(), server_default=sa.text('true'), autoincrement=False, nullable=False),
+    sa.Column('banned', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=False),
+    sa.Column('roles', sa.TEXT(), autoincrement=False, nullable=False),
+    sa.PrimaryKeyConstraint('id', name='idx_25210_primary')
+    )
+    # ### end Alembic commands ###

webapp/titanembeds/blueprints/admin/admin.py

@@ -1,9 +1,9 @@
 from flask import Blueprint, url_for, redirect, session, render_template, abort, request, jsonify
 from flask_socketio import emit
 from functools import wraps
-from titanembeds.database import db, get_administrators_list, Cosmetics, Guilds, UnauthenticatedUsers, UnauthenticatedBans, TitanTokens, TokenTransactions, get_titan_token, set_titan_token, list_disabled_guilds, DisabledGuilds, UserCSS, AuthenticatedUsers, DiscordBotsOrgTransactions, GuildMembers
+from titanembeds.database import db, get_administrators_list, Cosmetics, Guilds, UnauthenticatedUsers, UnauthenticatedBans, TitanTokens, TokenTransactions, get_titan_token, set_titan_token, list_disabled_guilds, DisabledGuilds, UserCSS, AuthenticatedUsers, DiscordBotsOrgTransactions
 from titanembeds.oauth import generate_guild_icon_url
-from titanembeds.utils import get_online_embed_user_keys
+from titanembeds.utils import get_online_embed_user_keys, redisqueue
 import datetime
 import json
 from sqlalchemy import func

webapp/titanembeds/blueprints/api/api.py

@@ -1,4 +1,4 @@
-from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, list_all_guild_members, get_guild_member, get_administrators_list, get_badges, DiscordBotsOrgTransactions
+from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, get_administrators_list, get_badges, DiscordBotsOrgTransactions
 from titanembeds.decorators import valid_session_required, discord_users_only, abort_if_guild_disabled
 from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles, get_online_embed_user_keys, redis_store, redisqueue
 from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild
@@ -79,8 +79,8 @@ def format_post_content(guild_id, channel_id, message, dbUser):
         else:
             username = session['username']
             if dbUser:
-                if dbUser.nickname:
-                    username = dbUser.nickname
+                if dbUser["nick"]:
+                    username = dbUser["nick"]
             message = u"**<{}#{}>** {}".format(username, session['discriminator'], message) # I would like to do a @ mention, but i am worried about notify spam
     return (message, illegal_post, illegal_reasons)
 
@@ -100,10 +100,10 @@ def filter_guild_channel(guild_id, channel_id, force_everyone=False):
     return None
 
 def get_online_discord_users(guild_id, embed):
-    apimembers = list_all_guild_members(guild_id)
+    apimembers = redisqueue.list_guild_members(guild_id)
     apimembers_filtered = {}
     for member in apimembers:
-        apimembers_filtered[member["user"]["id"]] = member
+        apimembers_filtered[member["id"]] = member
     guild_roles = json.loads(db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first().roles)
     guildroles_filtered = {}
     for role in guild_roles:
@@ -151,13 +151,13 @@ def get_online_embed_users(guild_id):
         users['unauthenticated'].append(meta)
     for user in auths:
         client_id = user.client_id
-        usrdb = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == client_id).first()
+        usrdb = redisqueue.get_guild_member(guild_id, client_id)
         meta = {
-            'id': str(usrdb.user_id),
-            'username': usrdb.username,
-            'nickname': usrdb.nickname,
-            'discriminator': usrdb.discriminator,
-            'avatar_url': generate_avatar_url(usrdb.user_id, usrdb.avatar),
+            'id': str(usrdb["id"]),
+            'username': usrdb["username"],
+            'nickname': usrdb["nick"],
+            'discriminator': usrdb["discriminator"],
+            'avatar_url': generate_avatar_url(usrdb["id"], usrdb["avatar"]),
         }
         users['authenticated'].append(meta)
     return users
@@ -194,16 +194,16 @@ def get_channel_webhook_url(guild_id, channel_id):
     return webhook["content"]
 
 def get_all_users(guild_id):
-    users = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.active == True).all()
+    users = redisqueue.list_guild_members(guild_id)
     mem = []
     for u in users:
         mem.append({
-            "id": str(u.user_id),
-            "avatar": u.avatar,
-            "avatar_url": generate_avatar_url(u.user_id, u.avatar, u.discriminator, True),
-            "username": u.username,
-            "nickname": u.nickname,
-            "discriminator": u.discriminator
+            "id": str(u["id"]),
+            "avatar": u["avatar"],
+            "avatar_url": generate_avatar_url(u["id"], u["avatar"], u["discriminator"], True),
+            "username": u["username"],
+            "nickname": u["nick"],
+            "discriminator": u["discriminator"]
         })
     return mem
 
@@ -295,7 +295,7 @@ def post():
     channel_id = request.form.get('channel_id')
     content = request.form.get('content')
     if "user_id" in session:
-        dbUser = GuildMembers.query.filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == str(session['user_id'])).first()
+        dbUser = redisqueue.get_guild_member(guild_id, session["user_id"])
     else:
         dbUser = None
     if user_unauthenticated():
@@ -331,8 +331,8 @@ def post():
                 else:
                     username = session["username"]
                     if dbUser:
-                        if dbUser.nickname:
-                            username = dbUser.nickname
+                        if dbUser["nick"]:
+                            username = dbUser["nick"]
                     # if content.startswith("(Titan Dev) "):
                     #     content = content[12:]
                     #     username = "(Titan Dev) " + username
@@ -527,13 +527,13 @@ def user_info(guild_id, user_id):
         "roles": [],
         "badges": [],
     }
-    member = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.user_id == user_id).first()
+    member = redisqueue.get_guild_member(guild_id, user_id)
     if member:
-        usr["id"] = str(member.user_id)
-        usr["username"] = member.username
-        usr["nickname"] = member.nickname
-        usr["discriminator"] = member.discriminator
-        usr["avatar"] = member.avatar
+        usr["id"] = str(member["id"])
+        usr["username"] = member["username"]
+        usr["nickname"] = member["nick"]
+        usr["discriminator"] = member["discriminator"]
+        usr["avatar"] = member["avatar"]
         usr["avatar_url"] = generate_avatar_url(usr["id"], usr["avatar"], usr["discriminator"], True)
         roles = get_member_roles(guild_id, user_id)
         dbguild = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first()
@@ -543,7 +543,7 @@ def user_info(guild_id, user_id):
                 if gr["id"] == r:
                     usr["roles"].append(gr)
         usr["badges"] = get_badges(user_id)
-        if redis_store.get("DiscordBotsOrgVoted/" + str(member.user_id)):
+        if redis_store.get("DiscordBotsOrgVoted/" + str(member["id"])):
             usr["badges"].append("discordbotsorgvoted")
     return jsonify(usr)
     

webapp/titanembeds/blueprints/gateway/gateway.py

@@ -1,5 +1,5 @@
-from titanembeds.utils import socketio, guild_accepts_visitors, get_client_ipaddr, discord_api, check_user_in_guild, get_guild_channels, update_user_status, guild_webhooks_enabled, redis_store
-from titanembeds.database import db, GuildMembers, get_guild_member, Guilds
+from titanembeds.utils import socketio, guild_accepts_visitors, get_client_ipaddr, discord_api, check_user_in_guild, get_guild_channels, update_user_status, guild_webhooks_enabled, redis_store, redisqueue
+from titanembeds.database import db, Guilds
 from flask_socketio import Namespace, emit, disconnect, join_room, leave_room
 import functools
 from flask import request, session
@@ -38,7 +38,7 @@ class Gateway(Namespace):
             if session["unauthenticated"]:
                 emit("embed_user_connect", {"unauthenticated": True, "username": session["username"], "discriminator": session["user_id"]}, room="GUILD_"+guild_id)
             else:
-                nickname = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.user_id == session["user_id"]).first().nickname
+                nickname = redisqueue.get_guild_member(guild_id, session["user_id"]).get("nickname")
                 emit("embed_user_connect", {"unauthenticated": False, "id": str(session["user_id"]), "nickname": nickname, "username": session["username"],"discriminator": session["discriminator"], "avatar_url": session["avatar"]}, room="GUILD_"+guild_id)
         emit("identified")
         self.teardown_db_session()
@@ -125,17 +125,17 @@ class Gateway(Namespace):
         
     def get_user_color(self, guild_id, user_id):
         color = None
-        member = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.user_id == user_id).first()
+        member = redisqueue.get_guild_member(guild_id, user_id)
         if not member:
             return None
         guild_roles = json.loads(db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first().roles)
         guildroles_filtered = {}
         for role in guild_roles:
             guildroles_filtered[role["id"]] = role
-        member_roleids = json.loads(member.roles)
+        member_roleids = member["roles"]
         member_roles = []
         for roleid in member_roleids:
-            role = guildroles_filtered.get(roleid)
+            role = guildroles_filtered.get(str(roleid))
             if not role:
                 continue
             member_roles.append(role)
@@ -148,6 +148,7 @@ class Gateway(Namespace):
         return color
     
     def on_lookup_user_info(self, data):
+        print("test")
         guild_id = data["guild_id"]
         name = data["name"]
         discriminator = data["discriminator"]
@@ -162,28 +163,36 @@ class Gateway(Namespace):
             "avatar_url": None,
             "discordbotsorgvoted": False,
         }
-        member = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.username == name, GuildMembers.discriminator == discriminator).first()
+        member = redisqueue.get_guild_member_named(guild_id, "{}#{}".format(name, discriminator))
+        print("test1")
         if member:
-            usr["id"] = str(member.user_id)
-            usr["username"] = member.username
-            usr["nickname"] = member.nickname
-            usr["avatar"] = member.avatar
+            print("test1.5")
+            usr["id"] = str(member["id"])
+            usr["username"] = member["username"]
+            usr["nickname"] = member["nick"]
+            usr["avatar"] = member["avatar"]
+            print("test1.54")
             usr["color"] = self.get_user_color(guild_id, usr["id"])
+            print("test1.55")
             if (usr["avatar"]):
                 usr["avatar_url"] = "https://cdn.discordapp.com/avatars/{}/{}.png".format(usr["id"], usr["avatar"])
-            usr["roles"] = json.loads(member.roles)
-            usr["discordbotsorgvoted"] = bool(redis_store.get("DiscordBotsOrgVoted/" + str(member.user_id)))
+            usr["roles"] = member["roles"]
+            usr["discordbotsorgvoted"] = bool(redis_store.get("DiscordBotsOrgVoted/" + str(member["id"])))
+            print("test1.6")
         else:
-            member = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.nickname == name, GuildMembers.discriminator == discriminator).first()
+            print("test2")
+            member = redisqueue.get_guild_member_named(guild_id, name)
+            print("test3")
             if member:
-                usr["id"] = str(member.user_id)
-                usr["username"] = member.username
-                usr["nickname"] = member.nickname
-                usr["avatar"] = member.avatar
+                usr["id"] = str(member["id"])
+                usr["username"] = member["username"]
+                usr["nickname"] = member["nick"]
+                usr["avatar"] = member["avatar"]
                 usr["color"] = self.get_user_color(guild_id, usr["id"])
                 if (usr["avatar"]):
                     usr["avatar_url"] = "https://cdn.discordapp.com/avatars/{}/{}.png".format(usr["id"], usr["avatar"])
-                usr["roles"] = json.loads(member.roles)
-                usr["discordbotsorgvoted"] = bool(redis_store.get("DiscordBotsOrgVoted/" + str(member.user_id)))
+                usr["roles"] = member["roles"]
+                usr["discordbotsorgvoted"] = bool(redis_store.get("DiscordBotsOrgVoted/" + str(member["id"])))
+        print("test4")
         emit("lookup_user_info", usr)
         self.teardown_db_session()

webapp/titanembeds/database/__init__.py

@@ -6,7 +6,6 @@ from .guilds import Guilds
 from .unauthenticated_users import UnauthenticatedUsers
 from .unauthenticated_bans import UnauthenticatedBans
 from .authenticated_users import AuthenticatedUsers
-from .guild_members import GuildMembers, list_all_guild_members, get_guild_member
 from .cosmetics import Cosmetics, set_badges, get_badges, add_badge, remove_badge
 from .user_css import UserCSS
 from .administrators import Administrators, get_administrators_list

webapp/titanembeds/database/guild_members.py

@@ -1,48 +0,0 @@
-from titanembeds.database import db
-import json
-
-class GuildMembers(db.Model):
-    __tablename__ = "guild_members"
-    id = db.Column(db.Integer, primary_key=True)                    # Auto incremented id
-    guild_id = db.Column(db.BigInteger, nullable=False)            # Discord guild id
-    user_id = db.Column(db.BigInteger, nullable=False)             # Discord user id
-    username = db.Column(db.String(255), nullable=False)            # Name
-    discriminator = db.Column(db.Integer, nullable=False)           # User discriminator
-    nickname = db.Column(db.String(255))                            # User nickname
-    avatar = db.Column(db.String(255))                              # The avatar str of the user
-    active = db.Column(db.Boolean(), nullable=False)                # If the user is a member of the guild
-    banned = db.Column(db.Boolean(), nullable=False)                # If the user is banned in the guild
-    roles = db.Column(db.Text(), nullable=False)                    # Member roles
-
-    def __init__(self, guild_id, user_id, username, discriminator, nickname, avatar, active, banned, roles):
-        self.guild_id = guild_id
-        self.user_id = user_id
-        self.username = username
-        self.discriminator = discriminator
-        self.nickname = nickname
-        self.avatar = avatar
-        self.active = active
-        self.banned = banned
-        self.roles = roles
-
-    def __repr__(self):
-        return '<GuildMembers {0} {1} {2} {3} {4}>'.format(self.id, self.guild_id, self.user_id, self.username, self.discriminator)
-
-def list_all_guild_members(guild_id):
-    memlist = []
-    members = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id).all()
-    for member in members:
-        memlist.append({
-            "user": {
-                "username": member.username,
-                "discriminator": member.discriminator,
-                "id": member.user_id,
-                "avatar": member.avatar
-            },
-            "roles": json.loads(member.roles),
-            "nickname": member.nickname,
-        })
-    return memlist
-
-def get_guild_member(guild_id, member_id):
-    return db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.user_id == member_id).first()

webapp/titanembeds/redisqueue.py

@@ -1,5 +1,4 @@
 from titanembeds.utils import redis_store
-from titanembeds.database import get_guild_member
 import json
 import time
 
@@ -16,20 +15,21 @@ class RedisQueue:
             "params": params
         }
         loop_count = 0
-        while not data and loop_count < 10:
-            if loop_count % 5 == 0:
+        while (not data and data != "") and loop_count < 50:
+            if loop_count % 25 == 0:
                 redis_store.publish("discord-api-req", json.dumps(payload))
-            time.sleep(0.5)
+            time.sleep(0.1)
             data = self._get(key, data_type)
             loop_count += 1
         redis_store.expire(key, 60 * 5)
-        if data == None:
+        if data == None or data == "":
             return None
         if data_type == "set":
             data = list(data)
             data_parsed = []
             for d in data:
-                data_parsed.append(json.loads(d))
+                if d != "":
+                    data_parsed.append(json.loads(d))
             return data_parsed
         return json.loads(data)
     
@@ -61,28 +61,51 @@ class RedisQueue:
                 "embeds": x["embeds"],
             }
             if message["author"]["id"] not in guild_members:
-                member = get_guild_member(guild_id, message["author"]["id"])
+                member = self.get_guild_member(guild_id, message["author"]["id"])
                 guild_members[message["author"]["id"]] = member
             else:
                 member = guild_members[message["author"]["id"]]
             message["author"]["nickname"] = None
             if member:
-                message["author"]["nickname"] = member.nickname
-                message["author"]["avatar"] = member.avatar
-                message["author"]["discriminator"] = member.discriminator
-                message["author"]["username"] = member.username
+                message["author"]["nickname"] = member["nick"]
+                message["author"]["avatar"] = member["avatar"]
+                message["author"]["discriminator"] = member["discriminator"]
+                message["author"]["username"] = member["username"]
             for mention in message["mentions"]:
                 if mention["id"] not in guild_members:
-                    author = get_guild_member(guild_id, mention["id"])
+                    author = self.get_guild_member(guild_id, mention["id"])
                     guild_members[mention["id"]] = author
                 else:
                     author = guild_members[mention["id"]]
                 mention["nickname"] = None
                 if author:
-                    mention["nickname"] = author.nickname
-                    mention["avatar"] = author.avatar
-                    mention["username"] = author.username
-                    mention["discriminator"] = author.discriminator
+                    mention["nickname"] = author["nick"]
+                    mention["avatar"] = author["avatar"]
+                    mention["username"] = author["username"]
+                    mention["discriminator"] = author["discriminator"]
             msgs.append(message)
         sorted_msgs = sorted(msgs, key=lambda k: k['id'], reverse=True) 
         return sorted_msgs
+
+    def get_guild_member(self, guild_id, user_id):
+        key = "/guilds/{}/members/{}".format(guild_id, user_id)
+        q = self.get(key, "get_guild_member", {"guild_id": guild_id, "user_id": user_id})
+        return q
+    
+    def get_guild_member_named(self, guild_id, query):
+        key = "/custom/guilds/{}/member_named/{}".format(guild_id, query)
+        guild_member_id = self.get(key, "get_guild_member_named", {"guild_id": guild_id, "query": query})
+        if guild_member_id:
+            return self.get_guild_member(guild_id, guild_member_id["user_id"])
+        return None
+    
+    def list_guild_members(self, guild_id):
+        key = "/guilds/{}/members".format(guild_id)
+        member_ids = self.get(key, "list_guild_members", {"guild_id": guild_id}, data_type="set")
+        members = []
+        for member_id in member_ids:
+            usr_id = member_id["user_id"]
+            member = self.get_guild_member(guild_id, usr_id)
+            if member:
+                members.append(member)
+        return members

webapp/titanembeds/utils.py

@@ -1,4 +1,4 @@
-from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, get_guild_member
+from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers
 from titanembeds.constants import LANGUAGES
 from flask import request, session
 from flask_limiter import Limiter
@@ -98,8 +98,8 @@ def checkUserRevoke(guild_id, user_key=None):
         banned = checkUserBanned(guild_id)
         if banned:
             return revoked
-        dbUser = GuildMembers.query.filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == session["user_id"]).first()
-        revoked = not dbUser or not dbUser.active
+        dbUser = redisqueue.get_guild_member(guild_id, session["user_id"])
+        revoked = not dbUser
     return revoked
     
 def checkUserBanned(guild_id, ip_address=None):
@@ -114,11 +114,9 @@ def checkUserBanned(guild_id, ip_address=None):
                     banned = False
     else:
         banned = False
-        dbUser = GuildMembers.query.filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == session["user_id"]).first()
-        if not dbUser:
-            banned = False
-        else:
-            banned = dbUser.banned
+        #dbUser = redisqueue.get_guild_member(guild_id, session["user_id"])
+        #if not dbUser:
+        #    banned = True # TODO: Figure out ban logic with guild member
     return banned
 
 from titanembeds.oauth import check_user_can_administrate_guild, user_has_permission
@@ -163,9 +161,9 @@ def update_user_status(guild_id, username, user_key=None):
         }
         if status['banned'] or status['revoked']:
             return status
-        dbMember = get_guild_member(guild_id, status["user_id"])
+        dbMember = redisqueue.get_guild_member(guild_id, status["user_id"])
         if dbMember:
-            status["nickname"] = dbMember.nickname
+            status["nickname"] = dbMember["nick"]
         bump_user_presence_timestamp(guild_id, "AuthenticatedUsers", status["user_id"])
     return status
 
@@ -195,18 +193,8 @@ def check_user_in_guild(guild_id):
         return dbUser is not None and not checkUserRevoke(guild_id)
 
 def get_member_roles(guild_id, user_id):
-    q = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == user_id).first()
-    roles = [guild_id]
-    if not q:
-        member = discord_api.get_guild_member(guild_id, user_id)
-        if member["success"]:
-            roles = member["content"]["roles"]
-            member = GuildMembers(guild_id, user_id, member["content"]["user"]["username"], int(member["content"]["user"]["discriminator"]), member["content"].get("nick", None), member["content"]["user"]["avatar"], True, False, json.dumps(roles))
-            db.session.add(member)
-            db.session.commit()
-    else:
-        roles = json.loads(q.roles)
-    return roles
+    q = redisqueue.get_guild_member(guild_id, user_id)
+    return q["roles"]
 
 def get_guild_channels(guild_id, force_everyone=False):
     if user_unauthenticated() or force_everyone: