From 86c7232b35b4ab216aa9bc3abea8aabeef5045b9 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Tue, 17 Jul 2018 23:47:30 +0000 Subject: [PATCH] Remove guild members table for storing in redis --- discordbot/titanembeds/bot.py | 28 +------ discordbot/titanembeds/database/__init__.py | 77 ------------------- .../titanembeds/database/guild_members.py | 14 ---- discordbot/titanembeds/redisqueue.py | 76 +++++++++++++++++- discordbot/titanembeds/utils.py | 6 +- ...d2c2dc41e86_removed_guild_members_table.py | 40 ++++++++++ webapp/titanembeds/blueprints/admin/admin.py | 4 +- webapp/titanembeds/blueprints/api/api.py | 56 +++++++------- .../titanembeds/blueprints/gateway/gateway.py | 49 +++++++----- webapp/titanembeds/database/__init__.py | 1 - webapp/titanembeds/database/guild_members.py | 48 ------------ webapp/titanembeds/redisqueue.py | 57 ++++++++++---- webapp/titanembeds/utils.py | 32 +++----- 13 files changed, 230 insertions(+), 258 deletions(-) delete mode 100644 discordbot/titanembeds/database/guild_members.py create mode 100644 webapp/alembic/versions/1d2c2dc41e86_removed_guild_members_table.py delete mode 100644 webapp/titanembeds/database/guild_members.py diff --git a/discordbot/titanembeds/bot.py b/discordbot/titanembeds/bot.py index eb58613..6fc3b99 100644 --- a/discordbot/titanembeds/bot.py +++ b/discordbot/titanembeds/bot.py @@ -105,23 +105,6 @@ class Titan(discord.AutoShardedClient): async def on_guild_join(self, guild): await self.database.update_guild(guild) - for member in guild.members: - await self.database.update_guild_member(member, True, False) - if guild.me.guild_permissions.ban_members: - banned = await guild.bans() - for ban in banned: - member = discord.Member(user={ - "username": ban.name, - "id": ban.id, - "discriminator": ban.discriminator, - "avatar": ban.avatar, - "bot": ban.bot - }) - await self.database.update_guild_member(member, False, True) - for channel in list(guild.channels): - chanperm = channel.permissions_for(channel.guild.me) - if not chanperm.read_messages or not chanperm.read_message_history or not isinstance(channel, discord.channel.TextChannel): - continue await self.postStats() async def on_guild_remove(self, guild): @@ -163,24 +146,21 @@ class Titan(discord.AutoShardedClient): await self.socketio.on_channel_update(channelafter) async def on_member_join(self, member): - await self.database.update_guild_member(member, active=True, banned=False) + await self.redisqueue.add_member(member) await self.socketio.on_guild_member_add(member) async def on_member_remove(self, member): - await self.database.update_guild_member(member, active=False, banned=False) + await self.redisqueue.remove_member(member) await self.socketio.on_guild_member_remove(member) async def on_member_update(self, memberbefore, memberafter): - await self.database.update_guild_member(memberafter) + await self.redisqueue.update_member(memberafter) await self.socketio.on_guild_member_update(memberafter) async def on_member_ban(self, guild, user): if self.user.id == user.id: return - await self.database.update_guild_member(user, active=False, banned=True, guild=guild) - - async def on_member_unban(self, guild, user): - await self.database.unban_server_user(user, guild) + await self.redisqueue.ban_member(guild, user) async def on_guild_emojis_update(self, guild, before, after): if len(after) == 0: diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index 53daf22..3415d5d 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -6,7 +6,6 @@ import datetime db = Gino() from titanembeds.database.guilds import Guilds -from titanembeds.database.guild_members import GuildMembers from titanembeds.database.unauthenticated_users import UnauthenticatedUsers from titanembeds.database.unauthenticated_bans import UnauthenticatedBans @@ -70,82 +69,6 @@ class DatabaseInterface(object): if gui: await Messages.delete.where(Messages.guild_id == int(guild.id)).gino.status() await gui.delete() - - async def update_guild_member(self, member, active=True, banned=False, guild=None): - if guild: - dbmember = await GuildMembers.query \ - .where(GuildMembers.guild_id == int(guild.id)) \ - .where(GuildMembers.user_id == int(member.id)) \ - .order_by(GuildMembers.id).gino.all() - else: - dbmember = await GuildMembers.query \ - .where(GuildMembers.guild_id == int(member.guild.id)) \ - .where(GuildMembers.user_id == int(member.id)) \ - .order_by(GuildMembers.id).gino.all() - if not dbmember: - await GuildMembers.create( - guild_id = int(member.guild.id), - user_id = int(member.id), - username = member.name, - discriminator = int(member.discriminator), - nickname = member.nick, - avatar = member.avatar, - active = active, - banned = banned, - roles = json.dumps(list_role_ids(member.roles)) - ) - else: - if len(dbmember) > 1: - for mem in dbmember[1:]: - await mem.delete() - dbmember = dbmember[0] - if dbmember.banned != banned or dbmember.active != active or dbmember.username != member.name or dbmember.discriminator != int(member.discriminator) or dbmember.nickname != member.nick or dbmember.avatar != member.avatar or set(json.loads(dbmember.roles)) != set(list_role_ids(member.roles)): - await dbmember.update( - banned = banned, - active = active, - username = member.name, - discriminator = int(member.discriminator), - nickname = member.nick, - avatar = member.avatar, - roles = json.dumps(list_role_ids(member.roles)) - ).apply() - - async def unban_server_user(self, user, server): - await GuildMembers.update.values(banned = False) \ - .where(GuildMembers.guild_id == int(server.id)) \ - .where(GuildMembers.user_id == int(user.id)) \ - .gino.status() - - - async def flag_unactive_guild_members(self, guild_id, guild_members): - async with db.transaction(): - async for member in GuildMembers.query \ - .where(GuildMembers.guild_id == int(guild_id)) \ - .where(GuildMembers.active == True).gino.iterate(): - dismember = discord.utils.get(guild_members, id=member.user_id) - if not dismember: - await member.update(active = False).apply() - - async def flag_unactive_bans(self, guild_id, guildbans): - for usr in guildbans: - dbusr = await GuildMembers.query \ - .where(GuildMembers.guild_id == int(guild_id)) \ - .where(GuildMembers.user_id == int(usr.id)) \ - .where(GuildMembers.active == False).gino.first() - if dbusr: - dbusr.update(banned=True).apply() - else: - await GuildMembers.create( - guild_id = int(guild_id), - user_id = int(usr.id), - username = usr.name, - discriminator = int(usr.discriminator), - nickname = None, - avatar = usr.avatar, - active = False, - banned = True, - roles = "[]" - ) async def ban_unauth_user_by_query(self, guild_id, placer_id, username, discriminator): dbuser = None diff --git a/discordbot/titanembeds/database/guild_members.py b/discordbot/titanembeds/database/guild_members.py deleted file mode 100644 index 5197039..0000000 --- a/discordbot/titanembeds/database/guild_members.py +++ /dev/null @@ -1,14 +0,0 @@ -from titanembeds.database import db - -class GuildMembers(db.Model): - __tablename__ = "guild_members" - id = db.Column(db.Integer, primary_key=True) # Auto incremented id - guild_id = db.Column(db.BigInteger) # Discord guild id - user_id = db.Column(db.BigInteger) # Discord user id - username = db.Column(db.String(255)) # Name - discriminator = db.Column(db.Integer) # User discriminator - nickname = db.Column(db.String(255)) # User nickname - avatar = db.Column(db.String(255)) # The avatar str of the user - active = db.Column(db.Boolean()) # If the user is a member of the guild - banned = db.Column(db.Boolean()) # If the user is banned in the guild - roles = db.Column(db.Text()) # Member roles \ No newline at end of file diff --git a/discordbot/titanembeds/redisqueue.py b/discordbot/titanembeds/redisqueue.py index 132a3c6..05fea91 100644 --- a/discordbot/titanembeds/redisqueue.py +++ b/discordbot/titanembeds/redisqueue.py @@ -1,4 +1,4 @@ -from titanembeds.utils import get_formatted_message +from titanembeds.utils import get_formatted_message, get_formatted_user from urllib.parse import urlparse import asyncio_redis import json @@ -77,6 +77,15 @@ class RedisQueue: break return (unformatted_item, formatted_item) + async def enforce_expiring_key(self, key): + ttl = await self.connection.ttl(key) + newttl = 0 + if ttl == -1: + newttl = 60 * 5 # 5 minutes + if ttl >= 0: + newttl = ttl + await self.connection.expire(key, newttl) + async def on_get_channel_messages(self, key, params): channel = self.bot.get_channel(int(params["channel_id"])) if not channel or not isinstance(channel, discord.channel.TextChannel): @@ -86,7 +95,7 @@ class RedisQueue: async for message in channel.history(limit=50): formatted = get_formatted_message(message) messages.append(json.dumps(formatted)) - await self.connection.sadd(key, messages) + await self.connection.sadd(key, [""] + messages) async def push_message(self, message): if message.guild: @@ -107,4 +116,65 @@ class RedisQueue: async def update_message(self, message): await self.delete_message(message) - await self.push_message(message) \ No newline at end of file + await self.push_message(message) + + async def on_get_guild_member(self, key, params): + member = self.bot.get_guild(int(params["guild_id"])).get_member(int(params["user_id"])) + if not member: + await self.connection.set(key, "") + return + user = get_formatted_user(member) + await self.enforce_expiring_key(key) + await self.connection.set(key, json.dumps(user)) + + async def on_get_guild_member_named(self, key, params): + guild = self.bot.get_guild(int(params["guild_id"])) + query = params["query"] + result = None + members = guild.members + if len(query) > 5 and query[-5] == '#': + potential_discriminator = query[-4:] + result = discord.utils.get(members, name=query[:-5], discriminator=potential_discriminator) + if not result: + result = discord.utils.get(members, nick=query[:-5], discriminator=potential_discriminator) + if not result: + result = "" + else: + result_id = result.id + result = json.dumps({"user_id": result_id}) + get_guild_member_key = "Queue/guilds/{}/members/{}".format(guild.id, result_id) + get_guild_member_param = {"guild_id": guild.id, "user_id": result_id} + await self.on_get_guild_member(get_guild_member_key, get_guild_member_param) + await self.connection.set(key, result) + + async def on_list_guild_members(self, key, params): + guild = self.bot.get_guild(int(params["guild_id"])) + members = guild.members + member_ids = [] + for member in members: + member_ids.append(json.dumps({"user_id": member.id})) + get_guild_member_key = "Queue/guilds/{}/members/{}".format(guild.id, member.id) + get_guild_member_param = {"guild_id": guild.id, "user_id": member.id} + await self.on_get_guild_member(get_guild_member_key, get_guild_member_param) + await self.connection.sadd(key, member_ids) + + async def add_member(self, member): + key = "Queue/guilds/{}/members".format(member.guild.id) + exists = await self.connection.exists(key) + if exists: + await self.connection.sadd(key, [json.dumps({"user_id": member.id})]) + + async def remove_member(self, member, guild=None): + if not guild: + guild = member.guild + guild_member_key = "Queue/guilds/{}/members/{}".format(guild.id, member.id) + list_members_key = "Queue/guilds/{}/members".format(guild.id) + await self.connection.srem(list_members_key, [json.dumps({"user_id": member.id})]) + await self.connection.delete([guild_member_key]) + + async def update_member(self, member): + await self.remove_member(member) + await self.add_member(member) + + async def ban_member(self, guild, user): + await self.remove_member(user, guild) \ No newline at end of file diff --git a/discordbot/titanembeds/utils.py b/discordbot/titanembeds/utils.py index b8a234a..9cd204f 100644 --- a/discordbot/titanembeds/utils.py +++ b/discordbot/titanembeds/utils.py @@ -50,6 +50,8 @@ def get_formatted_user(user): "status": str(user.status), "username": user.name, "nick": None, + "bot": user.bot, + "roles": [] } if userobj["color"] == "000000": userobj["color"] = None @@ -63,13 +65,13 @@ def get_formatted_user(user): } roles = sorted(user.roles, key=lambda k: k.position, reverse=True) for role in roles: - if role.hoist: + userobj["roles"].append(role.id) + if role.hoist and userobj["hoist-role"] == None: userobj["hoist-role"] = { "id": str(role.id), "name": role.name, "position": role.position, } - break return userobj def get_message_author(message): diff --git a/webapp/alembic/versions/1d2c2dc41e86_removed_guild_members_table.py b/webapp/alembic/versions/1d2c2dc41e86_removed_guild_members_table.py new file mode 100644 index 0000000..eac35d6 --- /dev/null +++ b/webapp/alembic/versions/1d2c2dc41e86_removed_guild_members_table.py @@ -0,0 +1,40 @@ +"""Removed guild members table + +Revision ID: 1d2c2dc41e86 +Revises: 52271b243ba2 +Create Date: 2018-07-17 23:45:18.803232 + +""" + +# revision identifiers, used by Alembic. +revision = '1d2c2dc41e86' +down_revision = '52271b243ba2' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('guild_members') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('guild_members', + sa.Column('id', sa.INTEGER(), nullable=False), + sa.Column('guild_id', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('user_id', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('username', sa.VARCHAR(length=255), autoincrement=False, nullable=False), + sa.Column('discriminator', sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column('nickname', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.Column('avatar', sa.VARCHAR(length=255), autoincrement=False, nullable=True), + sa.Column('active', sa.BOOLEAN(), server_default=sa.text('true'), autoincrement=False, nullable=False), + sa.Column('banned', sa.BOOLEAN(), server_default=sa.text('false'), autoincrement=False, nullable=False), + sa.Column('roles', sa.TEXT(), autoincrement=False, nullable=False), + sa.PrimaryKeyConstraint('id', name='idx_25210_primary') + ) + # ### end Alembic commands ### diff --git a/webapp/titanembeds/blueprints/admin/admin.py b/webapp/titanembeds/blueprints/admin/admin.py index c6696a5..029987e 100644 --- a/webapp/titanembeds/blueprints/admin/admin.py +++ b/webapp/titanembeds/blueprints/admin/admin.py @@ -1,9 +1,9 @@ from flask import Blueprint, url_for, redirect, session, render_template, abort, request, jsonify from flask_socketio import emit from functools import wraps -from titanembeds.database import db, get_administrators_list, Cosmetics, Guilds, UnauthenticatedUsers, UnauthenticatedBans, TitanTokens, TokenTransactions, get_titan_token, set_titan_token, list_disabled_guilds, DisabledGuilds, UserCSS, AuthenticatedUsers, DiscordBotsOrgTransactions, GuildMembers +from titanembeds.database import db, get_administrators_list, Cosmetics, Guilds, UnauthenticatedUsers, UnauthenticatedBans, TitanTokens, TokenTransactions, get_titan_token, set_titan_token, list_disabled_guilds, DisabledGuilds, UserCSS, AuthenticatedUsers, DiscordBotsOrgTransactions from titanembeds.oauth import generate_guild_icon_url -from titanembeds.utils import get_online_embed_user_keys +from titanembeds.utils import get_online_embed_user_keys, redisqueue import datetime import json from sqlalchemy import func diff --git a/webapp/titanembeds/blueprints/api/api.py b/webapp/titanembeds/blueprints/api/api.py index 1f241e6..0e3a03b 100644 --- a/webapp/titanembeds/blueprints/api/api.py +++ b/webapp/titanembeds/blueprints/api/api.py @@ -1,4 +1,4 @@ -from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, list_all_guild_members, get_guild_member, get_administrators_list, get_badges, DiscordBotsOrgTransactions +from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, get_administrators_list, get_badges, DiscordBotsOrgTransactions from titanembeds.decorators import valid_session_required, discord_users_only, abort_if_guild_disabled from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles, get_online_embed_user_keys, redis_store, redisqueue from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild @@ -79,8 +79,8 @@ def format_post_content(guild_id, channel_id, message, dbUser): else: username = session['username'] if dbUser: - if dbUser.nickname: - username = dbUser.nickname + if dbUser["nick"]: + username = dbUser["nick"] message = u"**<{}#{}>** {}".format(username, session['discriminator'], message) # I would like to do a @ mention, but i am worried about notify spam return (message, illegal_post, illegal_reasons) @@ -100,10 +100,10 @@ def filter_guild_channel(guild_id, channel_id, force_everyone=False): return None def get_online_discord_users(guild_id, embed): - apimembers = list_all_guild_members(guild_id) + apimembers = redisqueue.list_guild_members(guild_id) apimembers_filtered = {} for member in apimembers: - apimembers_filtered[member["user"]["id"]] = member + apimembers_filtered[member["id"]] = member guild_roles = json.loads(db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first().roles) guildroles_filtered = {} for role in guild_roles: @@ -151,13 +151,13 @@ def get_online_embed_users(guild_id): users['unauthenticated'].append(meta) for user in auths: client_id = user.client_id - usrdb = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == client_id).first() + usrdb = redisqueue.get_guild_member(guild_id, client_id) meta = { - 'id': str(usrdb.user_id), - 'username': usrdb.username, - 'nickname': usrdb.nickname, - 'discriminator': usrdb.discriminator, - 'avatar_url': generate_avatar_url(usrdb.user_id, usrdb.avatar), + 'id': str(usrdb["id"]), + 'username': usrdb["username"], + 'nickname': usrdb["nick"], + 'discriminator': usrdb["discriminator"], + 'avatar_url': generate_avatar_url(usrdb["id"], usrdb["avatar"]), } users['authenticated'].append(meta) return users @@ -194,16 +194,16 @@ def get_channel_webhook_url(guild_id, channel_id): return webhook["content"] def get_all_users(guild_id): - users = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.active == True).all() + users = redisqueue.list_guild_members(guild_id) mem = [] for u in users: mem.append({ - "id": str(u.user_id), - "avatar": u.avatar, - "avatar_url": generate_avatar_url(u.user_id, u.avatar, u.discriminator, True), - "username": u.username, - "nickname": u.nickname, - "discriminator": u.discriminator + "id": str(u["id"]), + "avatar": u["avatar"], + "avatar_url": generate_avatar_url(u["id"], u["avatar"], u["discriminator"], True), + "username": u["username"], + "nickname": u["nick"], + "discriminator": u["discriminator"] }) return mem @@ -295,7 +295,7 @@ def post(): channel_id = request.form.get('channel_id') content = request.form.get('content') if "user_id" in session: - dbUser = GuildMembers.query.filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == str(session['user_id'])).first() + dbUser = redisqueue.get_guild_member(guild_id, session["user_id"]) else: dbUser = None if user_unauthenticated(): @@ -331,8 +331,8 @@ def post(): else: username = session["username"] if dbUser: - if dbUser.nickname: - username = dbUser.nickname + if dbUser["nick"]: + username = dbUser["nick"] # if content.startswith("(Titan Dev) "): # content = content[12:] # username = "(Titan Dev) " + username @@ -527,13 +527,13 @@ def user_info(guild_id, user_id): "roles": [], "badges": [], } - member = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.user_id == user_id).first() + member = redisqueue.get_guild_member(guild_id, user_id) if member: - usr["id"] = str(member.user_id) - usr["username"] = member.username - usr["nickname"] = member.nickname - usr["discriminator"] = member.discriminator - usr["avatar"] = member.avatar + usr["id"] = str(member["id"]) + usr["username"] = member["username"] + usr["nickname"] = member["nick"] + usr["discriminator"] = member["discriminator"] + usr["avatar"] = member["avatar"] usr["avatar_url"] = generate_avatar_url(usr["id"], usr["avatar"], usr["discriminator"], True) roles = get_member_roles(guild_id, user_id) dbguild = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first() @@ -543,7 +543,7 @@ def user_info(guild_id, user_id): if gr["id"] == r: usr["roles"].append(gr) usr["badges"] = get_badges(user_id) - if redis_store.get("DiscordBotsOrgVoted/" + str(member.user_id)): + if redis_store.get("DiscordBotsOrgVoted/" + str(member["id"])): usr["badges"].append("discordbotsorgvoted") return jsonify(usr) diff --git a/webapp/titanembeds/blueprints/gateway/gateway.py b/webapp/titanembeds/blueprints/gateway/gateway.py index 5ccb465..f8487ec 100644 --- a/webapp/titanembeds/blueprints/gateway/gateway.py +++ b/webapp/titanembeds/blueprints/gateway/gateway.py @@ -1,5 +1,5 @@ -from titanembeds.utils import socketio, guild_accepts_visitors, get_client_ipaddr, discord_api, check_user_in_guild, get_guild_channels, update_user_status, guild_webhooks_enabled, redis_store -from titanembeds.database import db, GuildMembers, get_guild_member, Guilds +from titanembeds.utils import socketio, guild_accepts_visitors, get_client_ipaddr, discord_api, check_user_in_guild, get_guild_channels, update_user_status, guild_webhooks_enabled, redis_store, redisqueue +from titanembeds.database import db, Guilds from flask_socketio import Namespace, emit, disconnect, join_room, leave_room import functools from flask import request, session @@ -38,7 +38,7 @@ class Gateway(Namespace): if session["unauthenticated"]: emit("embed_user_connect", {"unauthenticated": True, "username": session["username"], "discriminator": session["user_id"]}, room="GUILD_"+guild_id) else: - nickname = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.user_id == session["user_id"]).first().nickname + nickname = redisqueue.get_guild_member(guild_id, session["user_id"]).get("nickname") emit("embed_user_connect", {"unauthenticated": False, "id": str(session["user_id"]), "nickname": nickname, "username": session["username"],"discriminator": session["discriminator"], "avatar_url": session["avatar"]}, room="GUILD_"+guild_id) emit("identified") self.teardown_db_session() @@ -125,17 +125,17 @@ class Gateway(Namespace): def get_user_color(self, guild_id, user_id): color = None - member = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.user_id == user_id).first() + member = redisqueue.get_guild_member(guild_id, user_id) if not member: return None guild_roles = json.loads(db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first().roles) guildroles_filtered = {} for role in guild_roles: guildroles_filtered[role["id"]] = role - member_roleids = json.loads(member.roles) + member_roleids = member["roles"] member_roles = [] for roleid in member_roleids: - role = guildroles_filtered.get(roleid) + role = guildroles_filtered.get(str(roleid)) if not role: continue member_roles.append(role) @@ -148,6 +148,7 @@ class Gateway(Namespace): return color def on_lookup_user_info(self, data): + print("test") guild_id = data["guild_id"] name = data["name"] discriminator = data["discriminator"] @@ -162,28 +163,36 @@ class Gateway(Namespace): "avatar_url": None, "discordbotsorgvoted": False, } - member = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.username == name, GuildMembers.discriminator == discriminator).first() + member = redisqueue.get_guild_member_named(guild_id, "{}#{}".format(name, discriminator)) + print("test1") if member: - usr["id"] = str(member.user_id) - usr["username"] = member.username - usr["nickname"] = member.nickname - usr["avatar"] = member.avatar + print("test1.5") + usr["id"] = str(member["id"]) + usr["username"] = member["username"] + usr["nickname"] = member["nick"] + usr["avatar"] = member["avatar"] + print("test1.54") usr["color"] = self.get_user_color(guild_id, usr["id"]) + print("test1.55") if (usr["avatar"]): usr["avatar_url"] = "https://cdn.discordapp.com/avatars/{}/{}.png".format(usr["id"], usr["avatar"]) - usr["roles"] = json.loads(member.roles) - usr["discordbotsorgvoted"] = bool(redis_store.get("DiscordBotsOrgVoted/" + str(member.user_id))) + usr["roles"] = member["roles"] + usr["discordbotsorgvoted"] = bool(redis_store.get("DiscordBotsOrgVoted/" + str(member["id"]))) + print("test1.6") else: - member = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.nickname == name, GuildMembers.discriminator == discriminator).first() + print("test2") + member = redisqueue.get_guild_member_named(guild_id, name) + print("test3") if member: - usr["id"] = str(member.user_id) - usr["username"] = member.username - usr["nickname"] = member.nickname - usr["avatar"] = member.avatar + usr["id"] = str(member["id"]) + usr["username"] = member["username"] + usr["nickname"] = member["nick"] + usr["avatar"] = member["avatar"] usr["color"] = self.get_user_color(guild_id, usr["id"]) if (usr["avatar"]): usr["avatar_url"] = "https://cdn.discordapp.com/avatars/{}/{}.png".format(usr["id"], usr["avatar"]) - usr["roles"] = json.loads(member.roles) - usr["discordbotsorgvoted"] = bool(redis_store.get("DiscordBotsOrgVoted/" + str(member.user_id))) + usr["roles"] = member["roles"] + usr["discordbotsorgvoted"] = bool(redis_store.get("DiscordBotsOrgVoted/" + str(member["id"]))) + print("test4") emit("lookup_user_info", usr) self.teardown_db_session() \ No newline at end of file diff --git a/webapp/titanembeds/database/__init__.py b/webapp/titanembeds/database/__init__.py index 72959c3..6801885 100644 --- a/webapp/titanembeds/database/__init__.py +++ b/webapp/titanembeds/database/__init__.py @@ -6,7 +6,6 @@ from .guilds import Guilds from .unauthenticated_users import UnauthenticatedUsers from .unauthenticated_bans import UnauthenticatedBans from .authenticated_users import AuthenticatedUsers -from .guild_members import GuildMembers, list_all_guild_members, get_guild_member from .cosmetics import Cosmetics, set_badges, get_badges, add_badge, remove_badge from .user_css import UserCSS from .administrators import Administrators, get_administrators_list diff --git a/webapp/titanembeds/database/guild_members.py b/webapp/titanembeds/database/guild_members.py deleted file mode 100644 index 0c262b8..0000000 --- a/webapp/titanembeds/database/guild_members.py +++ /dev/null @@ -1,48 +0,0 @@ -from titanembeds.database import db -import json - -class GuildMembers(db.Model): - __tablename__ = "guild_members" - id = db.Column(db.Integer, primary_key=True) # Auto incremented id - guild_id = db.Column(db.BigInteger, nullable=False) # Discord guild id - user_id = db.Column(db.BigInteger, nullable=False) # Discord user id - username = db.Column(db.String(255), nullable=False) # Name - discriminator = db.Column(db.Integer, nullable=False) # User discriminator - nickname = db.Column(db.String(255)) # User nickname - avatar = db.Column(db.String(255)) # The avatar str of the user - active = db.Column(db.Boolean(), nullable=False) # If the user is a member of the guild - banned = db.Column(db.Boolean(), nullable=False) # If the user is banned in the guild - roles = db.Column(db.Text(), nullable=False) # Member roles - - def __init__(self, guild_id, user_id, username, discriminator, nickname, avatar, active, banned, roles): - self.guild_id = guild_id - self.user_id = user_id - self.username = username - self.discriminator = discriminator - self.nickname = nickname - self.avatar = avatar - self.active = active - self.banned = banned - self.roles = roles - - def __repr__(self): - return ''.format(self.id, self.guild_id, self.user_id, self.username, self.discriminator) - -def list_all_guild_members(guild_id): - memlist = [] - members = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id).all() - for member in members: - memlist.append({ - "user": { - "username": member.username, - "discriminator": member.discriminator, - "id": member.user_id, - "avatar": member.avatar - }, - "roles": json.loads(member.roles), - "nickname": member.nickname, - }) - return memlist - -def get_guild_member(guild_id, member_id): - return db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.user_id == member_id).first() diff --git a/webapp/titanembeds/redisqueue.py b/webapp/titanembeds/redisqueue.py index 51fdb88..370b757 100644 --- a/webapp/titanembeds/redisqueue.py +++ b/webapp/titanembeds/redisqueue.py @@ -1,5 +1,4 @@ from titanembeds.utils import redis_store -from titanembeds.database import get_guild_member import json import time @@ -16,20 +15,21 @@ class RedisQueue: "params": params } loop_count = 0 - while not data and loop_count < 10: - if loop_count % 5 == 0: + while (not data and data != "") and loop_count < 50: + if loop_count % 25 == 0: redis_store.publish("discord-api-req", json.dumps(payload)) - time.sleep(0.5) + time.sleep(0.1) data = self._get(key, data_type) loop_count += 1 redis_store.expire(key, 60 * 5) - if data == None: + if data == None or data == "": return None if data_type == "set": data = list(data) data_parsed = [] for d in data: - data_parsed.append(json.loads(d)) + if d != "": + data_parsed.append(json.loads(d)) return data_parsed return json.loads(data) @@ -61,28 +61,51 @@ class RedisQueue: "embeds": x["embeds"], } if message["author"]["id"] not in guild_members: - member = get_guild_member(guild_id, message["author"]["id"]) + member = self.get_guild_member(guild_id, message["author"]["id"]) guild_members[message["author"]["id"]] = member else: member = guild_members[message["author"]["id"]] message["author"]["nickname"] = None if member: - message["author"]["nickname"] = member.nickname - message["author"]["avatar"] = member.avatar - message["author"]["discriminator"] = member.discriminator - message["author"]["username"] = member.username + message["author"]["nickname"] = member["nick"] + message["author"]["avatar"] = member["avatar"] + message["author"]["discriminator"] = member["discriminator"] + message["author"]["username"] = member["username"] for mention in message["mentions"]: if mention["id"] not in guild_members: - author = get_guild_member(guild_id, mention["id"]) + author = self.get_guild_member(guild_id, mention["id"]) guild_members[mention["id"]] = author else: author = guild_members[mention["id"]] mention["nickname"] = None if author: - mention["nickname"] = author.nickname - mention["avatar"] = author.avatar - mention["username"] = author.username - mention["discriminator"] = author.discriminator + mention["nickname"] = author["nick"] + mention["avatar"] = author["avatar"] + mention["username"] = author["username"] + mention["discriminator"] = author["discriminator"] msgs.append(message) sorted_msgs = sorted(msgs, key=lambda k: k['id'], reverse=True) - return sorted_msgs \ No newline at end of file + return sorted_msgs + + def get_guild_member(self, guild_id, user_id): + key = "/guilds/{}/members/{}".format(guild_id, user_id) + q = self.get(key, "get_guild_member", {"guild_id": guild_id, "user_id": user_id}) + return q + + def get_guild_member_named(self, guild_id, query): + key = "/custom/guilds/{}/member_named/{}".format(guild_id, query) + guild_member_id = self.get(key, "get_guild_member_named", {"guild_id": guild_id, "query": query}) + if guild_member_id: + return self.get_guild_member(guild_id, guild_member_id["user_id"]) + return None + + def list_guild_members(self, guild_id): + key = "/guilds/{}/members".format(guild_id) + member_ids = self.get(key, "list_guild_members", {"guild_id": guild_id}, data_type="set") + members = [] + for member_id in member_ids: + usr_id = member_id["user_id"] + member = self.get_guild_member(guild_id, usr_id) + if member: + members.append(member) + return members \ No newline at end of file diff --git a/webapp/titanembeds/utils.py b/webapp/titanembeds/utils.py index a3f221a..42542e6 100644 --- a/webapp/titanembeds/utils.py +++ b/webapp/titanembeds/utils.py @@ -1,4 +1,4 @@ -from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, get_guild_member +from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers from titanembeds.constants import LANGUAGES from flask import request, session from flask_limiter import Limiter @@ -98,8 +98,8 @@ def checkUserRevoke(guild_id, user_key=None): banned = checkUserBanned(guild_id) if banned: return revoked - dbUser = GuildMembers.query.filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == session["user_id"]).first() - revoked = not dbUser or not dbUser.active + dbUser = redisqueue.get_guild_member(guild_id, session["user_id"]) + revoked = not dbUser return revoked def checkUserBanned(guild_id, ip_address=None): @@ -114,11 +114,9 @@ def checkUserBanned(guild_id, ip_address=None): banned = False else: banned = False - dbUser = GuildMembers.query.filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == session["user_id"]).first() - if not dbUser: - banned = False - else: - banned = dbUser.banned + #dbUser = redisqueue.get_guild_member(guild_id, session["user_id"]) + #if not dbUser: + # banned = True # TODO: Figure out ban logic with guild member return banned from titanembeds.oauth import check_user_can_administrate_guild, user_has_permission @@ -163,9 +161,9 @@ def update_user_status(guild_id, username, user_key=None): } if status['banned'] or status['revoked']: return status - dbMember = get_guild_member(guild_id, status["user_id"]) + dbMember = redisqueue.get_guild_member(guild_id, status["user_id"]) if dbMember: - status["nickname"] = dbMember.nickname + status["nickname"] = dbMember["nick"] bump_user_presence_timestamp(guild_id, "AuthenticatedUsers", status["user_id"]) return status @@ -195,18 +193,8 @@ def check_user_in_guild(guild_id): return dbUser is not None and not checkUserRevoke(guild_id) def get_member_roles(guild_id, user_id): - q = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == user_id).first() - roles = [guild_id] - if not q: - member = discord_api.get_guild_member(guild_id, user_id) - if member["success"]: - roles = member["content"]["roles"] - member = GuildMembers(guild_id, user_id, member["content"]["user"]["username"], int(member["content"]["user"]["discriminator"]), member["content"].get("nick", None), member["content"]["user"]["avatar"], True, False, json.dumps(roles)) - db.session.add(member) - db.session.commit() - else: - roles = json.loads(q.roles) - return roles + q = redisqueue.get_guild_member(guild_id, user_id) + return q["roles"] def get_guild_channels(guild_id, force_everyone=False): if user_unauthenticated() or force_everyone: