diff --git a/discordbot/titanembeds/bot.py b/discordbot/titanembeds/bot.py index 3938b38..8d6af8f 100644 --- a/discordbot/titanembeds/bot.py +++ b/discordbot/titanembeds/bot.py @@ -81,7 +81,6 @@ class Titan(discord.AutoShardedClient): async def on_message(self, message): await self.socketio.on_message(message) - await self.database.push_message(message) await self.redisqueue.push_message(message) msg_arr = message.content.split() # split the message @@ -96,13 +95,11 @@ class Titan(discord.AutoShardedClient): await getattr(self.command, msg_cmd)(message) #actually run cmd, passing in msg obj async def on_message_edit(self, message_before, message_after): - await self.database.update_message(message_after) await self.redisqueue.update_message(message_after) await self.socketio.on_message_update(message_after) async def on_message_delete(self, message): self.delete_list.append(message.id) - await self.database.delete_message(message) await self.redisqueue.delete_message(message) await self.socketio.on_message_delete(message) diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index da62dce..53daf22 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -6,7 +6,6 @@ import datetime db = Gino() from titanembeds.database.guilds import Guilds -from titanembeds.database.messages import Messages from titanembeds.database.guild_members import GuildMembers from titanembeds.database.unauthenticated_users import UnauthenticatedUsers from titanembeds.database.unauthenticated_bans import UnauthenticatedBans @@ -19,37 +18,6 @@ class DatabaseInterface(object): async def connect(self, dburi): await db.set_bind(dburi) - - async def push_message(self, message): - if message.guild: - await Messages.create( - message_id = int(message.id), - guild_id = int(message.guild.id), - channel_id = int(message.channel.id), - content = message.content, - author = json.dumps(get_message_author(message)), - timestamp = message.created_at, - edited_timestamp = message.edited_at, - mentions = json.dumps(get_message_mentions(message.mentions)), - attachments = json.dumps(get_attachments_list(message.attachments)), - embeds = json.dumps(get_embeds_list(message.embeds)) - ) - - async def update_message(self, message): - if message.guild: - await Messages.update.values( - content = message.content, - timestamp = message.created_at, - edited_timestamp = message.edited_at, - mentions = json.dumps(get_message_mentions(message.mentions)), - attachments = json.dumps(get_attachments_list(message.attachments)), - embeds = json.dumps(get_embeds_list(message.embeds)), - author = json.dumps(get_message_author(message)) - ).where(Messages.message_id == int(message.id)).gino.status() - - async def delete_message(self, message): - if message.guild: - await Messages.delete.where(Messages.message_id == int(message.id)).gino.status() async def update_guild(self, guild): if guild.me.guild_permissions.manage_webhooks: diff --git a/discordbot/titanembeds/database/messages.py b/discordbot/titanembeds/database/messages.py deleted file mode 100644 index f8ba868..0000000 --- a/discordbot/titanembeds/database/messages.py +++ /dev/null @@ -1,14 +0,0 @@ -from titanembeds.database import db - -class Messages(db.Model): - __tablename__ = "messages" - message_id = db.Column(db.BigInteger, primary_key=True) # Message snowflake - guild_id = db.Column(db.BigInteger) # Discord guild id - channel_id = db.Column(db.BigInteger) # Channel id - content = db.Column(db.Text()) # Message contents - author = db.Column(db.Text()) # Author json - timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is created - edited_timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is edited - mentions = db.Column(db.Text()) # Mentions serialized - attachments = db.Column(db.Text()) # serialized attachments - embeds = db.Column(db.Text().with_variant(db.Text(length=4294967295), 'mysql')) # message embeds \ No newline at end of file diff --git a/discordbot/titanembeds/redisqueue.py b/discordbot/titanembeds/redisqueue.py index 132a3c6..765f507 100644 --- a/discordbot/titanembeds/redisqueue.py +++ b/discordbot/titanembeds/redisqueue.py @@ -1,4 +1,4 @@ -from titanembeds.utils import get_formatted_message +from titanembeds.utils import get_formatted_message, get_formatted_user from urllib.parse import urlparse import asyncio_redis import json @@ -107,4 +107,13 @@ class RedisQueue: async def update_message(self, message): await self.delete_message(message) - await self.push_message(message) \ No newline at end of file + await self.push_message(message) + + async def on_get_guild_member(self, key, params): + member = self.bot.get_guild(int(params["guild_id"])).get_member(int(params["user_id"])) + if not member: + await self.connection.set(key, "") + return + await self.connection.delete([key]) + user = get_formatted_user(member) + await self.connection.set(key, json.dumps(user)) \ No newline at end of file diff --git a/discordbot/titanembeds/utils.py b/discordbot/titanembeds/utils.py index b8a234a..d96e5ed 100644 --- a/discordbot/titanembeds/utils.py +++ b/discordbot/titanembeds/utils.py @@ -50,6 +50,7 @@ def get_formatted_user(user): "status": str(user.status), "username": user.name, "nick": None, + "bot": user.bot } if userobj["color"] == "000000": userobj["color"] = None diff --git a/webapp/alembic/versions/52271b243ba2_remove_messages_table_for_good.py b/webapp/alembic/versions/52271b243ba2_remove_messages_table_for_good.py new file mode 100644 index 0000000..5c3cd7d --- /dev/null +++ b/webapp/alembic/versions/52271b243ba2_remove_messages_table_for_good.py @@ -0,0 +1,40 @@ +"""Remove messages table for good + +Revision ID: 52271b243ba2 +Revises: 8e806bcb2228 +Create Date: 2018-07-16 23:33:31.600454 + +""" + +# revision identifiers, used by Alembic. +revision = '52271b243ba2' +down_revision = '8e806bcb2228' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('messages') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('messages', + sa.Column('guild_id', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('channel_id', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('message_id', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('content', sa.TEXT(), autoincrement=False, nullable=False), + sa.Column('author', sa.TEXT(), autoincrement=False, nullable=False), + sa.Column('timestamp', postgresql.TIMESTAMP(), autoincrement=False, nullable=False), + sa.Column('edited_timestamp', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('mentions', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('attachments', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('embeds', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('message_id', name='messages_pkey') + ) + # ### end Alembic commands ### diff --git a/webapp/titanembeds/blueprints/api/api.py b/webapp/titanembeds/blueprints/api/api.py index 3564f84..1f241e6 100644 --- a/webapp/titanembeds/blueprints/api/api.py +++ b/webapp/titanembeds/blueprints/api/api.py @@ -1,4 +1,4 @@ -from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, Messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges, DiscordBotsOrgTransactions +from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, list_all_guild_members, get_guild_member, get_administrators_list, get_badges, DiscordBotsOrgTransactions from titanembeds.decorators import valid_session_required, discord_users_only, abort_if_guild_disabled from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles, get_online_embed_user_keys, redis_store, redisqueue from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild diff --git a/webapp/titanembeds/database/__init__.py b/webapp/titanembeds/database/__init__.py index 79671b8..72959c3 100644 --- a/webapp/titanembeds/database/__init__.py +++ b/webapp/titanembeds/database/__init__.py @@ -7,7 +7,6 @@ from .unauthenticated_users import UnauthenticatedUsers from .unauthenticated_bans import UnauthenticatedBans from .authenticated_users import AuthenticatedUsers from .guild_members import GuildMembers, list_all_guild_members, get_guild_member -from .messages import Messages, get_channel_messages from .cosmetics import Cosmetics, set_badges, get_badges, add_badge, remove_badge from .user_css import UserCSS from .administrators import Administrators, get_administrators_list diff --git a/webapp/titanembeds/database/messages.py b/webapp/titanembeds/database/messages.py deleted file mode 100644 index 15c4f78..0000000 --- a/webapp/titanembeds/database/messages.py +++ /dev/null @@ -1,83 +0,0 @@ -from titanembeds.database import db, get_guild_member -from sqlalchemy import cast -import json - -class Messages(db.Model): - __tablename__ = "messages" - message_id = db.Column(db.BigInteger, nullable=False, primary_key=True) # Message snowflake - guild_id = db.Column(db.BigInteger, nullable=False) # Discord guild id - channel_id = db.Column(db.BigInteger, nullable=False) # Channel id - content = db.Column(db.Text(), nullable=False) # Message contents - author = db.Column(db.Text(), nullable=False) # Author - timestamp = db.Column(db.TIMESTAMP, nullable=False) # Timestamp of when content is created - edited_timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is edited - mentions = db.Column(db.Text()) # Mentions serialized - attachments = db.Column(db.Text()) # serialized attachments - embeds = db.Column(db.Text().with_variant(db.Text(length=4294967295), 'mysql')) # message embeds - - def __init__(self, guild_id, channel_id, message_id, content, author, timestamp, edited_timestamp, mentions, attachments, embeds): - self.guild_id = guild_id - self.channel_id = channel_id - self.message_id = message_id - self.content = content - self.author = author - self.timestamp = timestamp - self.edited_timestamp = edited_timestamp - self.mentions = mentions - self.attachments = attachments - self.embeds = embeds - - def __repr__(self): - return ''.format(self.id, self.guild_id, self.guild_id, self.channel_id, self.message_id) - -def get_channel_messages(guild_id, channel_id, after_snowflake=None): - if not after_snowflake: - q = db.session.query(Messages).filter(Messages.channel_id == channel_id).order_by(Messages.timestamp.desc()).limit(50) - else: - q = db.session.query(Messages).filter(Messages.channel_id == channel_id).filter(Messages.message_id > after_snowflake).order_by(Messages.timestamp.desc()).limit(50) - msgs = [] - snowflakes = [] - guild_members = {} - for x in q: - if x.message_id in snowflakes: - continue - snowflakes.append(x.message_id) - embeds = x.embeds - if not embeds: - embeds = "[]" - message = { - "attachments": json.loads(x.attachments), - "timestamp": x.timestamp, - "id": str(x.message_id), - "edited_timestamp": x.edited_timestamp, - "author": json.loads(x.author), - "content": x.content, - "channel_id": str(x.channel_id), - "mentions": json.loads(x.mentions), - "embeds": json.loads(embeds), - } - if message["author"]["id"] not in guild_members: - member = get_guild_member(guild_id, message["author"]["id"]) - guild_members[message["author"]["id"]] = member - else: - member = guild_members[message["author"]["id"]] - message["author"]["nickname"] = None - if member: - message["author"]["nickname"] = member.nickname - message["author"]["avatar"] = member.avatar - message["author"]["discriminator"] = member.discriminator - message["author"]["username"] = member.username - for mention in message["mentions"]: - if mention["id"] not in guild_members: - author = get_guild_member(guild_id, mention["id"]) - guild_members[mention["id"]] = author - else: - author = guild_members[mention["id"]] - mention["nickname"] = None - if author: - mention["nickname"] = author.nickname - mention["avatar"] = author.avatar - mention["username"] = author.username - mention["discriminator"] = author.discriminator - msgs.append(message) - return msgs diff --git a/webapp/titanembeds/redisqueue.py b/webapp/titanembeds/redisqueue.py index 51fdb88..6fc20cf 100644 --- a/webapp/titanembeds/redisqueue.py +++ b/webapp/titanembeds/redisqueue.py @@ -1,5 +1,4 @@ from titanembeds.utils import redis_store -from titanembeds.database import get_guild_member import json import time @@ -16,14 +15,14 @@ class RedisQueue: "params": params } loop_count = 0 - while not data and loop_count < 10: - if loop_count % 5 == 0: + while (not data and data != "") and loop_count < 50: + if loop_count % 25 == 0: redis_store.publish("discord-api-req", json.dumps(payload)) - time.sleep(0.5) + time.sleep(0.1) data = self._get(key, data_type) loop_count += 1 redis_store.expire(key, 60 * 5) - if data == None: + if data == None or data == "": return None if data_type == "set": data = list(data) @@ -61,28 +60,33 @@ class RedisQueue: "embeds": x["embeds"], } if message["author"]["id"] not in guild_members: - member = get_guild_member(guild_id, message["author"]["id"]) + member = self.get_guild_member(guild_id, message["author"]["id"]) guild_members[message["author"]["id"]] = member else: member = guild_members[message["author"]["id"]] message["author"]["nickname"] = None if member: - message["author"]["nickname"] = member.nickname - message["author"]["avatar"] = member.avatar - message["author"]["discriminator"] = member.discriminator - message["author"]["username"] = member.username + message["author"]["nickname"] = member["nick"] + message["author"]["avatar"] = member["avatar"] + message["author"]["discriminator"] = member["discriminator"] + message["author"]["username"] = member["username"] for mention in message["mentions"]: if mention["id"] not in guild_members: - author = get_guild_member(guild_id, mention["id"]) + author = self.get_guild_member(guild_id, mention["id"]) guild_members[mention["id"]] = author else: author = guild_members[mention["id"]] mention["nickname"] = None if author: - mention["nickname"] = author.nickname - mention["avatar"] = author.avatar - mention["username"] = author.username - mention["discriminator"] = author.discriminator + mention["nickname"] = author["nick"] + mention["avatar"] = author["avatar"] + mention["username"] = author["username"] + mention["discriminator"] = author["discriminator"] msgs.append(message) sorted_msgs = sorted(msgs, key=lambda k: k['id'], reverse=True) - return sorted_msgs \ No newline at end of file + return sorted_msgs + + def get_guild_member(self, guild_id, user_id): + key = "/guilds/{}/members/{}".format(guild_id, user_id) + q = self.get(key, "get_guild_member", {"guild_id": guild_id, "user_id": user_id}) + return q \ No newline at end of file