From 89b12d06abb5bba124938368e91204e33afce90f Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Mon, 16 Jul 2018 23:43:41 +0000 Subject: [PATCH] Revert "Remove messages table", Accidentally committed in dev code This reverts commit 6d6b39043192e66d4d24cdac5731f5bbc7a80a98. --- discordbot/titanembeds/bot.py | 3 + discordbot/titanembeds/database/__init__.py | 32 +++++++ discordbot/titanembeds/database/messages.py | 14 ++++ discordbot/titanembeds/redisqueue.py | 13 +-- discordbot/titanembeds/utils.py | 1 - ...1b243ba2_remove_messages_table_for_good.py | 40 --------- webapp/titanembeds/blueprints/api/api.py | 2 +- webapp/titanembeds/database/__init__.py | 1 + webapp/titanembeds/database/messages.py | 83 +++++++++++++++++++ webapp/titanembeds/redisqueue.py | 36 ++++---- 10 files changed, 152 insertions(+), 73 deletions(-) create mode 100644 discordbot/titanembeds/database/messages.py delete mode 100644 webapp/alembic/versions/52271b243ba2_remove_messages_table_for_good.py create mode 100644 webapp/titanembeds/database/messages.py diff --git a/discordbot/titanembeds/bot.py b/discordbot/titanembeds/bot.py index 8d6af8f..3938b38 100644 --- a/discordbot/titanembeds/bot.py +++ b/discordbot/titanembeds/bot.py @@ -81,6 +81,7 @@ class Titan(discord.AutoShardedClient): async def on_message(self, message): await self.socketio.on_message(message) + await self.database.push_message(message) await self.redisqueue.push_message(message) msg_arr = message.content.split() # split the message @@ -95,11 +96,13 @@ class Titan(discord.AutoShardedClient): await getattr(self.command, msg_cmd)(message) #actually run cmd, passing in msg obj async def on_message_edit(self, message_before, message_after): + await self.database.update_message(message_after) await self.redisqueue.update_message(message_after) await self.socketio.on_message_update(message_after) async def on_message_delete(self, message): self.delete_list.append(message.id) + await self.database.delete_message(message) await self.redisqueue.delete_message(message) await self.socketio.on_message_delete(message) diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index 53daf22..da62dce 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -6,6 +6,7 @@ import datetime db = Gino() from titanembeds.database.guilds import Guilds +from titanembeds.database.messages import Messages from titanembeds.database.guild_members import GuildMembers from titanembeds.database.unauthenticated_users import UnauthenticatedUsers from titanembeds.database.unauthenticated_bans import UnauthenticatedBans @@ -18,6 +19,37 @@ class DatabaseInterface(object): async def connect(self, dburi): await db.set_bind(dburi) + + async def push_message(self, message): + if message.guild: + await Messages.create( + message_id = int(message.id), + guild_id = int(message.guild.id), + channel_id = int(message.channel.id), + content = message.content, + author = json.dumps(get_message_author(message)), + timestamp = message.created_at, + edited_timestamp = message.edited_at, + mentions = json.dumps(get_message_mentions(message.mentions)), + attachments = json.dumps(get_attachments_list(message.attachments)), + embeds = json.dumps(get_embeds_list(message.embeds)) + ) + + async def update_message(self, message): + if message.guild: + await Messages.update.values( + content = message.content, + timestamp = message.created_at, + edited_timestamp = message.edited_at, + mentions = json.dumps(get_message_mentions(message.mentions)), + attachments = json.dumps(get_attachments_list(message.attachments)), + embeds = json.dumps(get_embeds_list(message.embeds)), + author = json.dumps(get_message_author(message)) + ).where(Messages.message_id == int(message.id)).gino.status() + + async def delete_message(self, message): + if message.guild: + await Messages.delete.where(Messages.message_id == int(message.id)).gino.status() async def update_guild(self, guild): if guild.me.guild_permissions.manage_webhooks: diff --git a/discordbot/titanembeds/database/messages.py b/discordbot/titanembeds/database/messages.py new file mode 100644 index 0000000..f8ba868 --- /dev/null +++ b/discordbot/titanembeds/database/messages.py @@ -0,0 +1,14 @@ +from titanembeds.database import db + +class Messages(db.Model): + __tablename__ = "messages" + message_id = db.Column(db.BigInteger, primary_key=True) # Message snowflake + guild_id = db.Column(db.BigInteger) # Discord guild id + channel_id = db.Column(db.BigInteger) # Channel id + content = db.Column(db.Text()) # Message contents + author = db.Column(db.Text()) # Author json + timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is created + edited_timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is edited + mentions = db.Column(db.Text()) # Mentions serialized + attachments = db.Column(db.Text()) # serialized attachments + embeds = db.Column(db.Text().with_variant(db.Text(length=4294967295), 'mysql')) # message embeds \ No newline at end of file diff --git a/discordbot/titanembeds/redisqueue.py b/discordbot/titanembeds/redisqueue.py index 765f507..132a3c6 100644 --- a/discordbot/titanembeds/redisqueue.py +++ b/discordbot/titanembeds/redisqueue.py @@ -1,4 +1,4 @@ -from titanembeds.utils import get_formatted_message, get_formatted_user +from titanembeds.utils import get_formatted_message from urllib.parse import urlparse import asyncio_redis import json @@ -107,13 +107,4 @@ class RedisQueue: async def update_message(self, message): await self.delete_message(message) - await self.push_message(message) - - async def on_get_guild_member(self, key, params): - member = self.bot.get_guild(int(params["guild_id"])).get_member(int(params["user_id"])) - if not member: - await self.connection.set(key, "") - return - await self.connection.delete([key]) - user = get_formatted_user(member) - await self.connection.set(key, json.dumps(user)) \ No newline at end of file + await self.push_message(message) \ No newline at end of file diff --git a/discordbot/titanembeds/utils.py b/discordbot/titanembeds/utils.py index d96e5ed..b8a234a 100644 --- a/discordbot/titanembeds/utils.py +++ b/discordbot/titanembeds/utils.py @@ -50,7 +50,6 @@ def get_formatted_user(user): "status": str(user.status), "username": user.name, "nick": None, - "bot": user.bot } if userobj["color"] == "000000": userobj["color"] = None diff --git a/webapp/alembic/versions/52271b243ba2_remove_messages_table_for_good.py b/webapp/alembic/versions/52271b243ba2_remove_messages_table_for_good.py deleted file mode 100644 index 5c3cd7d..0000000 --- a/webapp/alembic/versions/52271b243ba2_remove_messages_table_for_good.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Remove messages table for good - -Revision ID: 52271b243ba2 -Revises: 8e806bcb2228 -Create Date: 2018-07-16 23:33:31.600454 - -""" - -# revision identifiers, used by Alembic. -revision = '52271b243ba2' -down_revision = '8e806bcb2228' -branch_labels = None -depends_on = None - -from alembic import op -import sqlalchemy as sa -from sqlalchemy.dialects import postgresql - -def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.drop_table('messages') - # ### end Alembic commands ### - - -def downgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table('messages', - sa.Column('guild_id', sa.BIGINT(), autoincrement=False, nullable=False), - sa.Column('channel_id', sa.BIGINT(), autoincrement=False, nullable=False), - sa.Column('message_id', sa.BIGINT(), autoincrement=False, nullable=False), - sa.Column('content', sa.TEXT(), autoincrement=False, nullable=False), - sa.Column('author', sa.TEXT(), autoincrement=False, nullable=False), - sa.Column('timestamp', postgresql.TIMESTAMP(), autoincrement=False, nullable=False), - sa.Column('edited_timestamp', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), - sa.Column('mentions', sa.TEXT(), autoincrement=False, nullable=True), - sa.Column('attachments', sa.TEXT(), autoincrement=False, nullable=True), - sa.Column('embeds', sa.TEXT(), autoincrement=False, nullable=True), - sa.PrimaryKeyConstraint('message_id', name='messages_pkey') - ) - # ### end Alembic commands ### diff --git a/webapp/titanembeds/blueprints/api/api.py b/webapp/titanembeds/blueprints/api/api.py index 1f241e6..3564f84 100644 --- a/webapp/titanembeds/blueprints/api/api.py +++ b/webapp/titanembeds/blueprints/api/api.py @@ -1,4 +1,4 @@ -from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, list_all_guild_members, get_guild_member, get_administrators_list, get_badges, DiscordBotsOrgTransactions +from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, Messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges, DiscordBotsOrgTransactions from titanembeds.decorators import valid_session_required, discord_users_only, abort_if_guild_disabled from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles, get_online_embed_user_keys, redis_store, redisqueue from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild diff --git a/webapp/titanembeds/database/__init__.py b/webapp/titanembeds/database/__init__.py index 72959c3..79671b8 100644 --- a/webapp/titanembeds/database/__init__.py +++ b/webapp/titanembeds/database/__init__.py @@ -7,6 +7,7 @@ from .unauthenticated_users import UnauthenticatedUsers from .unauthenticated_bans import UnauthenticatedBans from .authenticated_users import AuthenticatedUsers from .guild_members import GuildMembers, list_all_guild_members, get_guild_member +from .messages import Messages, get_channel_messages from .cosmetics import Cosmetics, set_badges, get_badges, add_badge, remove_badge from .user_css import UserCSS from .administrators import Administrators, get_administrators_list diff --git a/webapp/titanembeds/database/messages.py b/webapp/titanembeds/database/messages.py new file mode 100644 index 0000000..15c4f78 --- /dev/null +++ b/webapp/titanembeds/database/messages.py @@ -0,0 +1,83 @@ +from titanembeds.database import db, get_guild_member +from sqlalchemy import cast +import json + +class Messages(db.Model): + __tablename__ = "messages" + message_id = db.Column(db.BigInteger, nullable=False, primary_key=True) # Message snowflake + guild_id = db.Column(db.BigInteger, nullable=False) # Discord guild id + channel_id = db.Column(db.BigInteger, nullable=False) # Channel id + content = db.Column(db.Text(), nullable=False) # Message contents + author = db.Column(db.Text(), nullable=False) # Author + timestamp = db.Column(db.TIMESTAMP, nullable=False) # Timestamp of when content is created + edited_timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is edited + mentions = db.Column(db.Text()) # Mentions serialized + attachments = db.Column(db.Text()) # serialized attachments + embeds = db.Column(db.Text().with_variant(db.Text(length=4294967295), 'mysql')) # message embeds + + def __init__(self, guild_id, channel_id, message_id, content, author, timestamp, edited_timestamp, mentions, attachments, embeds): + self.guild_id = guild_id + self.channel_id = channel_id + self.message_id = message_id + self.content = content + self.author = author + self.timestamp = timestamp + self.edited_timestamp = edited_timestamp + self.mentions = mentions + self.attachments = attachments + self.embeds = embeds + + def __repr__(self): + return ''.format(self.id, self.guild_id, self.guild_id, self.channel_id, self.message_id) + +def get_channel_messages(guild_id, channel_id, after_snowflake=None): + if not after_snowflake: + q = db.session.query(Messages).filter(Messages.channel_id == channel_id).order_by(Messages.timestamp.desc()).limit(50) + else: + q = db.session.query(Messages).filter(Messages.channel_id == channel_id).filter(Messages.message_id > after_snowflake).order_by(Messages.timestamp.desc()).limit(50) + msgs = [] + snowflakes = [] + guild_members = {} + for x in q: + if x.message_id in snowflakes: + continue + snowflakes.append(x.message_id) + embeds = x.embeds + if not embeds: + embeds = "[]" + message = { + "attachments": json.loads(x.attachments), + "timestamp": x.timestamp, + "id": str(x.message_id), + "edited_timestamp": x.edited_timestamp, + "author": json.loads(x.author), + "content": x.content, + "channel_id": str(x.channel_id), + "mentions": json.loads(x.mentions), + "embeds": json.loads(embeds), + } + if message["author"]["id"] not in guild_members: + member = get_guild_member(guild_id, message["author"]["id"]) + guild_members[message["author"]["id"]] = member + else: + member = guild_members[message["author"]["id"]] + message["author"]["nickname"] = None + if member: + message["author"]["nickname"] = member.nickname + message["author"]["avatar"] = member.avatar + message["author"]["discriminator"] = member.discriminator + message["author"]["username"] = member.username + for mention in message["mentions"]: + if mention["id"] not in guild_members: + author = get_guild_member(guild_id, mention["id"]) + guild_members[mention["id"]] = author + else: + author = guild_members[mention["id"]] + mention["nickname"] = None + if author: + mention["nickname"] = author.nickname + mention["avatar"] = author.avatar + mention["username"] = author.username + mention["discriminator"] = author.discriminator + msgs.append(message) + return msgs diff --git a/webapp/titanembeds/redisqueue.py b/webapp/titanembeds/redisqueue.py index 6fc20cf..51fdb88 100644 --- a/webapp/titanembeds/redisqueue.py +++ b/webapp/titanembeds/redisqueue.py @@ -1,4 +1,5 @@ from titanembeds.utils import redis_store +from titanembeds.database import get_guild_member import json import time @@ -15,14 +16,14 @@ class RedisQueue: "params": params } loop_count = 0 - while (not data and data != "") and loop_count < 50: - if loop_count % 25 == 0: + while not data and loop_count < 10: + if loop_count % 5 == 0: redis_store.publish("discord-api-req", json.dumps(payload)) - time.sleep(0.1) + time.sleep(0.5) data = self._get(key, data_type) loop_count += 1 redis_store.expire(key, 60 * 5) - if data == None or data == "": + if data == None: return None if data_type == "set": data = list(data) @@ -60,33 +61,28 @@ class RedisQueue: "embeds": x["embeds"], } if message["author"]["id"] not in guild_members: - member = self.get_guild_member(guild_id, message["author"]["id"]) + member = get_guild_member(guild_id, message["author"]["id"]) guild_members[message["author"]["id"]] = member else: member = guild_members[message["author"]["id"]] message["author"]["nickname"] = None if member: - message["author"]["nickname"] = member["nick"] - message["author"]["avatar"] = member["avatar"] - message["author"]["discriminator"] = member["discriminator"] - message["author"]["username"] = member["username"] + message["author"]["nickname"] = member.nickname + message["author"]["avatar"] = member.avatar + message["author"]["discriminator"] = member.discriminator + message["author"]["username"] = member.username for mention in message["mentions"]: if mention["id"] not in guild_members: - author = self.get_guild_member(guild_id, mention["id"]) + author = get_guild_member(guild_id, mention["id"]) guild_members[mention["id"]] = author else: author = guild_members[mention["id"]] mention["nickname"] = None if author: - mention["nickname"] = author["nick"] - mention["avatar"] = author["avatar"] - mention["username"] = author["username"] - mention["discriminator"] = author["discriminator"] + mention["nickname"] = author.nickname + mention["avatar"] = author.avatar + mention["username"] = author.username + mention["discriminator"] = author.discriminator msgs.append(message) sorted_msgs = sorted(msgs, key=lambda k: k['id'], reverse=True) - return sorted_msgs - - def get_guild_member(self, guild_id, user_id): - key = "/guilds/{}/members/{}".format(guild_id, user_id) - q = self.get(key, "get_guild_member", {"guild_id": guild_id, "user_id": user_id}) - return q \ No newline at end of file + return sorted_msgs \ No newline at end of file