From 56ba9bc12797a09d126abdfc72ceef8f147e4253 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Mon, 16 Jul 2018 23:47:25 +0000 Subject: [PATCH] Reimplement remove message table --- discordbot/titanembeds/bot.py | 8 -- discordbot/titanembeds/database/__init__.py | 32 ------- discordbot/titanembeds/database/messages.py | 14 ---- ...1b243ba2_remove_messages_table_for_good.py | 40 +++++++++ webapp/titanembeds/blueprints/api/api.py | 2 +- webapp/titanembeds/database/__init__.py | 1 - webapp/titanembeds/database/messages.py | 83 ------------------- 7 files changed, 41 insertions(+), 139 deletions(-) delete mode 100644 discordbot/titanembeds/database/messages.py create mode 100644 webapp/alembic/versions/52271b243ba2_remove_messages_table_for_good.py delete mode 100644 webapp/titanembeds/database/messages.py diff --git a/discordbot/titanembeds/bot.py b/discordbot/titanembeds/bot.py index 3938b38..eb58613 100644 --- a/discordbot/titanembeds/bot.py +++ b/discordbot/titanembeds/bot.py @@ -81,7 +81,6 @@ class Titan(discord.AutoShardedClient): async def on_message(self, message): await self.socketio.on_message(message) - await self.database.push_message(message) await self.redisqueue.push_message(message) msg_arr = message.content.split() # split the message @@ -96,13 +95,11 @@ class Titan(discord.AutoShardedClient): await getattr(self.command, msg_cmd)(message) #actually run cmd, passing in msg obj async def on_message_edit(self, message_before, message_after): - await self.database.update_message(message_after) await self.redisqueue.update_message(message_after) await self.socketio.on_message_update(message_after) async def on_message_delete(self, message): self.delete_list.append(message.id) - await self.database.delete_message(message) await self.redisqueue.delete_message(message) await self.socketio.on_message_delete(message) @@ -125,11 +122,6 @@ class Titan(discord.AutoShardedClient): chanperm = channel.permissions_for(channel.guild.me) if not chanperm.read_messages or not chanperm.read_message_history or not isinstance(channel, discord.channel.TextChannel): continue - async for message in channel.history(limit=50, reverse=True): - try: - await self.database.push_message(message) - except: - pass await self.postStats() async def on_guild_remove(self, guild): diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index da62dce..53daf22 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -6,7 +6,6 @@ import datetime db = Gino() from titanembeds.database.guilds import Guilds -from titanembeds.database.messages import Messages from titanembeds.database.guild_members import GuildMembers from titanembeds.database.unauthenticated_users import UnauthenticatedUsers from titanembeds.database.unauthenticated_bans import UnauthenticatedBans @@ -19,37 +18,6 @@ class DatabaseInterface(object): async def connect(self, dburi): await db.set_bind(dburi) - - async def push_message(self, message): - if message.guild: - await Messages.create( - message_id = int(message.id), - guild_id = int(message.guild.id), - channel_id = int(message.channel.id), - content = message.content, - author = json.dumps(get_message_author(message)), - timestamp = message.created_at, - edited_timestamp = message.edited_at, - mentions = json.dumps(get_message_mentions(message.mentions)), - attachments = json.dumps(get_attachments_list(message.attachments)), - embeds = json.dumps(get_embeds_list(message.embeds)) - ) - - async def update_message(self, message): - if message.guild: - await Messages.update.values( - content = message.content, - timestamp = message.created_at, - edited_timestamp = message.edited_at, - mentions = json.dumps(get_message_mentions(message.mentions)), - attachments = json.dumps(get_attachments_list(message.attachments)), - embeds = json.dumps(get_embeds_list(message.embeds)), - author = json.dumps(get_message_author(message)) - ).where(Messages.message_id == int(message.id)).gino.status() - - async def delete_message(self, message): - if message.guild: - await Messages.delete.where(Messages.message_id == int(message.id)).gino.status() async def update_guild(self, guild): if guild.me.guild_permissions.manage_webhooks: diff --git a/discordbot/titanembeds/database/messages.py b/discordbot/titanembeds/database/messages.py deleted file mode 100644 index f8ba868..0000000 --- a/discordbot/titanembeds/database/messages.py +++ /dev/null @@ -1,14 +0,0 @@ -from titanembeds.database import db - -class Messages(db.Model): - __tablename__ = "messages" - message_id = db.Column(db.BigInteger, primary_key=True) # Message snowflake - guild_id = db.Column(db.BigInteger) # Discord guild id - channel_id = db.Column(db.BigInteger) # Channel id - content = db.Column(db.Text()) # Message contents - author = db.Column(db.Text()) # Author json - timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is created - edited_timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is edited - mentions = db.Column(db.Text()) # Mentions serialized - attachments = db.Column(db.Text()) # serialized attachments - embeds = db.Column(db.Text().with_variant(db.Text(length=4294967295), 'mysql')) # message embeds \ No newline at end of file diff --git a/webapp/alembic/versions/52271b243ba2_remove_messages_table_for_good.py b/webapp/alembic/versions/52271b243ba2_remove_messages_table_for_good.py new file mode 100644 index 0000000..bb99666 --- /dev/null +++ b/webapp/alembic/versions/52271b243ba2_remove_messages_table_for_good.py @@ -0,0 +1,40 @@ +"""Remove messages table for good + +Revision ID: 52271b243ba2 +Revises: 8e806bcb2228 +Create Date: 2018-07-16 23:33:31.600454 + +""" + +# revision identifiers, used by Alembic. +revision = '52271b243ba2' +down_revision = '8e806bcb2228' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_table('messages') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('messages', + sa.Column('guild_id', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('channel_id', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('message_id', sa.BIGINT(), autoincrement=False, nullable=False), + sa.Column('content', sa.TEXT(), autoincrement=False, nullable=False), + sa.Column('author', sa.TEXT(), autoincrement=False, nullable=False), + sa.Column('timestamp', postgresql.TIMESTAMP(), autoincrement=False, nullable=False), + sa.Column('edited_timestamp', postgresql.TIMESTAMP(), autoincrement=False, nullable=True), + sa.Column('mentions', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('attachments', sa.TEXT(), autoincrement=False, nullable=True), + sa.Column('embeds', sa.TEXT(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('message_id', name='messages_pkey') + ) + # ### end Alembic commands ### \ No newline at end of file diff --git a/webapp/titanembeds/blueprints/api/api.py b/webapp/titanembeds/blueprints/api/api.py index 3564f84..1f241e6 100644 --- a/webapp/titanembeds/blueprints/api/api.py +++ b/webapp/titanembeds/blueprints/api/api.py @@ -1,4 +1,4 @@ -from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, Messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges, DiscordBotsOrgTransactions +from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, list_all_guild_members, get_guild_member, get_administrators_list, get_badges, DiscordBotsOrgTransactions from titanembeds.decorators import valid_session_required, discord_users_only, abort_if_guild_disabled from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles, get_online_embed_user_keys, redis_store, redisqueue from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild diff --git a/webapp/titanembeds/database/__init__.py b/webapp/titanembeds/database/__init__.py index 79671b8..72959c3 100644 --- a/webapp/titanembeds/database/__init__.py +++ b/webapp/titanembeds/database/__init__.py @@ -7,7 +7,6 @@ from .unauthenticated_users import UnauthenticatedUsers from .unauthenticated_bans import UnauthenticatedBans from .authenticated_users import AuthenticatedUsers from .guild_members import GuildMembers, list_all_guild_members, get_guild_member -from .messages import Messages, get_channel_messages from .cosmetics import Cosmetics, set_badges, get_badges, add_badge, remove_badge from .user_css import UserCSS from .administrators import Administrators, get_administrators_list diff --git a/webapp/titanembeds/database/messages.py b/webapp/titanembeds/database/messages.py deleted file mode 100644 index 15c4f78..0000000 --- a/webapp/titanembeds/database/messages.py +++ /dev/null @@ -1,83 +0,0 @@ -from titanembeds.database import db, get_guild_member -from sqlalchemy import cast -import json - -class Messages(db.Model): - __tablename__ = "messages" - message_id = db.Column(db.BigInteger, nullable=False, primary_key=True) # Message snowflake - guild_id = db.Column(db.BigInteger, nullable=False) # Discord guild id - channel_id = db.Column(db.BigInteger, nullable=False) # Channel id - content = db.Column(db.Text(), nullable=False) # Message contents - author = db.Column(db.Text(), nullable=False) # Author - timestamp = db.Column(db.TIMESTAMP, nullable=False) # Timestamp of when content is created - edited_timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is edited - mentions = db.Column(db.Text()) # Mentions serialized - attachments = db.Column(db.Text()) # serialized attachments - embeds = db.Column(db.Text().with_variant(db.Text(length=4294967295), 'mysql')) # message embeds - - def __init__(self, guild_id, channel_id, message_id, content, author, timestamp, edited_timestamp, mentions, attachments, embeds): - self.guild_id = guild_id - self.channel_id = channel_id - self.message_id = message_id - self.content = content - self.author = author - self.timestamp = timestamp - self.edited_timestamp = edited_timestamp - self.mentions = mentions - self.attachments = attachments - self.embeds = embeds - - def __repr__(self): - return ''.format(self.id, self.guild_id, self.guild_id, self.channel_id, self.message_id) - -def get_channel_messages(guild_id, channel_id, after_snowflake=None): - if not after_snowflake: - q = db.session.query(Messages).filter(Messages.channel_id == channel_id).order_by(Messages.timestamp.desc()).limit(50) - else: - q = db.session.query(Messages).filter(Messages.channel_id == channel_id).filter(Messages.message_id > after_snowflake).order_by(Messages.timestamp.desc()).limit(50) - msgs = [] - snowflakes = [] - guild_members = {} - for x in q: - if x.message_id in snowflakes: - continue - snowflakes.append(x.message_id) - embeds = x.embeds - if not embeds: - embeds = "[]" - message = { - "attachments": json.loads(x.attachments), - "timestamp": x.timestamp, - "id": str(x.message_id), - "edited_timestamp": x.edited_timestamp, - "author": json.loads(x.author), - "content": x.content, - "channel_id": str(x.channel_id), - "mentions": json.loads(x.mentions), - "embeds": json.loads(embeds), - } - if message["author"]["id"] not in guild_members: - member = get_guild_member(guild_id, message["author"]["id"]) - guild_members[message["author"]["id"]] = member - else: - member = guild_members[message["author"]["id"]] - message["author"]["nickname"] = None - if member: - message["author"]["nickname"] = member.nickname - message["author"]["avatar"] = member.avatar - message["author"]["discriminator"] = member.discriminator - message["author"]["username"] = member.username - for mention in message["mentions"]: - if mention["id"] not in guild_members: - author = get_guild_member(guild_id, mention["id"]) - guild_members[mention["id"]] = author - else: - author = guild_members[mention["id"]] - mention["nickname"] = None - if author: - mention["nickname"] = author.nickname - mention["avatar"] = author.avatar - mention["username"] = author.username - mention["discriminator"] = author.discriminator - msgs.append(message) - return msgs