Revert "Remove messages table", Accidentally committed in dev code

This reverts commit 6d6b390431.

discordbot/titanembeds/bot.py

@@ -81,6 +81,7 @@ class Titan(discord.AutoShardedClient):
 
     async def on_message(self, message):
         await self.socketio.on_message(message)
+        await self.database.push_message(message)
         await self.redisqueue.push_message(message)
 
         msg_arr = message.content.split() # split the message
@@ -95,11 +96,13 @@ class Titan(discord.AutoShardedClient):
                         await getattr(self.command, msg_cmd)(message) #actually run cmd, passing in msg obj
 
     async def on_message_edit(self, message_before, message_after):
+        await self.database.update_message(message_after)
         await self.redisqueue.update_message(message_after)
         await self.socketio.on_message_update(message_after)
 
     async def on_message_delete(self, message):
         self.delete_list.append(message.id)
+        await self.database.delete_message(message)
         await self.redisqueue.delete_message(message)
         await self.socketio.on_message_delete(message)
 

discordbot/titanembeds/database/__init__.py

@@ -6,6 +6,7 @@ import datetime
 db = Gino()
 
 from titanembeds.database.guilds import Guilds
+from titanembeds.database.messages import Messages
 from titanembeds.database.guild_members import GuildMembers
 from titanembeds.database.unauthenticated_users import UnauthenticatedUsers
 from titanembeds.database.unauthenticated_bans import UnauthenticatedBans
@@ -19,6 +20,37 @@ class DatabaseInterface(object):
     async def connect(self, dburi):
         await db.set_bind(dburi)
 
+    async def push_message(self, message):
+        if message.guild:
+            await Messages.create(
+                message_id = int(message.id),
+                guild_id = int(message.guild.id),
+                channel_id = int(message.channel.id),
+                content = message.content,
+                author = json.dumps(get_message_author(message)),
+                timestamp = message.created_at,
+                edited_timestamp = message.edited_at,
+                mentions = json.dumps(get_message_mentions(message.mentions)),
+                attachments = json.dumps(get_attachments_list(message.attachments)),
+                embeds = json.dumps(get_embeds_list(message.embeds))
+            )
+                
+    async def update_message(self, message):
+        if message.guild:
+            await Messages.update.values(
+                content = message.content,
+                timestamp = message.created_at,
+                edited_timestamp = message.edited_at,
+                mentions = json.dumps(get_message_mentions(message.mentions)),
+                attachments = json.dumps(get_attachments_list(message.attachments)),
+                embeds = json.dumps(get_embeds_list(message.embeds)),
+                author = json.dumps(get_message_author(message))
+            ).where(Messages.message_id == int(message.id)).gino.status()
+
+    async def delete_message(self, message):
+        if message.guild:
+            await Messages.delete.where(Messages.message_id == int(message.id)).gino.status()
+    
     async def update_guild(self, guild):
         if guild.me.guild_permissions.manage_webhooks:
             try:

discordbot/titanembeds/database/messages.py

@@ -0,0 +1,14 @@
+from titanembeds.database import db
+
+class Messages(db.Model):
+    __tablename__ = "messages"
+    message_id = db.Column(db.BigInteger, primary_key=True)          # Message snowflake
+    guild_id = db.Column(db.BigInteger)            # Discord guild id
+    channel_id = db.Column(db.BigInteger)          # Channel id
+    content = db.Column(db.Text())                  # Message contents
+    author = db.Column(db.Text())                   # Author json
+    timestamp = db.Column(db.TIMESTAMP)             # Timestamp of when content is created
+    edited_timestamp = db.Column(db.TIMESTAMP)      # Timestamp of when content is edited
+    mentions = db.Column(db.Text())                 # Mentions serialized
+    attachments = db.Column(db.Text())              # serialized attachments
+    embeds = db.Column(db.Text().with_variant(db.Text(length=4294967295), 'mysql')) # message embeds

discordbot/titanembeds/redisqueue.py

@@ -1,4 +1,4 @@
-from titanembeds.utils import get_formatted_message, get_formatted_user
+from titanembeds.utils import get_formatted_message
 from urllib.parse import urlparse
 import asyncio_redis
 import json
@@ -108,12 +108,3 @@ class RedisQueue:
     async def update_message(self, message):
         await self.delete_message(message)
         await self.push_message(message)
-    
-    async def on_get_guild_member(self, key, params):
-        member = self.bot.get_guild(int(params["guild_id"])).get_member(int(params["user_id"]))
-        if not member:
-            await self.connection.set(key, "")
-            return
-        await self.connection.delete([key])
-        user = get_formatted_user(member)
-        await self.connection.set(key, json.dumps(user))

discordbot/titanembeds/utils.py

@@ -50,7 +50,6 @@ def get_formatted_user(user):
         "status": str(user.status),
         "username": user.name,
         "nick": None,
-        "bot": user.bot
     }
     if userobj["color"] == "000000":
         userobj["color"] = None

webapp/alembic/versions/52271b243ba2_remove_messages_table_for_good.py

@@ -1,40 +0,0 @@
-"""Remove messages table for good
-
-Revision ID: 52271b243ba2
-Revises: 8e806bcb2228
-Create Date: 2018-07-16 23:33:31.600454
-
-"""
-
-# revision identifiers, used by Alembic.
-revision = '52271b243ba2'
-down_revision = '8e806bcb2228'
-branch_labels = None
-depends_on = None
-
-from alembic import op
-import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
-def upgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.drop_table('messages')
-    # ### end Alembic commands ###
-
-
-def downgrade():
-    # ### commands auto generated by Alembic - please adjust! ###
-    op.create_table('messages',
-    sa.Column('guild_id', sa.BIGINT(), autoincrement=False, nullable=False),
-    sa.Column('channel_id', sa.BIGINT(), autoincrement=False, nullable=False),
-    sa.Column('message_id', sa.BIGINT(), autoincrement=False, nullable=False),
-    sa.Column('content', sa.TEXT(), autoincrement=False, nullable=False),
-    sa.Column('author', sa.TEXT(), autoincrement=False, nullable=False),
-    sa.Column('timestamp', postgresql.TIMESTAMP(), autoincrement=False, nullable=False),
-    sa.Column('edited_timestamp', postgresql.TIMESTAMP(), autoincrement=False, nullable=True),
-    sa.Column('mentions', sa.TEXT(), autoincrement=False, nullable=True),
-    sa.Column('attachments', sa.TEXT(), autoincrement=False, nullable=True),
-    sa.Column('embeds', sa.TEXT(), autoincrement=False, nullable=True),
-    sa.PrimaryKeyConstraint('message_id', name='messages_pkey')
-    )
-    # ### end Alembic commands ###

webapp/titanembeds/blueprints/api/api.py

@@ -1,4 +1,4 @@
-from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, list_all_guild_members, get_guild_member, get_administrators_list, get_badges, DiscordBotsOrgTransactions
+from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, Messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges, DiscordBotsOrgTransactions
 from titanembeds.decorators import valid_session_required, discord_users_only, abort_if_guild_disabled
 from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles, get_online_embed_user_keys, redis_store, redisqueue
 from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild

webapp/titanembeds/database/__init__.py

@@ -7,6 +7,7 @@ from .unauthenticated_users import UnauthenticatedUsers
 from .unauthenticated_bans import UnauthenticatedBans
 from .authenticated_users import AuthenticatedUsers
 from .guild_members import GuildMembers, list_all_guild_members, get_guild_member
+from .messages import Messages, get_channel_messages
 from .cosmetics import Cosmetics, set_badges, get_badges, add_badge, remove_badge
 from .user_css import UserCSS
 from .administrators import Administrators, get_administrators_list

webapp/titanembeds/database/messages.py

@@ -0,0 +1,83 @@
+from titanembeds.database import db, get_guild_member
+from sqlalchemy import cast
+import json
+
+class Messages(db.Model):
+    __tablename__ = "messages"
+    message_id = db.Column(db.BigInteger, nullable=False, primary_key=True) # Message snowflake
+    guild_id = db.Column(db.BigInteger, nullable=False)            # Discord guild id
+    channel_id = db.Column(db.BigInteger, nullable=False)          # Channel id
+    content = db.Column(db.Text(), nullable=False)                  # Message contents
+    author = db.Column(db.Text(), nullable=False)                   # Author
+    timestamp = db.Column(db.TIMESTAMP, nullable=False)             # Timestamp of when content is created
+    edited_timestamp = db.Column(db.TIMESTAMP)                      # Timestamp of when content is edited
+    mentions = db.Column(db.Text())                                 # Mentions serialized
+    attachments = db.Column(db.Text())                              # serialized attachments
+    embeds = db.Column(db.Text().with_variant(db.Text(length=4294967295), 'mysql')) # message embeds
+
+    def __init__(self, guild_id, channel_id, message_id, content, author, timestamp, edited_timestamp, mentions, attachments, embeds):
+        self.guild_id = guild_id
+        self.channel_id = channel_id
+        self.message_id = message_id
+        self.content = content
+        self.author = author
+        self.timestamp = timestamp
+        self.edited_timestamp = edited_timestamp
+        self.mentions = mentions
+        self.attachments = attachments
+        self.embeds = embeds
+
+    def __repr__(self):
+        return '<Messages {0} {1} {2} {3} {4}>'.format(self.id, self.guild_id, self.guild_id, self.channel_id, self.message_id)
+
+def get_channel_messages(guild_id, channel_id, after_snowflake=None):
+    if not after_snowflake:
+        q = db.session.query(Messages).filter(Messages.channel_id == channel_id).order_by(Messages.timestamp.desc()).limit(50)
+    else:
+        q = db.session.query(Messages).filter(Messages.channel_id == channel_id).filter(Messages.message_id > after_snowflake).order_by(Messages.timestamp.desc()).limit(50)
+    msgs = []
+    snowflakes = []
+    guild_members = {}
+    for x in q:
+        if x.message_id in snowflakes:
+            continue
+        snowflakes.append(x.message_id)
+        embeds = x.embeds
+        if not embeds:
+            embeds = "[]"
+        message = {
+            "attachments": json.loads(x.attachments),
+            "timestamp": x.timestamp,
+            "id": str(x.message_id),
+            "edited_timestamp": x.edited_timestamp,
+            "author": json.loads(x.author),
+            "content": x.content,
+            "channel_id": str(x.channel_id),
+            "mentions": json.loads(x.mentions),
+            "embeds": json.loads(embeds),
+        }
+        if message["author"]["id"] not in guild_members:
+            member = get_guild_member(guild_id, message["author"]["id"])
+            guild_members[message["author"]["id"]] = member
+        else:
+            member = guild_members[message["author"]["id"]]
+        message["author"]["nickname"] = None
+        if member:
+            message["author"]["nickname"] = member.nickname
+            message["author"]["avatar"] = member.avatar
+            message["author"]["discriminator"] = member.discriminator
+            message["author"]["username"] = member.username
+        for mention in message["mentions"]:
+            if mention["id"] not in guild_members:
+                author = get_guild_member(guild_id, mention["id"])
+                guild_members[mention["id"]] = author
+            else:
+                author = guild_members[mention["id"]]
+            mention["nickname"] = None
+            if author:
+                mention["nickname"] = author.nickname
+                mention["avatar"] = author.avatar
+                mention["username"] = author.username
+                mention["discriminator"] = author.discriminator
+        msgs.append(message)
+    return msgs

webapp/titanembeds/redisqueue.py

@@ -1,4 +1,5 @@
 from titanembeds.utils import redis_store
+from titanembeds.database import get_guild_member
 import json
 import time
 
@@ -15,14 +16,14 @@ class RedisQueue:
             "params": params
         }
         loop_count = 0
-        while (not data and data != "") and loop_count < 50:
+        while not data and loop_count < 10:
-            if loop_count % 25 == 0:
+            if loop_count % 5 == 0:
                 redis_store.publish("discord-api-req", json.dumps(payload))
-            time.sleep(0.1)
+            time.sleep(0.5)
             data = self._get(key, data_type)
             loop_count += 1
         redis_store.expire(key, 60 * 5)
-        if data == None or data == "":
+        if data == None:
             return None
         if data_type == "set":
             data = list(data)
@@ -60,33 +61,28 @@ class RedisQueue:
                 "embeds": x["embeds"],
             }
             if message["author"]["id"] not in guild_members:
-                member = self.get_guild_member(guild_id, message["author"]["id"])
+                member = get_guild_member(guild_id, message["author"]["id"])
                 guild_members[message["author"]["id"]] = member
             else:
                 member = guild_members[message["author"]["id"]]
             message["author"]["nickname"] = None
             if member:
-                message["author"]["nickname"] = member["nick"]
+                message["author"]["nickname"] = member.nickname
-                message["author"]["avatar"] = member["avatar"]
+                message["author"]["avatar"] = member.avatar
-                message["author"]["discriminator"] = member["discriminator"]
+                message["author"]["discriminator"] = member.discriminator
-                message["author"]["username"] = member["username"]
+                message["author"]["username"] = member.username
             for mention in message["mentions"]:
                 if mention["id"] not in guild_members:
-                    author = self.get_guild_member(guild_id, mention["id"])
+                    author = get_guild_member(guild_id, mention["id"])
                     guild_members[mention["id"]] = author
                 else:
                     author = guild_members[mention["id"]]
                 mention["nickname"] = None
                 if author:
-                    mention["nickname"] = author["nick"]
+                    mention["nickname"] = author.nickname
-                    mention["avatar"] = author["avatar"]
+                    mention["avatar"] = author.avatar
-                    mention["username"] = author["username"]
+                    mention["username"] = author.username
-                    mention["discriminator"] = author["discriminator"]
+                    mention["discriminator"] = author.discriminator
             msgs.append(message)
         sorted_msgs = sorted(msgs, key=lambda k: k['id'], reverse=True) 
         return sorted_msgs
-    
-    def get_guild_member(self, guild_id, user_id):
-        key = "/guilds/{}/members/{}".format(guild_id, user_id)
-        q = self.get(key, "get_guild_member", {"guild_id": guild_id, "user_id": user_id})
-        return q