From 9c240939018d2e0ef90e18bc81e45146218ef55e Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sat, 6 May 2017 17:19:12 -0700 Subject: [PATCH] Fetching database for messages instead --- .gitignore | 4 +- discordbot/titanembeds/database/__init__.py | 29 ++++++++++- discordbot/titanembeds/database/messages.py | 4 +- webapp/titanembeds/blueprints/api/api.py | 11 ++--- webapp/titanembeds/database/__init__.py | 2 + .../titanembeds/database/custom_redislite.py | 35 ------------- webapp/titanembeds/database/guild_members.py | 28 +++++++++++ webapp/titanembeds/database/guilds.py | 12 ++++- webapp/titanembeds/database/messages.py | 49 +++++++++++++++++++ 9 files changed, 127 insertions(+), 47 deletions(-) delete mode 100644 webapp/titanembeds/database/custom_redislite.py create mode 100644 webapp/titanembeds/database/guild_members.py create mode 100644 webapp/titanembeds/database/messages.py diff --git a/.gitignore b/.gitignore index 1eb9de4..7f26ef5 100644 --- a/.gitignore +++ b/.gitignore @@ -92,5 +92,5 @@ ENV/ config.py redislite.db redislite.db.settings -tmp/* -!tmp/.gitinclude +webapp/tmp/* +!webapp/tmp/.gitinclude diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index af01c5a..ee2c08a 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -54,14 +54,38 @@ class DatabaseInterface(object): message.channel.id, message.id, message.content, + json.dumps(self.get_message_author(message)), str(message.timestamp), edit_ts, - json.dumps(message.mentions), + json.dumps(self.get_message_mentions(message.mentions)), json.dumps(message.attachments) ) session.add(msg) session.commit() + def get_message_author(self, message): + author = message.author + obj = { + "username": author.name, + "discriminator": author.discriminator, + "bot": author.bot, + "id": author.id, + "avatar": author.avatar + } + return obj + + def get_message_mentions(self, mentions): + ments = [] + for author in mentions: + ments.append({ + "username": author.name, + "discriminator": author.discriminator, + "bot": author.bot, + "id": author.id, + "avatar": author.avatar + }) + return ments + async def update_message(self, message): if message.server: async with threadpool(): @@ -73,8 +97,9 @@ class DatabaseInterface(object): if msg: msg.content = message.content msg.edited_timestamp = message.edited_timestamp - msg.mentions = json.dumps(message.mentions) + msg.mentions = json.dumps(self.get_message_mentions(message.mentions)) msg.attachments = json.dumps(message.attachments) + msg.author = json.dumps(self.get_message_author(message)) session.commit() async def delete_message(self, message): diff --git a/discordbot/titanembeds/database/messages.py b/discordbot/titanembeds/database/messages.py index d4bad0c..d7a9f20 100644 --- a/discordbot/titanembeds/database/messages.py +++ b/discordbot/titanembeds/database/messages.py @@ -7,16 +7,18 @@ class Messages(Base): channel_id = db.Column(db.String(255)) # Channel id message_id = db.Column(db.String(255)) # Message snowflake content = db.Column(db.Text()) # Message contents + author = db.Column(db.Text()) # Author json timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is created edited_timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is edited mentions = db.Column(db.Text()) # Mentions serialized attachments = db.Column(db.Text()) # serialized attachments - def __init__(self, guild_id, channel_id, message_id, content, timestamp, edited_timestamp, mentions, attachments): + def __init__(self, guild_id, channel_id, message_id, content, author, timestamp, edited_timestamp, mentions, attachments): self.guild_id = guild_id self.channel_id = channel_id self.message_id = message_id self.content = content + self.author = author self.timestamp = timestamp self.edited_timestamp = edited_timestamp self.mentions = mentions diff --git a/webapp/titanembeds/blueprints/api/api.py b/webapp/titanembeds/blueprints/api/api.py index 772d0c4..fe1d828 100644 --- a/webapp/titanembeds/blueprints/api/api.py +++ b/webapp/titanembeds/blueprints/api/api.py @@ -1,4 +1,4 @@ -from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, KeyValueProperties +from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, KeyValueProperties, get_channel_messages from titanembeds.decorators import valid_session_required, discord_users_only from titanembeds.utils import check_guild_existance, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild @@ -265,10 +265,9 @@ def fetch(): if not chan.get("read"): status_code = 401 else: - messages = discord_api.get_channel_messages(channel_id, after_snowflake) - status_code = messages['code'] - response = jsonify(messages=messages.get('content', messages), status=status) - response.status_code = status_code + messages = get_channel_messages(channel_id, after_snowflake) + response = jsonify(messages=messages, status=status) + response.status_code = 200 return response @api.route("/post", methods=["POST"]) @@ -382,4 +381,4 @@ def cleanup_keyval_db(): db.session.delete(m) db.session.commit() return ('', 204) - abort(401) \ No newline at end of file + abort(401) diff --git a/webapp/titanembeds/database/__init__.py b/webapp/titanembeds/database/__init__.py index 85fdafa..d62c160 100644 --- a/webapp/titanembeds/database/__init__.py +++ b/webapp/titanembeds/database/__init__.py @@ -6,4 +6,6 @@ from guilds import Guilds from unauthenticated_users import UnauthenticatedUsers from unauthenticated_bans import UnauthenticatedBans from authenticated_users import AuthenticatedUsers +from guild_members import GuildMembers from keyvalue_properties import KeyValueProperties, set_keyvalproperty, get_keyvalproperty, getexpir_keyvalproperty, setexpir_keyvalproperty, ifexists_keyvalproperty, delete_keyvalproperty +from messages import Messages, get_channel_messages diff --git a/webapp/titanembeds/database/custom_redislite.py b/webapp/titanembeds/database/custom_redislite.py deleted file mode 100644 index 4cade78..0000000 --- a/webapp/titanembeds/database/custom_redislite.py +++ /dev/null @@ -1,35 +0,0 @@ -import urlparse -from limits.storage import Storage -from redislite import Redis -import time - -class LimitsRedisLite(Storage): # For Python Limits - STORAGE_SCHEME = "redislite" - def __init__(self, uri, **options): - self.redis_instance = Redis(urlparse.urlparse(uri).netloc) - - def check(self): - return True - - def get_expiry(self, key): - return (self.redis_instance.ttl(key) or 0) + time.time() - - def incr(self, key, expiry, elastic_expiry=False): - if not self.redis_instance.exists(key): - self.redis_instance.set(key, 1, ex=expiry) - else: - oldexp = oldexp = self.get_expiry(key) - time.time() - if oldexp <= 0: - self.redis_instance.delete(key) - return self.incr(key, expiry, elastic_expiry) - self.redis_instance.set(key, int(self.redis_instance.get(key))+1, ex=int(round(oldexp))) - return int(self.get(key)) - - def get(self, key): - value = self.redis_instance.get(key) - if value: - return int(value) - return 0 - - def reset(self): - return self.redis_instance.flushdb() diff --git a/webapp/titanembeds/database/guild_members.py b/webapp/titanembeds/database/guild_members.py new file mode 100644 index 0000000..8488091 --- /dev/null +++ b/webapp/titanembeds/database/guild_members.py @@ -0,0 +1,28 @@ +from titanembeds.database import db + +class GuildMembers(db.Model): + __tablename__ = "guild_members" + id = db.Column(db.Integer, primary_key=True) # Auto incremented id + guild_id = db.Column(db.String(255)) # Discord guild id + user_id = db.Column(db.String(255)) # Discord user id + username = db.Column(db.String(255)) # Name + discriminator = db.Column(db.Integer) # User discriminator + nickname = db.Column(db.String(255)) # User nickname + avatar = db.Column(db.String(255)) # The avatar str of the user + active = db.Column(db.Boolean()) # If the user is a member of the guild + banned = db.Column(db.Boolean()) # If the user is banned in the guild + roles = db.Column(db.Text()) # Member roles + + def __init__(self, guild_id, user_id, username, discriminator, nickname, avatar, active, banned, roles): + self.guild_id = guild_id + self.user_id = user_id + self.username = username + self.discriminator = discriminator + self.nickname = nickname + self.avatar = avatar + self.active = active + self.banned = banned + self.roles = roles + + def __repr__(self): + return ''.format(self.id, self.guild_id, self.user_id, self.username, self.discriminator) diff --git a/webapp/titanembeds/database/guilds.py b/webapp/titanembeds/database/guilds.py index 858be37..e308dfc 100644 --- a/webapp/titanembeds/database/guilds.py +++ b/webapp/titanembeds/database/guilds.py @@ -4,11 +4,21 @@ class Guilds(db.Model): __tablename__ = "guilds" id = db.Column(db.Integer, primary_key=True) # Auto incremented id guild_id = db.Column(db.String(255)) # Discord guild id + name = db.Column(db.String(255)) # Name unauth_users = db.Column(db.Boolean()) # If allowed unauth users + roles = db.Column(db.Text()) # Guild Roles + channels = db.Column(db.Text()) # Guild channels + owner_id = db.Column(db.String(255)) # Snowflake of the owner + icon = db.Column(db.String(255)) # The icon string, null if none - def __init__(self, guild_id): + def __init__(self, guild_id, name, roles, channels, owner_id, icon): self.guild_id = guild_id + self.name = name self.unauth_users = True # defaults to true + self.roles = roles + self.channels = channels + self.owner_id = owner_id + self.icon = icon def __repr__(self): return ''.format(self.id, self.guild_id) diff --git a/webapp/titanembeds/database/messages.py b/webapp/titanembeds/database/messages.py new file mode 100644 index 0000000..66c7a76 --- /dev/null +++ b/webapp/titanembeds/database/messages.py @@ -0,0 +1,49 @@ +from titanembeds.database import db +from sqlalchemy import cast +import json + +class Messages(db.Model): + __tablename__ = "messages" + id = db.Column(db.Integer, primary_key=True) # Auto incremented id + guild_id = db.Column(db.String(255)) # Discord guild id + channel_id = db.Column(db.String(255)) # Channel id + message_id = db.Column(db.String(255)) # Message snowflake + content = db.Column(db.Text()) # Message contents + author = db.Column(db.Text()) # Author + timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is created + edited_timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is edited + mentions = db.Column(db.Text()) # Mentions serialized + attachments = db.Column(db.Text()) # serialized attachments + + def __init__(self, guild_id, channel_id, message_id, content, author, timestamp, edited_timestamp, mentions, attachments): + self.guild_id = guild_id + self.channel_id = channel_id + self.message_id = message_id + self.content = content + self.author = author + self.timestamp = timestamp + self.edited_timestamp = edited_timestamp + self.mentions = mentions + self.attachments = attachments + + def __repr__(self): + return ''.format(self.id, self.guild_id, self.guild_id, self.channel_id, self.message_id) + +def get_channel_messages(channel_id, after_snowflake=None): + if not after_snowflake: + q = db.session.query(Messages).filter(Messages.channel_id == channel_id).order_by(Messages.id.desc()).limit(50) + else: + q = db.session.query(Messages).filter(cast(Messages.channel_id, db.Integer) == int(channel_id)).filter(Messages.message_id > after_snowflake).order_by(Messages.id.desc()).limit(50) + msgs = [] + for x in q: + msgs.append({ + "attachments": json.loads(x.attachments), + "timestamp": x.timestamp, + "id": x.message_id, + "edited_timestamp": x.edited_timestamp, + "author": json.loads(x.author), + "content": x.content, + "channel_id": x.channel_id, + "mentions": json.loads(x.mentions) + }) + return msgs