Merge pull request #4 from EndenDragon/websockets-bot

Websockets bot implementation

.gitignore

@@ -92,5 +92,5 @@ ENV/
 config.py
 redislite.db
 redislite.db.settings
-tmp/*
+webapp/tmp/*
-!tmp/.gitinclude
+!webapp/tmp/.gitinclude

README.md

@@ -9,15 +9,10 @@ There was a time when Discord doesn't support embedding the chat on a webpage. B
 - Moderation Features (Kick & ban users by IP addresses, toggling guest users)
 - Discord OAuth support. (Allows those who have a discord account to access the embed)
 - Responsive material design! (Thanks materializecss!!)
-- All features are done via REST apis (respects discord's rate limiting). Although do not provide consistant connection to Discord, they are easier to maintain and does not often "disconnects" from Discord servers.
 
 # Installation
-Would you like to run your own copy of Titan Embeds?
+Would you like to run your own copy of Titan Embeds? There are two parts that integrate nicely together. The webapp (website) handles the frontend and communication with the database to retrieve server messages, etc. The discordbot (bot) handles the communcation
-1. Clone the repo (make sure you have python 2.7 installed on your system. This project depends on that specific python version)
+between Discord's websockets and pushing out the data to the database for the webapp. Check out the respective folder for their installation (pay attention to the python versions!) instructions.
-2. Install the pip requirements `pip install -r requirements.txt`
-3. Clone `config.example.py` and rename it to `config.py`. Edit the file to your standards
-4. Make sure that the bot is online in the websockets once. This is required because the bot cannot send messages until it has used the ws. Use something like discord.py to log the bot into discord websockets. You can close it afterwards. So basically if the bot account is online ONCE in it's lifespan- you're good.
-5. Run the development web via `python run.py` -- Though we suggest to use a better server software (look into gunicorn, nginx, uwsgi, etc)
 
 
 ## Join us!

discordbot/README.md

@@ -0,0 +1,9 @@
+# Titan - DiscordBot Portion
+The DiscordBot portion handles the communcation with Discord's websockets to provide real-time updates. The bot's primary role is to push content to the webapp's database to be retrieved at a later time.
+It also includes misc. features to moderate guest users, etc. right in your discord server!
+
+# Installation
+1. Clone the repo (make sure you have **Python 3.5** installed on your system. This discordbot portion depends on that specifc Python version)
+2. Install the pip requirements `pip install -r requirements.txt`
+3. Clone `config.example.py` and rename it to `config.py`. Edit the file to your standards
+4. Start the bot using `python run.py`

discordbot/config.example.py

@@ -0,0 +1,5 @@
+config = {
+    'bot-token': "Discord bot token",
+    
+    'database-uri': "driver://username:password@host:port/database",
+}

discordbot/requirements.txt

@@ -0,0 +1,3 @@
+discord.py
+sqlalchemy
+asyncio_extras

discordbot/run.py

@@ -0,0 +1,11 @@
+from titanembeds import Titan
+import gc
+
+def main():
+    print("Starting...")
+    te = Titan()
+    te.run()
+    gc.collect()
+
+if __name__ == '__main__':
+    main()

discordbot/titanembeds/__init__.py

@@ -0,0 +1 @@
+from titanembeds.bot import Titan

discordbot/titanembeds/bot.py

@@ -0,0 +1,131 @@
+from config import config
+from titanembeds.database import DatabaseInterface
+import discord
+import aiohttp
+import asyncio
+
+class Titan(discord.Client):
+    def __init__(self):
+        super().__init__()
+        self.aiosession = aiohttp.ClientSession(loop=self.loop)
+        self.http.user_agent += ' TitanEmbeds-Bot'
+        self.database = DatabaseInterface(self)
+
+    def _cleanup(self):
+        try:
+            self.loop.run_until_complete(self.logout())
+        except: # Can be ignored
+            pass
+        pending = asyncio.Task.all_tasks()
+        gathered = asyncio.gather(*pending)
+        try:
+            gathered.cancel()
+            self.loop.run_until_complete(gathered)
+            gathered.exception()
+        except: # Can be ignored
+            pass
+
+    def run(self):
+        try:
+            self.loop.run_until_complete(self.start(config["bot-token"]))
+        except discord.errors.LoginFailure:
+            print("Invalid bot token in config!")
+        finally:
+            try:
+                self._cleanup()
+            except Exception as e:
+                print("Error in cleanup:", e)
+            self.loop.close()
+
+    async def on_ready(self):
+        print('Titan [DiscordBot]')
+        print('Logged in as the following user:')
+        print(self.user.name)
+        print(self.user.id)
+        print('------')
+
+        await self.change_presence(
+            game=discord.Game(name="iFrame your server! Visit https://TitanEmbeds.tk/ today!"), status=discord.Status.online
+        )
+
+        try:
+            await self.database.connect(config["database-uri"] + "?charset=utf8")
+        except Exception:
+            self.logger.error("Unable to connect to specified database!")
+            traceback.print_exc()
+            await self.logout()
+            return
+
+        for server in self.servers:
+            await self.database.update_guild(server)
+            if server.large:
+                await request_offline_members(server)
+            server_bans = await self.get_bans(server)
+            for member in server.members:
+                banned = member.id in [u.id for u in server_bans]
+                await self.database.update_guild_member(
+                    member,
+                    True,
+                    banned
+                )
+            await self.database.flag_unactive_guild_members(server.id, server.members)
+            await self.database.flag_unactive_bans(server.id, server_bans)
+        await self.database.remove_unused_guilds(self.servers)
+
+    async def on_message(self, message):
+        await self.database.push_message(message)
+        # TODO: Will add command handler + ban/kick command
+
+    async def on_message_edit(self, message_before, message_after):
+        await self.database.update_message(message_after)
+
+    async def on_message_delete(self, message):
+        await self.database.delete_message(message)
+
+    async def on_server_join(self, guild):
+        await self.database.update_guild(guild)
+
+    async def on_server_remove(self, guild):
+        await self.database.remove_guild(guild)
+
+    async def on_server_update(self, guildbefore, guildafter):
+        await self.database.update_guild(guildafter)
+
+    async def on_server_role_create(self, role):
+        if role.name == self.user.name and role.managed:
+            await asyncio.sleep(2)
+        await self.database.update_guild(role.server)
+
+    async def on_server_role_delete(self, role):
+        if role.server.me not in role.server.members:
+            return
+        await self.database.update_guild(role.server)
+
+    async def on_server_role_update(self, rolebefore, roleafter):
+        await self.database.update_guild(roleafter.server)
+
+    async def on_channel_delete(self, channel):
+        await self.database.update_guild(channel.server)
+
+    async def on_channel_create(self, channel):
+        await self.database.update_guild(channel.server)
+
+    async def on_channel_update(self, channelbefore, channelafter):
+        await self.database.update_guild(channelafter.server)
+
+    async def on_member_join(self, member):
+        await self.database.update_guild_member(member, active=True, banned=False)
+
+    async def on_member_remove(self, member):
+        await self.database.update_guild_member(member, active=False, banned=False)
+
+    async def on_member_update(self, memberbefore, memberafter):
+        await self.database.update_guild_member(memberafter)
+
+    async def on_member_ban(self, member):
+        if role.server.me not in role.server.members:
+            return
+        await self.database.update_guild_member(member, active=False, banned=True)
+
+    async def on_member_unban(self, member):
+        await self.database.update_guild_member(member, active=False, banned=False)

discordbot/titanembeds/database/__init__.py

@@ -0,0 +1,266 @@
+from contextlib import contextmanager
+from asyncio_extras import threadpool
+import sqlalchemy as db
+from sqlalchemy.engine import Engine, create_engine
+from sqlalchemy.orm import sessionmaker, Session
+from sqlalchemy.ext.declarative import declarative_base
+
+import json
+import discord
+
+Base = declarative_base()
+
+from titanembeds.database.guilds import Guilds
+from titanembeds.database.messages import Messages
+from titanembeds.database.guild_members import GuildMembers
+
+class DatabaseInterface(object):
+    # Courtesy of https://github.com/SunDwarf/Jokusoramame
+    def __init__(self, bot):
+        self.bot = bot
+
+        self.engine = None  # type: Engine
+        self._sessionmaker = None  # type: sessionmaker
+
+    async def connect(self, dburi):
+        async with threadpool():
+            self.engine = create_engine(dburi, pool_recycle=10)
+            self._sessionmaker = sessionmaker(bind=self.engine, expire_on_commit=False)
+
+    @contextmanager
+    def get_session(self) -> Session:
+        session = self._sessionmaker()  # type: Session
+        try:
+            yield session
+            session.commit()
+        except:
+            session.rollback()
+            raise
+        finally:
+            session.close()
+
+    async def push_message(self, message):
+        if message.server:
+            async with threadpool():
+                with self.get_session() as session:
+                    edit_ts = message.edited_timestamp
+                    if not edit_ts:
+                        edit_ts = None
+                    else:
+                        edit_ts = str(edit_ts)
+
+                    msg = Messages(
+                        message.server.id,
+                        message.channel.id,
+                        message.id,
+                        message.content,
+                        json.dumps(self.get_message_author(message)),
+                        str(message.timestamp),
+                        edit_ts,
+                        json.dumps(self.get_message_mentions(message.mentions)),
+                        json.dumps(message.attachments)
+                    )
+                    session.add(msg)
+                    session.commit()
+
+    def get_message_author(self, message):
+        author = message.author
+        obj = {
+            "username": author.name,
+            "discriminator": author.discriminator,
+            "bot": author.bot,
+            "id": author.id,
+            "avatar": author.avatar
+        }
+        return obj
+
+    def get_message_mentions(self, mentions):
+        ments = []
+        for author in mentions:
+            ments.append({
+                "username": author.name,
+                "discriminator": author.discriminator,
+                "bot": author.bot,
+                "id": author.id,
+                "avatar": author.avatar
+            })
+        return ments
+
+    async def update_message(self, message):
+        if message.server:
+            async with threadpool():
+                with self.get_session() as session:
+                    msg = session.query(Messages) \
+                        .filter(Messages.guild_id == message.server.id) \
+                        .filter(Messages.channel_id == message.channel.id) \
+                        .filter(Messages.message_id == message.id).first()
+                    if msg:
+                        msg.content = message.content
+                        msg.edited_timestamp = message.edited_timestamp
+                        msg.mentions = json.dumps(self.get_message_mentions(message.mentions))
+                        msg.attachments = json.dumps(message.attachments)
+                        msg.author = json.dumps(self.get_message_author(message))
+                        session.commit()
+
+    async def delete_message(self, message):
+        if message.server:
+            async with threadpool():
+                with self.get_session() as session:
+                    msg = session.query(Messages) \
+                        .filter(Messages.guild_id == message.server.id) \
+                        .filter(Messages.channel_id == message.channel.id) \
+                        .filter(Messages.message_id == message.id).first()
+                    if msg:
+                        session.delete(msg)
+                        session.commit()
+
+    async def update_guild(self, guild):
+        async with threadpool():
+            with self.get_session() as session:
+                gui = session.query(Guilds).filter(Guilds.guild_id == guild.id).first()
+                if not gui:
+                    gui = Guilds(
+                        guild.id,
+                        guild.name,
+                        json.dumps(self.get_roles_list(guild.roles)),
+                        json.dumps(self.get_channels_list(guild.channels)),
+                        guild.owner_id,
+                        guild.icon
+                    )
+                    session.add(gui)
+                else:
+                    gui.name = guild.name
+                    gui.roles = json.dumps(self.get_roles_list(guild.roles))
+                    gui.channels = json.dumps(self.get_channels_list(guild.channels))
+                    gui.owner_id = guild.owner_id
+                    gui.icon = guild.icon
+                session.commit()
+
+    def get_roles_list(self, guildroles):
+        roles = []
+        for role in guildroles:
+            roles.append({
+                "id": role.id,
+                "name": role.name,
+                "color": role.color.value,
+                "hoist": role.hoist,
+                "position": role.position,
+                "permissions": role.permissions.value
+            })
+        return roles
+
+    def get_channels_list(self, guildchannels):
+        channels = []
+        for channel in guildchannels:
+            if str(channel.type) == "text":
+                overwrites = []
+                for target, overwrite in channel.overwrites:
+                    if isinstance(target, discord.Role):
+                        type = "role"
+                    else:
+                        type = "member"
+                    allow, deny = overwrite.pair()
+                    allow = allow.value
+                    deny = deny.value
+                    overwrites.append({
+                        "id": target.id,
+                        "type": type,
+                        "allow": allow,
+                        "deny": deny,
+                    })
+
+                channels.append({
+                    "id": channel.id,
+                    "name": channel.name,
+                    "topic": channel.topic,
+                    "position": channel.position,
+                    "type": str(channel.type),
+                    "permission_overwrites": overwrites
+                })
+        return channels
+
+    async def remove_unused_guilds(self, guilds):
+        async with threadpool():
+            with self.get_session() as session:
+                dbguilds = session.query(Guilds).all()
+                changed = False
+                for guild in dbguilds:
+                    disguild = discord.utils.get(guilds, id=guild.guild_id)
+                    if not disguild:
+                        changed = True
+                        session.delete(guild)
+                if changed:
+                    session.commit()
+
+    async def remove_guild(self, guild):
+        async with threadpool():
+            with self.get_session() as session:
+                gui = session.query(Guilds).filter(Guilds.guild_id == guild.id).first()
+                if gui:
+                    session.delete(gui)
+                    session.commit()
+
+    async def update_guild_member(self, member, active=True, banned=False):
+        async with threadpool():
+            with self.get_session() as session:
+                dbmember = session.query(GuildMembers) \
+                    .filter(GuildMembers.guild_id == member.server.id) \
+                    .filter(GuildMembers.user_id == member.id).first()
+                if not dbmember:
+                    dbmember = GuildMembers(
+                        member.server.id,
+                        member.id,
+                        member.name,
+                        member.discriminator,
+                        member.nick,
+                        member.avatar,
+                        active,
+                        banned,
+                        json.dumps(self.list_role_ids(member.roles))
+                    )
+                    session.add(dbmember)
+                else:
+                    dbmember.banned = banned
+                    dbmember.active = active
+                    dbmember.username = member.name
+                    dbmember.discriminator = member.discriminator
+                    dbmember.nick = member.nick
+                    dbmember.avatar = member.avatar
+                    dbmember.roles = json.dumps(self.list_role_ids(member.roles))
+                session.commit()
+
+    async def flag_unactive_guild_members(self, guild_id, guild_members):
+        async with threadpool():
+            with self.get_session() as session:
+                changed = False
+                dbmembers = session.query(GuildMembers) \
+                    .filter(GuildMembers.guild_id == guild_id) \
+                    .filter(GuildMembers.active == True).all()
+                for member in dbmembers:
+                    dismember = discord.utils.get(guild_members, id=member.user_id)
+                    if not dismember:
+                        changed = True
+                        member.active = False
+                if changed:
+                    session.commit()
+
+    def list_role_ids(self, usr_roles):
+        ids = []
+        for role in usr_roles:
+            ids.append(role.id)
+        return ids
+
+    async def flag_unactive_bans(self, guild_id, guildbans):
+        async with threadpool():
+            with self.get_session() as session:
+                changed = False
+                for usr in guildbans:
+                    dbusr = session.query(GuildMembers) \
+                        .filter(GuildMembers.guild_id == guild_id) \
+                        .filter(GuildMembers.user_id == usr.id) \
+                        .filter(GuildMembers.active == False).first()
+                    if dbusr:
+                        changed = True
+                        dbusr.banned = True
+                if changed:
+                    session.commit()

discordbot/titanembeds/database/guild_members.py

@@ -0,0 +1,28 @@
+from titanembeds.database import db, Base
+
+class GuildMembers(Base):
+    __tablename__ = "guild_members"
+    id = db.Column(db.Integer, primary_key=True)    # Auto incremented id
+    guild_id = db.Column(db.String(255))            # Discord guild id
+    user_id = db.Column(db.String(255))             # Discord user id
+    username = db.Column(db.String(255))            # Name
+    discriminator = db.Column(db.Integer)           # User discriminator
+    nickname = db.Column(db.String(255))            # User nickname
+    avatar = db.Column(db.String(255))              # The avatar str of the user
+    active = db.Column(db.Boolean())                # If the user is a member of the guild
+    banned = db.Column(db.Boolean())                # If the user is banned in the guild
+    roles = db.Column(db.Text())                    # Member roles
+
+    def __init__(self, guild_id, user_id, username, discriminator, nickname, avatar, active, banned, roles):
+        self.guild_id = guild_id
+        self.user_id = user_id
+        self.username = username
+        self.discriminator = discriminator
+        self.nickname = nickname
+        self.avatar = avatar
+        self.active = active
+        self.banned = banned
+        self.roles = roles
+
+    def __repr__(self):
+        return '<GuildMembers {0} {1} {2} {3} {4}>'.format(self.id, self.guild_id, self.user_id, self.username, self.discriminator)

discordbot/titanembeds/database/guilds.py

@@ -0,0 +1,24 @@
+from titanembeds.database import db, Base
+
+class Guilds(Base):
+    __tablename__ = "guilds"
+    id = db.Column(db.Integer, primary_key=True)    # Auto incremented id
+    guild_id = db.Column(db.String(255))            # Discord guild id
+    name = db.Column(db.String(255))                # Name
+    unauth_users = db.Column(db.Boolean())          # If allowed unauth users
+    roles = db.Column(db.Text())                    # Guild Roles
+    channels = db.Column(db.Text())                 # Guild channels
+    owner_id = db.Column(db.String(255))            # Snowflake of the owner
+    icon = db.Column(db.String(255))                # The icon string, null if none
+
+    def __init__(self, guild_id, name, roles, channels, owner_id, icon):
+        self.guild_id = guild_id
+        self.name = name
+        self.unauth_users = True # defaults to true
+        self.roles = roles
+        self.channels = channels
+        self.owner_id = owner_id
+        self.icon = icon
+
+    def __repr__(self):
+        return '<Guilds {0} {1}>'.format(self.id, self.guild_id)

discordbot/titanembeds/database/messages.py

@@ -0,0 +1,28 @@
+from titanembeds.database import db, Base
+
+class Messages(Base):
+    __tablename__ = "messages"
+    id = db.Column(db.Integer, primary_key=True)    # Auto incremented id
+    guild_id = db.Column(db.String(255))            # Discord guild id
+    channel_id = db.Column(db.String(255))          # Channel id
+    message_id = db.Column(db.String(255))          # Message snowflake
+    content = db.Column(db.Text())                  # Message contents
+    author = db.Column(db.Text())                   # Author json
+    timestamp = db.Column(db.TIMESTAMP)             # Timestamp of when content is created
+    edited_timestamp = db.Column(db.TIMESTAMP)      # Timestamp of when content is edited
+    mentions = db.Column(db.Text())                 # Mentions serialized
+    attachments = db.Column(db.Text())              # serialized attachments
+
+    def __init__(self, guild_id, channel_id, message_id, content, author, timestamp, edited_timestamp, mentions, attachments):
+        self.guild_id = guild_id
+        self.channel_id = channel_id
+        self.message_id = message_id
+        self.content = content
+        self.author = author
+        self.timestamp = timestamp
+        self.edited_timestamp = edited_timestamp
+        self.mentions = mentions
+        self.attachments = attachments
+
+    def __repr__(self):
+        return '<Messages {0} {1} {2} {3} {4}>'.format(self.id, self.guild_id, self.guild_id, self.channel_id, self.message_id)

titanembeds/database/custom_redislite.py

@@ -1,35 +0,0 @@
-import urlparse
-from limits.storage import Storage
-from redislite import Redis
-import time
-
-class LimitsRedisLite(Storage): # For Python Limits
-    STORAGE_SCHEME = "redislite"
-    def __init__(self, uri, **options):
-        self.redis_instance = Redis(urlparse.urlparse(uri).netloc)
-
-    def check(self):
-        return True
-
-    def get_expiry(self, key):
-        return (self.redis_instance.ttl(key) or 0) + time.time()
-
-    def incr(self, key, expiry, elastic_expiry=False):
-        if not self.redis_instance.exists(key):
-            self.redis_instance.set(key, 1, ex=expiry)
-        else:
-            oldexp = oldexp = self.get_expiry(key) - time.time()
-            if oldexp <= 0:
-                self.redis_instance.delete(key)
-                return self.incr(key, expiry, elastic_expiry)
-            self.redis_instance.set(key, int(self.redis_instance.get(key))+1, ex=int(round(oldexp)))
-        return int(self.get(key))
-
-    def get(self, key):
-        value = self.redis_instance.get(key)
-        if value:
-            return int(value)
-        return 0
-
-    def reset(self):
-        return self.redis_instance.flushdb()

webapp/README.md

@@ -0,0 +1,8 @@
+# Titan - WebApp Portion
+The webapp portion handles the frontend (it's what the users see). The webapp highly depends on the discordbot to push websockets data to the database.
+
+# Installation
+1. Clone the repo (make sure you have **Python 2.7** installed on your system. This webapp portion depends on that specific python version)
+2. Install the pip requirements `pip install -r requirements.txt`
+3. Clone `config.example.py` and rename it to `config.py`. Edit the file to your standards
+4. Run the development web via `python run.py` -- Though we suggest to use a better server software (look into gunicorn, nginx, uwsgi, etc)

webapp/config.example.py

@@ -5,7 +5,7 @@ config = {
     'client-secret': "Your discord client secret",
     'bot-token': "Discord bot token",
 
-    'app-location': "/var/www/Titan/",
+    'app-location': "/var/www/Titan/webapp/",
     'app-secret': "Type something random here, go wild.",
 
     'database-uri': "driver://username:password@host:port/database",

webapp/titanembeds/blueprints/api/api.py

@@ -1,4 +1,4 @@
-from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, KeyValueProperties
+from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, KeyValueProperties, GuildMembers, get_channel_messages, list_all_guild_members
 from titanembeds.decorators import valid_session_required, discord_users_only
 from titanembeds.utils import check_guild_existance, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key
 from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild
@@ -27,9 +27,8 @@ def checkUserRevoke(guild_id, user_key=None):
         banned = checkUserBanned(guild_id)
         if banned:
             return revoked
-        member = discord_api.get_guild_member_nocache(guild_id, session['user_id'])
+        dbUser = GuildMembers.query.filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == session["user_id"]).first()
-        if member['code'] == 200:
+        revoked = not dbUser.active
-            revoked = False
     return revoked
 
 def checkUserBanned(guild_id, ip_address=None):
@@ -44,10 +43,8 @@ def checkUserBanned(guild_id, ip_address=None):
                     banned = False
     else:
         banned = False
-        bans = discord_api.get_guild_bans(guild_id)['content']
+        dbUser = GuildMembers.query.filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == session["user_id"]).first()
-        for user in bans:
+        banned = dbUser.banned
-            if session['user_id'] == user['user']['id']:
-                return True
     return banned
 
 def update_user_status(guild_id, username, user_key=None):
@@ -97,7 +94,7 @@ def check_user_in_guild(guild_id):
         return guild_id in session['user_keys']
     else:
         dbUser = db.session.query(AuthenticatedUsers).filter(and_(AuthenticatedUsers.guild_id == guild_id, AuthenticatedUsers.client_id == session['user_id'])).first()
-        return 200 == discord_api.get_guild_member_nocache(guild_id, session['user_id'])['code'] and dbUser is not None
+        return not checkUserRevoke(guild_id) and dbUser is not None
 
 def format_post_content(message):
     message = message.replace("<", "\<")
@@ -114,20 +111,28 @@ def format_post_content(message):
         message = "**<{}#{}>** {}".format(session['username'], session['discriminator'], message) # I would like to do a @ mention, but i am worried about notif spam
     return message
 
+def get_member_roles(guild_id, user_id):
+    q = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == user_id).first()
+    return json.loads(q.roles)
+
+def get_dbguild_channels(guild_id):
+    q = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first()
+    return json.loads(q.channels)
+
 def get_guild_channels(guild_id):
     if user_unauthenticated():
         member_roles = [guild_id] #equivilant to @everyone role
     else:
-        member = discord_api.get_guild_member(guild_id, session['user_id'])['content']
+        member_roles = get_member_roles(guild_id, session['user_id'])
-        member_roles = member['roles']
         if guild_id not in member_roles:
             member_roles.append(guild_id)
-    guild_channels = discord_api.get_guild_channels(guild_id)['content']
+    dbguild = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first()
-    guild_roles = discord_api.get_guild_roles(guild_id)["content"]
+    guild_channels = json.loads(dbguild.channels)
-    guild_owner = discord_api.get_guild(guild_id)['content']['owner_id']
+    guild_roles = json.loads(dbguild.roles)
+    guild_owner = json.loads(dbguild.owner_id)
     result_channels = []
     for channel in guild_channels:
-        if channel['type'] == 0:
+        if channel['type'] == "text":
             result = {"channel": channel, "read": False, "write": False}
             if guild_owner == session['user_id']:
                 result["read"] = True
@@ -192,17 +197,17 @@ def get_guild_channels(guild_id):
 def filter_guild_channel(guild_id, channel_id):
     channels = get_guild_channels(guild_id)
     for chan in channels:
-        if chan["channel"]["id"] == guild_id:
+        if chan["channel"]["id"] == channel_id:
             return chan
     return None
 
 def get_online_discord_users(guild_id):
     embed = discord_api.get_widget(guild_id)
-    apimembers = discord_api.list_all_guild_members(guild_id)
+    apimembers = list_all_guild_members(guild_id)
     apimembers_filtered = {}
     for member in apimembers:
         apimembers_filtered[member["user"]["id"]] = member
-    guild_roles = discord_api.get_guild_roles(guild_id)["content"]
+    guild_roles = json.loads(db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first().roles)
     guildroles_filtered = {}
     for role in guild_roles:
         guildroles_filtered[role["id"]] = role
@@ -235,12 +240,12 @@ def get_online_embed_users(guild_id):
         users['unauthenticated'].append(meta)
     for user in auths:
         client_id = user.client_id
-        u = discord_api.get_guild_member(guild_id, client_id)['content']['user']
+        usrdb = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == client_id).first()
         meta = {
-            'id': u['id'],
+            'id': usrdb.user_id,
-            'username': u['username'],
+            'username': usrdb.username,
-            'discriminator': u['discriminator'],
+            'discriminator': usrdb.discriminator,
-            'avatar_url': generate_avatar_url(u['id'], u['avatar']),
+            'avatar_url': generate_avatar_url(usrdb.user_id, usrdb.avatar),
         }
         users['authenticated'].append(meta)
     return users
@@ -265,10 +270,9 @@ def fetch():
         if not chan.get("read"):
             status_code = 401
         else:
-            messages = discord_api.get_channel_messages(channel_id, after_snowflake)
+            messages = get_channel_messages(channel_id, after_snowflake)
-            status_code = messages['code']
+    response = jsonify(messages=messages, status=status)
-    response = jsonify(messages=messages.get('content', messages), status=status)
+    response.status_code = 200
-    response.status_code = status_code
     return response
 
 @api.route("/post", methods=["POST"])

webapp/titanembeds/blueprints/embed/embed.py

@@ -1,6 +1,7 @@
 from flask import Blueprint, render_template, abort, redirect, url_for, session
-from titanembeds.utils import check_guild_existance, discord_api, guild_query_unauth_users_bool
+from titanembeds.utils import check_guild_existance, guild_query_unauth_users_bool
 from titanembeds.oauth import generate_guild_icon_url, generate_avatar_url
+from titanembeds.database import db, Guilds
 from config import config
 import random
 
@@ -22,7 +23,7 @@ def get_logingreeting():
 @embed.route("/<string:guild_id>")
 def guild_embed(guild_id):
     if check_guild_existance(guild_id):
-        guild = discord_api.get_guild(guild_id)['content']
+        guild = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first()
         return render_template("embed.html.j2",
             login_greeting=get_logingreeting(),
             guild_id=guild_id, guild=guild,

webapp/titanembeds/blueprints/user/user.py

@@ -1,7 +1,6 @@
 from flask import Blueprint, request, redirect, jsonify, abort, session, url_for, render_template
 from config import config
 from titanembeds.decorators import discord_users_only
-from titanembeds.utils import discord_api
 from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans
 from titanembeds.oauth import authorize_url, token_url, make_authenticated_session, get_current_authenticated_user, get_user_managed_servers, check_user_can_administrate_guild, check_user_permission, generate_avatar_url, generate_guild_icon_url, generate_bot_invite_url
 import time
@@ -74,16 +73,11 @@ def dashboard():
 def administrate_guild(guild_id):
     if not check_user_can_administrate_guild(guild_id):
         return redirect(url_for("user.dashboard"))
-    guild = discord_api.get_guild(guild_id)
+    db_guild = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first()
-    if guild['code'] != 200:
+    if not db_guild:
         session["redirect"] = url_for("user.administrate_guild", guild_id=guild_id, _external=True)
         return redirect(generate_bot_invite_url(guild_id))
     session["redirect"] = None
-    db_guild = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first()
-    if not db_guild:
-        db_guild = Guilds(guild_id)
-        db.session.add(db_guild)
-        db.session.commit()
     permissions=[]
     if check_user_permission(guild_id, 5):
         permissions.append("Manage Embed Settings")
@@ -95,18 +89,15 @@ def administrate_guild(guild_id):
     all_bans = db.session.query(UnauthenticatedBans).filter(UnauthenticatedBans.guild_id == guild_id).all()
     users = prepare_guild_members_list(all_members, all_bans)
     dbguild_dict = {"unauth_users": db_guild.unauth_users}
-    return render_template("administrate_guild.html.j2", guild=guild['content'], dbguild=dbguild_dict, members=users, permissions=permissions)
+    return render_template("administrate_guild.html.j2", guild=db_guild, dbguild=dbguild_dict, members=users, permissions=permissions)
 
 @user.route("/administrate_guild/<guild_id>", methods=["POST"])
 @discord_users_only()
 def update_administrate_guild(guild_id):
     if not check_user_can_administrate_guild(guild_id):
         abort(403)
-    guild = discord_api.get_guild(guild_id)
-    if guild['code'] != 200:
-        abort(guild['code'])
     db_guild = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first()
-    if db_guild is None:
+    if not db_guild:
         abort(400)
     db_guild.unauth_users = request.form.get("unauth_users", db_guild.unauth_users) in ["true", True]
     db.session.commit()

webapp/titanembeds/database/__init__.py

@@ -6,4 +6,6 @@ from guilds import Guilds
 from unauthenticated_users import UnauthenticatedUsers
 from unauthenticated_bans import UnauthenticatedBans
 from authenticated_users import AuthenticatedUsers
+from guild_members import GuildMembers, list_all_guild_members
 from keyvalue_properties import KeyValueProperties, set_keyvalproperty, get_keyvalproperty, getexpir_keyvalproperty, setexpir_keyvalproperty, ifexists_keyvalproperty, delete_keyvalproperty
+from messages import Messages, get_channel_messages

webapp/titanembeds/database/guild_members.py

@@ -0,0 +1,45 @@
+from titanembeds.database import db
+import json
+
+class GuildMembers(db.Model):
+    __tablename__ = "guild_members"
+    id = db.Column(db.Integer, primary_key=True)    # Auto incremented id
+    guild_id = db.Column(db.String(255))            # Discord guild id
+    user_id = db.Column(db.String(255))             # Discord user id
+    username = db.Column(db.String(255))            # Name
+    discriminator = db.Column(db.Integer)           # User discriminator
+    nickname = db.Column(db.String(255))            # User nickname
+    avatar = db.Column(db.String(255))              # The avatar str of the user
+    active = db.Column(db.Boolean())                # If the user is a member of the guild
+    banned = db.Column(db.Boolean())                # If the user is banned in the guild
+    roles = db.Column(db.Text())                    # Member roles
+
+    def __init__(self, guild_id, user_id, username, discriminator, nickname, avatar, active, banned, roles):
+        self.guild_id = guild_id
+        self.user_id = user_id
+        self.username = username
+        self.discriminator = discriminator
+        self.nickname = nickname
+        self.avatar = avatar
+        self.active = active
+        self.banned = banned
+        self.roles = roles
+
+    def __repr__(self):
+        return '<GuildMembers {0} {1} {2} {3} {4}>'.format(self.id, self.guild_id, self.user_id, self.username, self.discriminator)
+
+def list_all_guild_members(guild_id):
+    memlist = []
+    members = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id).all()
+    for member in members:
+        memlist.append({
+            "user": {
+                "username": member.username,
+                "discriminator": member.discriminator,
+                "id": member.user_id,
+                "avatar": member.avatar
+            },
+            "roles": json.loads(member.roles),
+            "nickname": member.nickname,
+        })
+    return memlist

webapp/titanembeds/database/guilds.py

@@ -4,11 +4,21 @@ class Guilds(db.Model):
     __tablename__ = "guilds"
     id = db.Column(db.Integer, primary_key=True)    # Auto incremented id
     guild_id = db.Column(db.String(255))            # Discord guild id
+    name = db.Column(db.String(255))                # Name
     unauth_users = db.Column(db.Boolean())          # If allowed unauth users
+    roles = db.Column(db.Text())                    # Guild Roles
+    channels = db.Column(db.Text())                 # Guild channels
+    owner_id = db.Column(db.String(255))            # Snowflake of the owner
+    icon = db.Column(db.String(255))                # The icon string, null if none
 
-    def __init__(self, guild_id):
+    def __init__(self, guild_id, name, roles, channels, owner_id, icon):
         self.guild_id = guild_id
+        self.name = name
         self.unauth_users = True # defaults to true
+        self.roles = roles
+        self.channels = channels
+        self.owner_id = owner_id
+        self.icon = icon
 
     def __repr__(self):
         return '<Guilds {0} {1}>'.format(self.id, self.guild_id)

webapp/titanembeds/database/messages.py

@@ -0,0 +1,49 @@
+from titanembeds.database import db
+from sqlalchemy import cast
+import json
+
+class Messages(db.Model):
+    __tablename__ = "messages"
+    id = db.Column(db.Integer, primary_key=True)    # Auto incremented id
+    guild_id = db.Column(db.String(255))            # Discord guild id
+    channel_id = db.Column(db.String(255))          # Channel id
+    message_id = db.Column(db.String(255))          # Message snowflake
+    content = db.Column(db.Text())                  # Message contents
+    author = db.Column(db.Text())                   # Author
+    timestamp = db.Column(db.TIMESTAMP)             # Timestamp of when content is created
+    edited_timestamp = db.Column(db.TIMESTAMP)      # Timestamp of when content is edited
+    mentions = db.Column(db.Text())                 # Mentions serialized
+    attachments = db.Column(db.Text())              # serialized attachments
+
+    def __init__(self, guild_id, channel_id, message_id, content, author, timestamp, edited_timestamp, mentions, attachments):
+        self.guild_id = guild_id
+        self.channel_id = channel_id
+        self.message_id = message_id
+        self.content = content
+        self.author = author
+        self.timestamp = timestamp
+        self.edited_timestamp = edited_timestamp
+        self.mentions = mentions
+        self.attachments = attachments
+
+    def __repr__(self):
+        return '<Messages {0} {1} {2} {3} {4}>'.format(self.id, self.guild_id, self.guild_id, self.channel_id, self.message_id)
+
+def get_channel_messages(channel_id, after_snowflake=None):
+    if not after_snowflake:
+        q = db.session.query(Messages).filter(Messages.channel_id == channel_id).order_by(Messages.id.desc()).limit(50)
+    else:
+        q = db.session.query(Messages).filter(cast(Messages.channel_id, db.Integer) == int(channel_id)).filter(Messages.message_id > after_snowflake).order_by(Messages.id.desc()).limit(50)
+    msgs = []
+    for x in q:
+        msgs.append({
+            "attachments": json.loads(x.attachments),
+            "timestamp": x.timestamp,
+            "id": x.message_id,
+            "edited_timestamp": x.edited_timestamp,
+            "author": json.loads(x.author),
+            "content": x.content,
+            "channel_id": x.channel_id,
+            "mentions": json.loads(x.mentions)
+        })
+    return msgs

webapp/titanembeds/templates/embed.html.j2

@@ -10,7 +10,7 @@
     <!--Let browser know website is optimized for mobile-->
     <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
 
-    <title>{{ guild['name'] }} - Embed - Titan Embeds for Discord</title>
+    <title>{{ guild.name }} - Embed - Titan Embeds for Discord</title>
     {% include 'google_analytics.html.j2' %}
   </head>
   <body>
@@ -32,16 +32,16 @@
       <ul id="guild-nav" class="side-nav">
         <li>
           <div class="userView">
-            {% if guild['icon'] %}
+            {% if guild.icon %}
-            <img class="circle" src="{{ generate_guild_icon( guild['id'], guild['icon']  ) }}">
+            <img class="circle" src="{{ generate_guild_icon( guild.id, guild.icon  ) }}">
             {% endif %}
-            <span class="name">{{ guild['name'] }}</span>
+            <span class="name">{{ guild.name }}</span>
           </div>
         </li>
 
         <li><a class="subheader">Actions</a></li>
-        <li><a href="{{ url_for("user.administrate_guild", guild_id=guild['id']) }}" class="waves-effect" target="_blank" id="administrate_link" style="display: none;">Manage Guild Embed</a></li>
+        <li><a href="{{ url_for("user.administrate_guild", guild_id=guild.id) }}" class="waves-effect" target="_blank" id="administrate_link" style="display: none;">Manage Guild Embed</a></li>
-        <li><a href="https://discordapp.com/channels/{{ guild['id'] }}/" class="waves-effect" target="_blank">Open Server on Discordapp</a></li>
+        <li><a href="https://discordapp.com/channels/{{ guild.id }}/" class="waves-effect" target="_blank">Open Server on Discordapp</a></li>
 
         <li><div class="divider"></div></li>
 

webapp/titanembeds/utils.py

@@ -67,8 +67,8 @@ def check_guild_existance(guild_id):
     dbGuild = Guilds.query.filter_by(guild_id=guild_id).first()
     if not dbGuild:
         return False
-    guild = discord_api.get_guild(guild_id)
+    else:
-    return guild['code'] == 200
+        return True
 
 def guild_query_unauth_users_bool(guild_id):
     dbGuild = db.session.query(Guilds).filter(Guilds.guild_id==guild_id).first()