From a6766b20080c4366992a0972747dcf2fd7a995d4 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Tue, 2 May 2017 18:37:24 +0000 Subject: [PATCH 01/13] Initial structure for discordbot addition --- README.md | 9 ++------- discordbot/README.md | 6 ++++++ webapp/README.md | 8 ++++++++ config.example.py => webapp/config.example.py | 2 +- requirements.txt => webapp/requirements.txt | 0 run.py => webapp/run.py | 0 run_c9.py => webapp/run_c9.py | 0 {titanembeds => webapp/titanembeds}/__init__.py | 0 {titanembeds => webapp/titanembeds}/app.py | 0 .../titanembeds}/blueprints/__init__.py | 0 .../titanembeds}/blueprints/api/__init__.py | 0 .../titanembeds}/blueprints/api/api.py | 0 .../titanembeds}/blueprints/embed/__init__.py | 0 .../titanembeds}/blueprints/embed/embed.py | 0 .../titanembeds}/blueprints/user/__init__.py | 0 .../titanembeds}/blueprints/user/user.py | 0 .../titanembeds}/database/__init__.py | 0 .../titanembeds}/database/authenticated_users.py | 0 .../titanembeds}/database/custom_redislite.py | 0 .../titanembeds}/database/guilds.py | 0 .../titanembeds}/database/keyvalue_properties.py | 0 .../titanembeds}/database/unauthenticated_bans.py | 0 .../titanembeds}/database/unauthenticated_users.py | 0 {titanembeds => webapp/titanembeds}/decorators.py | 0 {titanembeds => webapp/titanembeds}/discordrest.py | 0 {titanembeds => webapp/titanembeds}/oauth.py | 0 .../titanembeds}/static/css/administrate_guild.css | 0 .../titanembeds}/static/css/embedstyle.css | 0 .../titanembeds}/static/css/style.css | 0 .../titanembeds}/static/js/administrate_guild.js | 0 .../titanembeds}/static/js/embed.js | 0 .../titanembeds}/static/titanembeds.mp4 | Bin .../titanembeds}/static/titanembeds.ogg | Bin .../titanembeds}/static/titanembeds.webm | Bin .../templates/administrate_guild.html.j2 | 0 .../titanembeds}/templates/dashboard.html.j2 | 0 .../titanembeds}/templates/embed.html.j2 | 0 .../titanembeds}/templates/google_analytics.html.j2 | 0 .../titanembeds}/templates/index.html.j2 | 0 .../titanembeds}/templates/signin_complete.html.j2 | 0 .../titanembeds}/templates/site_layout.html.j2 | 0 {titanembeds => webapp/titanembeds}/utils.py | 0 {tmp => webapp/tmp}/.gitinclude | 0 43 files changed, 17 insertions(+), 8 deletions(-) create mode 100644 discordbot/README.md create mode 100644 webapp/README.md rename config.example.py => webapp/config.example.py (89%) rename requirements.txt => webapp/requirements.txt (100%) rename run.py => webapp/run.py (100%) rename run_c9.py => webapp/run_c9.py (100%) rename {titanembeds => webapp/titanembeds}/__init__.py (100%) rename {titanembeds => webapp/titanembeds}/app.py (100%) rename {titanembeds => webapp/titanembeds}/blueprints/__init__.py (100%) rename {titanembeds => webapp/titanembeds}/blueprints/api/__init__.py (100%) rename {titanembeds => webapp/titanembeds}/blueprints/api/api.py (100%) rename {titanembeds => webapp/titanembeds}/blueprints/embed/__init__.py (100%) rename {titanembeds => webapp/titanembeds}/blueprints/embed/embed.py (100%) rename {titanembeds => webapp/titanembeds}/blueprints/user/__init__.py (100%) rename {titanembeds => webapp/titanembeds}/blueprints/user/user.py (100%) rename {titanembeds => webapp/titanembeds}/database/__init__.py (100%) rename {titanembeds => webapp/titanembeds}/database/authenticated_users.py (100%) rename {titanembeds => webapp/titanembeds}/database/custom_redislite.py (100%) rename {titanembeds => webapp/titanembeds}/database/guilds.py (100%) rename {titanembeds => webapp/titanembeds}/database/keyvalue_properties.py (100%) rename {titanembeds => webapp/titanembeds}/database/unauthenticated_bans.py (100%) rename {titanembeds => webapp/titanembeds}/database/unauthenticated_users.py (100%) rename {titanembeds => webapp/titanembeds}/decorators.py (100%) rename {titanembeds => webapp/titanembeds}/discordrest.py (100%) rename {titanembeds => webapp/titanembeds}/oauth.py (100%) rename {titanembeds => webapp/titanembeds}/static/css/administrate_guild.css (100%) rename {titanembeds => webapp/titanembeds}/static/css/embedstyle.css (100%) rename {titanembeds => webapp/titanembeds}/static/css/style.css (100%) rename {titanembeds => webapp/titanembeds}/static/js/administrate_guild.js (100%) rename {titanembeds => webapp/titanembeds}/static/js/embed.js (100%) rename {titanembeds => webapp/titanembeds}/static/titanembeds.mp4 (100%) rename {titanembeds => webapp/titanembeds}/static/titanembeds.ogg (100%) rename {titanembeds => webapp/titanembeds}/static/titanembeds.webm (100%) rename {titanembeds => webapp/titanembeds}/templates/administrate_guild.html.j2 (100%) rename {titanembeds => webapp/titanembeds}/templates/dashboard.html.j2 (100%) rename {titanembeds => webapp/titanembeds}/templates/embed.html.j2 (100%) rename {titanembeds => webapp/titanembeds}/templates/google_analytics.html.j2 (100%) rename {titanembeds => webapp/titanembeds}/templates/index.html.j2 (100%) rename {titanembeds => webapp/titanembeds}/templates/signin_complete.html.j2 (100%) rename {titanembeds => webapp/titanembeds}/templates/site_layout.html.j2 (100%) rename {titanembeds => webapp/titanembeds}/utils.py (100%) rename {tmp => webapp/tmp}/.gitinclude (100%) diff --git a/README.md b/README.md index 5661420..2abbc78 100644 --- a/README.md +++ b/README.md @@ -9,15 +9,10 @@ There was a time when Discord doesn't support embedding the chat on a webpage. B - Moderation Features (Kick & ban users by IP addresses, toggling guest users) - Discord OAuth support. (Allows those who have a discord account to access the embed) - Responsive material design! (Thanks materializecss!!) -- All features are done via REST apis (respects discord's rate limiting). Although do not provide consistant connection to Discord, they are easier to maintain and does not often "disconnects" from Discord servers. # Installation -Would you like to run your own copy of Titan Embeds? -1. Clone the repo (make sure you have python 2.7 installed on your system. This project depends on that specific python version) -2. Install the pip requirements `pip install -r requirements.txt` -3. Clone `config.example.py` and rename it to `config.py`. Edit the file to your standards -4. Make sure that the bot is online in the websockets once. This is required because the bot cannot send messages until it has used the ws. Use something like discord.py to log the bot into discord websockets. You can close it afterwards. So basically if the bot account is online ONCE in it's lifespan- you're good. -5. Run the development web via `python run.py` -- Though we suggest to use a better server software (look into gunicorn, nginx, uwsgi, etc) +Would you like to run your own copy of Titan Embeds? There are two parts that integrate nicely together. The webapp (website) handles the frontend and communication with the database to retrieve server messages, etc. The discordbot (bot) handles the communcation +between Discord's websockets and pushing out the data to the database for the webapp. Check out the respective folder for their installation (pay attention to the python versions!) instructions. ## Join us! diff --git a/discordbot/README.md b/discordbot/README.md new file mode 100644 index 0000000..0472b08 --- /dev/null +++ b/discordbot/README.md @@ -0,0 +1,6 @@ +# Titan - DiscordBot Portion +The DiscordBot portion handles the communcation with Discord's websockets to provide real-time updates. The bot's primary role is to push content to the webapp's database to be retrieved at a later time. +It also includes misc. features to moderate guest users, etc. + +# Installation +1. TODO, Sorry about that! (Though if you want to get ahead, install the latest discord.py pip package and that will require **Python 3.5**) \ No newline at end of file diff --git a/webapp/README.md b/webapp/README.md new file mode 100644 index 0000000..8fb833b --- /dev/null +++ b/webapp/README.md @@ -0,0 +1,8 @@ +# Titan - WebApp Portion +The webapp portion handles the frontend (it's what the users see). The webapp highly depends on the discordbot to push websockets data to the database. + +# Installation +1. Clone the repo (make sure you have **Python 2.7** installed on your system. This webapp portion depends on that specific python version) +2. Install the pip requirements `pip install -r requirements.txt` +3. Clone `config.example.py` and rename it to `config.py`. Edit the file to your standards +4. Run the development web via `python run.py` -- Though we suggest to use a better server software (look into gunicorn, nginx, uwsgi, etc) \ No newline at end of file diff --git a/config.example.py b/webapp/config.example.py similarity index 89% rename from config.example.py rename to webapp/config.example.py index 0e427b2..786e614 100644 --- a/config.example.py +++ b/webapp/config.example.py @@ -5,7 +5,7 @@ config = { 'client-secret': "Your discord client secret", 'bot-token': "Discord bot token", - 'app-location': "/var/www/Titan/", + 'app-location': "/var/www/Titan/webapp/", 'app-secret': "Type something random here, go wild.", 'database-uri': "driver://username:password@host:port/database", diff --git a/requirements.txt b/webapp/requirements.txt similarity index 100% rename from requirements.txt rename to webapp/requirements.txt diff --git a/run.py b/webapp/run.py similarity index 100% rename from run.py rename to webapp/run.py diff --git a/run_c9.py b/webapp/run_c9.py similarity index 100% rename from run_c9.py rename to webapp/run_c9.py diff --git a/titanembeds/__init__.py b/webapp/titanembeds/__init__.py similarity index 100% rename from titanembeds/__init__.py rename to webapp/titanembeds/__init__.py diff --git a/titanembeds/app.py b/webapp/titanembeds/app.py similarity index 100% rename from titanembeds/app.py rename to webapp/titanembeds/app.py diff --git a/titanembeds/blueprints/__init__.py b/webapp/titanembeds/blueprints/__init__.py similarity index 100% rename from titanembeds/blueprints/__init__.py rename to webapp/titanembeds/blueprints/__init__.py diff --git a/titanembeds/blueprints/api/__init__.py b/webapp/titanembeds/blueprints/api/__init__.py similarity index 100% rename from titanembeds/blueprints/api/__init__.py rename to webapp/titanembeds/blueprints/api/__init__.py diff --git a/titanembeds/blueprints/api/api.py b/webapp/titanembeds/blueprints/api/api.py similarity index 100% rename from titanembeds/blueprints/api/api.py rename to webapp/titanembeds/blueprints/api/api.py diff --git a/titanembeds/blueprints/embed/__init__.py b/webapp/titanembeds/blueprints/embed/__init__.py similarity index 100% rename from titanembeds/blueprints/embed/__init__.py rename to webapp/titanembeds/blueprints/embed/__init__.py diff --git a/titanembeds/blueprints/embed/embed.py b/webapp/titanembeds/blueprints/embed/embed.py similarity index 100% rename from titanembeds/blueprints/embed/embed.py rename to webapp/titanembeds/blueprints/embed/embed.py diff --git a/titanembeds/blueprints/user/__init__.py b/webapp/titanembeds/blueprints/user/__init__.py similarity index 100% rename from titanembeds/blueprints/user/__init__.py rename to webapp/titanembeds/blueprints/user/__init__.py diff --git a/titanembeds/blueprints/user/user.py b/webapp/titanembeds/blueprints/user/user.py similarity index 100% rename from titanembeds/blueprints/user/user.py rename to webapp/titanembeds/blueprints/user/user.py diff --git a/titanembeds/database/__init__.py b/webapp/titanembeds/database/__init__.py similarity index 100% rename from titanembeds/database/__init__.py rename to webapp/titanembeds/database/__init__.py diff --git a/titanembeds/database/authenticated_users.py b/webapp/titanembeds/database/authenticated_users.py similarity index 100% rename from titanembeds/database/authenticated_users.py rename to webapp/titanembeds/database/authenticated_users.py diff --git a/titanembeds/database/custom_redislite.py b/webapp/titanembeds/database/custom_redislite.py similarity index 100% rename from titanembeds/database/custom_redislite.py rename to webapp/titanembeds/database/custom_redislite.py diff --git a/titanembeds/database/guilds.py b/webapp/titanembeds/database/guilds.py similarity index 100% rename from titanembeds/database/guilds.py rename to webapp/titanembeds/database/guilds.py diff --git a/titanembeds/database/keyvalue_properties.py b/webapp/titanembeds/database/keyvalue_properties.py similarity index 100% rename from titanembeds/database/keyvalue_properties.py rename to webapp/titanembeds/database/keyvalue_properties.py diff --git a/titanembeds/database/unauthenticated_bans.py b/webapp/titanembeds/database/unauthenticated_bans.py similarity index 100% rename from titanembeds/database/unauthenticated_bans.py rename to webapp/titanembeds/database/unauthenticated_bans.py diff --git a/titanembeds/database/unauthenticated_users.py b/webapp/titanembeds/database/unauthenticated_users.py similarity index 100% rename from titanembeds/database/unauthenticated_users.py rename to webapp/titanembeds/database/unauthenticated_users.py diff --git a/titanembeds/decorators.py b/webapp/titanembeds/decorators.py similarity index 100% rename from titanembeds/decorators.py rename to webapp/titanembeds/decorators.py diff --git a/titanembeds/discordrest.py b/webapp/titanembeds/discordrest.py similarity index 100% rename from titanembeds/discordrest.py rename to webapp/titanembeds/discordrest.py diff --git a/titanembeds/oauth.py b/webapp/titanembeds/oauth.py similarity index 100% rename from titanembeds/oauth.py rename to webapp/titanembeds/oauth.py diff --git a/titanembeds/static/css/administrate_guild.css b/webapp/titanembeds/static/css/administrate_guild.css similarity index 100% rename from titanembeds/static/css/administrate_guild.css rename to webapp/titanembeds/static/css/administrate_guild.css diff --git a/titanembeds/static/css/embedstyle.css b/webapp/titanembeds/static/css/embedstyle.css similarity index 100% rename from titanembeds/static/css/embedstyle.css rename to webapp/titanembeds/static/css/embedstyle.css diff --git a/titanembeds/static/css/style.css b/webapp/titanembeds/static/css/style.css similarity index 100% rename from titanembeds/static/css/style.css rename to webapp/titanembeds/static/css/style.css diff --git a/titanembeds/static/js/administrate_guild.js b/webapp/titanembeds/static/js/administrate_guild.js similarity index 100% rename from titanembeds/static/js/administrate_guild.js rename to webapp/titanembeds/static/js/administrate_guild.js diff --git a/titanembeds/static/js/embed.js b/webapp/titanembeds/static/js/embed.js similarity index 100% rename from titanembeds/static/js/embed.js rename to webapp/titanembeds/static/js/embed.js diff --git a/titanembeds/static/titanembeds.mp4 b/webapp/titanembeds/static/titanembeds.mp4 similarity index 100% rename from titanembeds/static/titanembeds.mp4 rename to webapp/titanembeds/static/titanembeds.mp4 diff --git a/titanembeds/static/titanembeds.ogg b/webapp/titanembeds/static/titanembeds.ogg similarity index 100% rename from titanembeds/static/titanembeds.ogg rename to webapp/titanembeds/static/titanembeds.ogg diff --git a/titanembeds/static/titanembeds.webm b/webapp/titanembeds/static/titanembeds.webm similarity index 100% rename from titanembeds/static/titanembeds.webm rename to webapp/titanembeds/static/titanembeds.webm diff --git a/titanembeds/templates/administrate_guild.html.j2 b/webapp/titanembeds/templates/administrate_guild.html.j2 similarity index 100% rename from titanembeds/templates/administrate_guild.html.j2 rename to webapp/titanembeds/templates/administrate_guild.html.j2 diff --git a/titanembeds/templates/dashboard.html.j2 b/webapp/titanembeds/templates/dashboard.html.j2 similarity index 100% rename from titanembeds/templates/dashboard.html.j2 rename to webapp/titanembeds/templates/dashboard.html.j2 diff --git a/titanembeds/templates/embed.html.j2 b/webapp/titanembeds/templates/embed.html.j2 similarity index 100% rename from titanembeds/templates/embed.html.j2 rename to webapp/titanembeds/templates/embed.html.j2 diff --git a/titanembeds/templates/google_analytics.html.j2 b/webapp/titanembeds/templates/google_analytics.html.j2 similarity index 100% rename from titanembeds/templates/google_analytics.html.j2 rename to webapp/titanembeds/templates/google_analytics.html.j2 diff --git a/titanembeds/templates/index.html.j2 b/webapp/titanembeds/templates/index.html.j2 similarity index 100% rename from titanembeds/templates/index.html.j2 rename to webapp/titanembeds/templates/index.html.j2 diff --git a/titanembeds/templates/signin_complete.html.j2 b/webapp/titanembeds/templates/signin_complete.html.j2 similarity index 100% rename from titanembeds/templates/signin_complete.html.j2 rename to webapp/titanembeds/templates/signin_complete.html.j2 diff --git a/titanembeds/templates/site_layout.html.j2 b/webapp/titanembeds/templates/site_layout.html.j2 similarity index 100% rename from titanembeds/templates/site_layout.html.j2 rename to webapp/titanembeds/templates/site_layout.html.j2 diff --git a/titanembeds/utils.py b/webapp/titanembeds/utils.py similarity index 100% rename from titanembeds/utils.py rename to webapp/titanembeds/utils.py diff --git a/tmp/.gitinclude b/webapp/tmp/.gitinclude similarity index 100% rename from tmp/.gitinclude rename to webapp/tmp/.gitinclude From cef4012738010ce4a5b83236b9e553fec12daafd Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Tue, 2 May 2017 18:53:18 +0000 Subject: [PATCH 02/13] Inital Database configuration --- discordbot/config.example.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 discordbot/config.example.py diff --git a/discordbot/config.example.py b/discordbot/config.example.py new file mode 100644 index 0000000..7fda5e8 --- /dev/null +++ b/discordbot/config.example.py @@ -0,0 +1,5 @@ +config = { + 'bot-token': "Discord bot token", + + 'database-uri': "driver://username:password@host:port/database", +} \ No newline at end of file From 1e1a3ab43ee42283fbf477351ee87b310d5fd24f Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Thu, 4 May 2017 03:22:27 +0000 Subject: [PATCH 03/13] Partial discordbot work --- discordbot/requirements.txt | 2 ++ discordbot/run.py | 4 ++++ discordbot/titanembeds/__init__.py | 0 discordbot/titanembeds/bot.py | 17 +++++++++++++++++ discordbot/titanembeds/database/__init__.py | 17 +++++++++++++++++ discordbot/titanembeds/database/guilds.py | 14 ++++++++++++++ 6 files changed, 54 insertions(+) create mode 100644 discordbot/requirements.txt create mode 100644 discordbot/run.py create mode 100644 discordbot/titanembeds/__init__.py create mode 100644 discordbot/titanembeds/bot.py create mode 100644 discordbot/titanembeds/database/__init__.py create mode 100644 discordbot/titanembeds/database/guilds.py diff --git a/discordbot/requirements.txt b/discordbot/requirements.txt new file mode 100644 index 0000000..ee91f00 --- /dev/null +++ b/discordbot/requirements.txt @@ -0,0 +1,2 @@ +discord.py +sqlalchemy-aio \ No newline at end of file diff --git a/discordbot/run.py b/discordbot/run.py new file mode 100644 index 0000000..1ec70b8 --- /dev/null +++ b/discordbot/run.py @@ -0,0 +1,4 @@ +from titanembeds.bot import client +from config import config + +client.run(config["bot-token"]) diff --git a/discordbot/titanembeds/__init__.py b/discordbot/titanembeds/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/discordbot/titanembeds/bot.py b/discordbot/titanembeds/bot.py new file mode 100644 index 0000000..55457f9 --- /dev/null +++ b/discordbot/titanembeds/bot.py @@ -0,0 +1,17 @@ +from config import config +import discord + +client = discord.Client() + +@client.event +async def on_ready(): + print('Titan -- DiscordBot') + print('Logged in as the following user:') + print(client.user.name) + print(client.user.id) + print('------') + await test() + +async def test(): + from titanembeds.database import db, Guilds, session + session.query(Guilds).all() \ No newline at end of file diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py new file mode 100644 index 0000000..0c8a8db --- /dev/null +++ b/discordbot/titanembeds/database/__init__.py @@ -0,0 +1,17 @@ +from config import config +from sqlalchemy_aio import ASYNCIO_STRATEGY +import sqlalchemy as db +from sqlalchemy.ext.declarative import declarative_base + +Base = declarative_base() + +from .guilds import Guilds + +engine = db.create_engine(config["database-uri"]) + +Base.metadata.create_all(engine) + +from sqlalchemy.orm import sessionmaker +DBSession = sessionmaker() +DBSession.bind = engine +session = DBSession() \ No newline at end of file diff --git a/discordbot/titanembeds/database/guilds.py b/discordbot/titanembeds/database/guilds.py new file mode 100644 index 0000000..d80e9f2 --- /dev/null +++ b/discordbot/titanembeds/database/guilds.py @@ -0,0 +1,14 @@ +from titanembeds.database import db, Base + +class Guilds(Base): + __tablename__ = "guilds" + id = db.Column(db.Integer, primary_key=True) # Auto incremented id + guild_id = db.Column(db.String(255)) # Discord guild id + unauth_users = db.Column(db.Boolean()) # If allowed unauth users + + def __init__(self, guild_id): + self.guild_id = guild_id + self.unauth_users = True # defaults to true + + def __repr__(self): + return ''.format(self.id, self.guild_id) \ No newline at end of file From 7762862623c3c0f9528345a56fec8481847b71ef Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Wed, 3 May 2017 22:16:49 -0700 Subject: [PATCH 04/13] Discordbot database setup inital/partial --- discordbot/requirements.txt | 3 +- discordbot/run.py | 13 ++++- discordbot/titanembeds/__init__.py | 1 + discordbot/titanembeds/bot.py | 64 +++++++++++++++++---- discordbot/titanembeds/database/__init__.py | 36 +++++++++--- 5 files changed, 92 insertions(+), 25 deletions(-) diff --git a/discordbot/requirements.txt b/discordbot/requirements.txt index ee91f00..400bc22 100644 --- a/discordbot/requirements.txt +++ b/discordbot/requirements.txt @@ -1,2 +1,3 @@ discord.py -sqlalchemy-aio \ No newline at end of file +sqlalchemy +asyncio_extras diff --git a/discordbot/run.py b/discordbot/run.py index 1ec70b8..c6b058c 100644 --- a/discordbot/run.py +++ b/discordbot/run.py @@ -1,4 +1,11 @@ -from titanembeds.bot import client -from config import config +from titanembeds import Titan +import gc -client.run(config["bot-token"]) +def main(): + print("Starting...") + te = Titan() + te.run() + gc.collect() + +if __name__ == '__main__': + main() diff --git a/discordbot/titanembeds/__init__.py b/discordbot/titanembeds/__init__.py index e69de29..9960b6e 100644 --- a/discordbot/titanembeds/__init__.py +++ b/discordbot/titanembeds/__init__.py @@ -0,0 +1 @@ +from titanembeds.bot import Titan diff --git a/discordbot/titanembeds/bot.py b/discordbot/titanembeds/bot.py index 55457f9..b538bed 100644 --- a/discordbot/titanembeds/bot.py +++ b/discordbot/titanembeds/bot.py @@ -1,17 +1,57 @@ from config import config +from titanembeds.database import DatabaseInterface import discord +import aiohttp +import asyncio -client = discord.Client() +class Titan(discord.Client): + def __init__(self): + super().__init__() + self.aiosession = aiohttp.ClientSession(loop=self.loop) + self.http.user_agent += ' TitanEmbeds-Bot' + self.database = DatabaseInterface(self) -@client.event -async def on_ready(): - print('Titan -- DiscordBot') - print('Logged in as the following user:') - print(client.user.name) - print(client.user.id) - print('------') - await test() + def _cleanup(self): + try: + self.loop.run_until_complete(self.logout()) + except: # Can be ignored + pass + pending = asyncio.Task.all_tasks() + gathered = asyncio.gather(*pending) + try: + gathered.cancel() + self.loop.run_until_complete(gathered) + gathered.exception() + except: # Can be ignored + pass -async def test(): - from titanembeds.database import db, Guilds, session - session.query(Guilds).all() \ No newline at end of file + def run(self): + try: + self.loop.run_until_complete(self.start(config["bot-token"])) + except discord.errors.LoginFailure: + print("Invalid bot token in config!") + finally: + try: + self._cleanup() + except Exception as e: + print("Error in cleanup:", e) + self.loop.close() + + async def on_ready(self): + print('Titan [DiscordBot]') + print('Logged in as the following user:') + print(self.user.name) + print(self.user.id) + print('------') + + await self.change_presence( + game=discord.Game(name="Get your own @ https://TitanEmbeds.tk/"), status=discord.Status.online + ) + + try: + await self.database.connect(config["database-uri"]) + except Exception: + self.logger.error("Unable to connect to specified database!") + traceback.print_exc() + await self.logout() + return diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index 0c8a8db..56431c6 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -1,17 +1,35 @@ -from config import config -from sqlalchemy_aio import ASYNCIO_STRATEGY +from contextlib import contextmanager +from asyncio_extras import threadpool import sqlalchemy as db +from sqlalchemy.engine import Engine, create_engine +from sqlalchemy.orm import sessionmaker, Session from sqlalchemy.ext.declarative import declarative_base Base = declarative_base() -from .guilds import Guilds +from titanembeds.database.guilds import Guilds -engine = db.create_engine(config["database-uri"]) +class DatabaseInterface(object): + # Courtesy of https://github.com/SunDwarf/Jokusoramame + def __init__(self, bot): + self.bot = bot -Base.metadata.create_all(engine) + self.engine = None # type: Engine + self._sessionmaker = None # type: sessionmaker -from sqlalchemy.orm import sessionmaker -DBSession = sessionmaker() -DBSession.bind = engine -session = DBSession() \ No newline at end of file + async def connect(self, dburi): + async with threadpool(): + self.engine = create_engine(dburi) + self._sessionmaker = sessionmaker(bind=self.engine, expire_on_commit=False) + + @contextmanager + def get_session(self) -> Session: + session = self._sessionmaker() # type: Session + try: + yield session + session.commit() + except: + session.rollback() + raise + finally: + session.close() From ba7b47f193df988ff43799a01307da18a6ea3157 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sat, 6 May 2017 01:03:52 -0700 Subject: [PATCH 05/13] Messages database --- discordbot/titanembeds/bot.py | 7 ++++ discordbot/titanembeds/database/__init__.py | 40 +++++++++++++++++++++ discordbot/titanembeds/database/messages.py | 26 ++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 discordbot/titanembeds/database/messages.py diff --git a/discordbot/titanembeds/bot.py b/discordbot/titanembeds/bot.py index b538bed..1c33940 100644 --- a/discordbot/titanembeds/bot.py +++ b/discordbot/titanembeds/bot.py @@ -55,3 +55,10 @@ class Titan(discord.Client): traceback.print_exc() await self.logout() return + + async def on_message(self, message): + await self.database.push_message(message) + # TODO: Will add command handler + ban/kick command + + async def on_message_edit(self, message_before, message_after): + await self.database.update_message(message_after) diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index 56431c6..3b03a80 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -5,9 +5,12 @@ from sqlalchemy.engine import Engine, create_engine from sqlalchemy.orm import sessionmaker, Session from sqlalchemy.ext.declarative import declarative_base +import json + Base = declarative_base() from titanembeds.database.guilds import Guilds +from titanembeds.database.messages import Messages class DatabaseInterface(object): # Courtesy of https://github.com/SunDwarf/Jokusoramame @@ -33,3 +36,40 @@ class DatabaseInterface(object): raise finally: session.close() + + async def push_message(self, message): + if message.server: + async with threadpool(): + with self.get_session() as session: + edit_ts = message.edited_timestamp + if not edit_ts: + edit_ts = None + else: + edit_ts = str(edit_ts) + + msg = Messages( + message.server.id, + message.channel.id, + message.id, + message.content, + str(message.timestamp), + edit_ts, + json.dumps(message.mentions), + json.dumps(message.attachments) + ) + session.add(msg) + session.commit() + + async def update_message(self, message): + if message.server: + async with threadpool(): + with self.get_session() as session: + msg = session.query(Messages) \ + .filter(Messages.guild_id == message.server.id) \ + .filter(Messages.channel_id == message.channel.id) \ + .filter(Messages.message_id == message.id).first() + msg.content = message.content + msg.edited_timestamp = message.edited_timestamp + msg.mentions = json.dumps(message.mentions) + msg.attachments = json.dumps(message.attachments) + session.commit() diff --git a/discordbot/titanembeds/database/messages.py b/discordbot/titanembeds/database/messages.py new file mode 100644 index 0000000..d4bad0c --- /dev/null +++ b/discordbot/titanembeds/database/messages.py @@ -0,0 +1,26 @@ +from titanembeds.database import db, Base + +class Messages(Base): + __tablename__ = "messages" + id = db.Column(db.Integer, primary_key=True) # Auto incremented id + guild_id = db.Column(db.String(255)) # Discord guild id + channel_id = db.Column(db.String(255)) # Channel id + message_id = db.Column(db.String(255)) # Message snowflake + content = db.Column(db.Text()) # Message contents + timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is created + edited_timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is edited + mentions = db.Column(db.Text()) # Mentions serialized + attachments = db.Column(db.Text()) # serialized attachments + + def __init__(self, guild_id, channel_id, message_id, content, timestamp, edited_timestamp, mentions, attachments): + self.guild_id = guild_id + self.channel_id = channel_id + self.message_id = message_id + self.content = content + self.timestamp = timestamp + self.edited_timestamp = edited_timestamp + self.mentions = mentions + self.attachments = attachments + + def __repr__(self): + return ''.format(self.id, self.guild_id, self.guild_id, self.channel_id, self.message_id) From b43d29bca58751629a1c99365a7fb171b4202be2 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sat, 6 May 2017 01:04:34 -0700 Subject: [PATCH 06/13] Prevent update msg when no message exists --- discordbot/titanembeds/database/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index 3b03a80..b68cfa1 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -68,8 +68,9 @@ class DatabaseInterface(object): .filter(Messages.guild_id == message.server.id) \ .filter(Messages.channel_id == message.channel.id) \ .filter(Messages.message_id == message.id).first() - msg.content = message.content - msg.edited_timestamp = message.edited_timestamp - msg.mentions = json.dumps(message.mentions) - msg.attachments = json.dumps(message.attachments) - session.commit() + if msg: + msg.content = message.content + msg.edited_timestamp = message.edited_timestamp + msg.mentions = json.dumps(message.mentions) + msg.attachments = json.dumps(message.attachments) + session.commit() From e7e0c6e505b3262a227c58b9f109b2dce96b7a7d Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sat, 6 May 2017 01:09:54 -0700 Subject: [PATCH 07/13] Discord bot db pool recycle set --- discordbot/titanembeds/database/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index b68cfa1..dfaa9e4 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -22,7 +22,7 @@ class DatabaseInterface(object): async def connect(self, dburi): async with threadpool(): - self.engine = create_engine(dburi) + self.engine = create_engine(dburi, pool_recycle=250) self._sessionmaker = sessionmaker(bind=self.engine, expire_on_commit=False) @contextmanager From b537aacb1c594e6939420108ba44153bf20b02e8 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sat, 6 May 2017 15:27:07 -0700 Subject: [PATCH 08/13] Basic pushing information to the database of members and guilds --- discordbot/titanembeds/bot.py | 71 +++++++- discordbot/titanembeds/database/__init__.py | 166 +++++++++++++++++- .../titanembeds/database/guild_members.py | 28 +++ discordbot/titanembeds/database/guilds.py | 14 +- 4 files changed, 274 insertions(+), 5 deletions(-) create mode 100644 discordbot/titanembeds/database/guild_members.py diff --git a/discordbot/titanembeds/bot.py b/discordbot/titanembeds/bot.py index 1c33940..ccf8bef 100644 --- a/discordbot/titanembeds/bot.py +++ b/discordbot/titanembeds/bot.py @@ -45,20 +45,87 @@ class Titan(discord.Client): print('------') await self.change_presence( - game=discord.Game(name="Get your own @ https://TitanEmbeds.tk/"), status=discord.Status.online + game=discord.Game(name="iFrame your server! Visit https://TitanEmbeds.tk/ today!"), status=discord.Status.online ) try: - await self.database.connect(config["database-uri"]) + await self.database.connect(config["database-uri"] + "?charset=utf8") except Exception: self.logger.error("Unable to connect to specified database!") traceback.print_exc() await self.logout() return + for server in self.servers: + await self.database.update_guild(server) + if server.large: + await request_offline_members(server) + server_bans = await self.get_bans(server) + for member in server.members: + banned = member.id in [u.id for u in server_bans] + await self.database.update_guild_member( + member, + True, + banned + ) + await self.database.flag_unactive_guild_members(server.id, server.members) + await self.database.flag_unactive_bans(server.id, server_bans) + await self.database.remove_unused_guilds(self.servers) + async def on_message(self, message): await self.database.push_message(message) # TODO: Will add command handler + ban/kick command async def on_message_edit(self, message_before, message_after): await self.database.update_message(message_after) + + async def on_message_delete(self, message): + await self.database.delete_message(message) + + async def on_server_join(self, guild): + await self.database.update_guild(guild) + + async def on_server_remove(self, guild): + await self.database.remove_guild(guild) + + async def on_server_update(self, guildbefore, guildafter): + await self.database.update_guild(guildafter) + + async def on_server_role_create(self, role): + if role.name == self.user.name and role.managed: + await asyncio.sleep(2) + await self.database.update_guild(role.server) + + async def on_server_role_delete(self, role): + if role.server.me not in role.server.members: + return + await self.database.update_guild(role.server) + + async def on_server_role_update(self, rolebefore, roleafter): + await self.database.update_guild(roleafter.server) + + async def on_channel_delete(self, channel): + await self.database.update_guild(channel.server) + + async def on_channel_create(self, channel): + await self.database.update_guild(channel.server) + + async def on_channel_update(self, channelbefore, channelafter): + await self.database.update_guild(channelafter.server) + + async def on_member_join(self, member): + await self.database.update_guild_member(member, active=True, banned=False) + + async def on_member_remove(self, member): + await self.database.update_guild_member(member, active=False, banned=False) + + async def on_member_update(self, memberbefore, memberafter): + await self.database.update_guild_member(memberafter) + + async def on_member_ban(self, member): + if role.server.me not in role.server.members: + return + await self.database.update_guild_member(member, active=False, banned=True) + + async def on_member_unban(self, member): + await self.database.update_guild_member(member, active=False, banned=False) diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index dfaa9e4..af01c5a 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -6,11 +6,13 @@ from sqlalchemy.orm import sessionmaker, Session from sqlalchemy.ext.declarative import declarative_base import json +import discord Base = declarative_base() from titanembeds.database.guilds import Guilds from titanembeds.database.messages import Messages +from titanembeds.database.guild_members import GuildMembers class DatabaseInterface(object): # Courtesy of https://github.com/SunDwarf/Jokusoramame @@ -22,7 +24,7 @@ class DatabaseInterface(object): async def connect(self, dburi): async with threadpool(): - self.engine = create_engine(dburi, pool_recycle=250) + self.engine = create_engine(dburi, pool_recycle=10) self._sessionmaker = sessionmaker(bind=self.engine, expire_on_commit=False) @contextmanager @@ -74,3 +76,165 @@ class DatabaseInterface(object): msg.mentions = json.dumps(message.mentions) msg.attachments = json.dumps(message.attachments) session.commit() + + async def delete_message(self, message): + if message.server: + async with threadpool(): + with self.get_session() as session: + msg = session.query(Messages) \ + .filter(Messages.guild_id == message.server.id) \ + .filter(Messages.channel_id == message.channel.id) \ + .filter(Messages.message_id == message.id).first() + if msg: + session.delete(msg) + session.commit() + + async def update_guild(self, guild): + async with threadpool(): + with self.get_session() as session: + gui = session.query(Guilds).filter(Guilds.guild_id == guild.id).first() + if not gui: + gui = Guilds( + guild.id, + guild.name, + json.dumps(self.get_roles_list(guild.roles)), + json.dumps(self.get_channels_list(guild.channels)), + guild.owner_id, + guild.icon + ) + session.add(gui) + else: + gui.name = guild.name + gui.roles = json.dumps(self.get_roles_list(guild.roles)) + gui.channels = json.dumps(self.get_channels_list(guild.channels)) + gui.owner_id = guild.owner_id + gui.icon = guild.icon + session.commit() + + def get_roles_list(self, guildroles): + roles = [] + for role in guildroles: + roles.append({ + "id": role.id, + "name": role.name, + "color": role.color.value, + "hoist": role.hoist, + "position": role.position, + }) + return roles + + def get_channels_list(self, guildchannels): + channels = [] + for channel in guildchannels: + if str(channel.type) == "text": + overwrites = [] + for target, overwrite in channel.overwrites: + if isinstance(target, discord.Role): + type = "role" + else: + type = "member" + allow, deny = overwrite.pair() + allow = allow.value + deny = deny.value + overwrites.append({ + "id": target.id, + "type": type, + "allow": allow, + "deny": deny, + }) + + channels.append({ + "id": channel.id, + "name": channel.name, + "topic": channel.topic, + "position": channel.position, + "type": str(channel.type), + "permission_overwrites": overwrites + }) + return channels + + async def remove_unused_guilds(self, guilds): + async with threadpool(): + with self.get_session() as session: + dbguilds = session.query(Guilds).all() + changed = False + for guild in dbguilds: + disguild = discord.utils.get(guilds, id=guild.guild_id) + if not disguild: + changed = True + session.delete(guild) + if changed: + session.commit() + + async def remove_guild(self, guild): + async with threadpool(): + with self.get_session() as session: + gui = session.query(Guilds).filter(Guilds.guild_id == guild.id).first() + if gui: + session.delete(gui) + session.commit() + + async def update_guild_member(self, member, active=True, banned=False): + async with threadpool(): + with self.get_session() as session: + dbmember = session.query(GuildMembers) \ + .filter(GuildMembers.guild_id == member.server.id) \ + .filter(GuildMembers.user_id == member.id).first() + if not dbmember: + dbmember = GuildMembers( + member.server.id, + member.id, + member.name, + member.discriminator, + member.nick, + member.avatar, + active, + banned, + json.dumps(self.list_role_ids(member.roles)) + ) + session.add(dbmember) + else: + dbmember.banned = banned + dbmember.active = active + dbmember.username = member.name + dbmember.discriminator = member.discriminator + dbmember.nick = member.nick + dbmember.avatar = member.avatar + dbmember.roles = json.dumps(self.list_role_ids(member.roles)) + session.commit() + + async def flag_unactive_guild_members(self, guild_id, guild_members): + async with threadpool(): + with self.get_session() as session: + changed = False + dbmembers = session.query(GuildMembers) \ + .filter(GuildMembers.guild_id == guild_id) \ + .filter(GuildMembers.active == True).all() + for member in dbmembers: + dismember = discord.utils.get(guild_members, id=member.user_id) + if not dismember: + changed = True + member.active = False + if changed: + session.commit() + + def list_role_ids(self, usr_roles): + ids = [] + for role in usr_roles: + ids.append(role.id) + return ids + + async def flag_unactive_bans(self, guild_id, guildbans): + async with threadpool(): + with self.get_session() as session: + changed = False + for usr in guildbans: + dbusr = session.query(GuildMembers) \ + .filter(GuildMembers.guild_id == guild_id) \ + .filter(GuildMembers.user_id == usr.id) \ + .filter(GuildMembers.active == False).first() + if dbusr: + changed = True + dbusr.banned = True + if changed: + session.commit() diff --git a/discordbot/titanembeds/database/guild_members.py b/discordbot/titanembeds/database/guild_members.py new file mode 100644 index 0000000..28d5532 --- /dev/null +++ b/discordbot/titanembeds/database/guild_members.py @@ -0,0 +1,28 @@ +from titanembeds.database import db, Base + +class GuildMembers(Base): + __tablename__ = "guild_members" + id = db.Column(db.Integer, primary_key=True) # Auto incremented id + guild_id = db.Column(db.String(255)) # Discord guild id + user_id = db.Column(db.String(255)) # Discord user id + username = db.Column(db.String(255)) # Name + discriminator = db.Column(db.Integer) # User discriminator + nickname = db.Column(db.String(255)) # User nickname + avatar = db.Column(db.String(255)) # The avatar str of the user + active = db.Column(db.Boolean()) # If the user is a member of the guild + banned = db.Column(db.Boolean()) # If the user is banned in the guild + roles = db.Column(db.Text()) # Member roles + + def __init__(self, guild_id, user_id, username, discriminator, nickname, avatar, active, banned, roles): + self.guild_id = guild_id + self.user_id = user_id + self.username = username + self.discriminator = discriminator + self.nickname = nickname + self.avatar = avatar + self.active = active + self.banned = banned + self.roles = roles + + def __repr__(self): + return ''.format(self.id, self.guild_id, self.user_id, self.username, self.discriminator) diff --git a/discordbot/titanembeds/database/guilds.py b/discordbot/titanembeds/database/guilds.py index d80e9f2..f540328 100644 --- a/discordbot/titanembeds/database/guilds.py +++ b/discordbot/titanembeds/database/guilds.py @@ -4,11 +4,21 @@ class Guilds(Base): __tablename__ = "guilds" id = db.Column(db.Integer, primary_key=True) # Auto incremented id guild_id = db.Column(db.String(255)) # Discord guild id + name = db.Column(db.String(255)) # Name unauth_users = db.Column(db.Boolean()) # If allowed unauth users + roles = db.Column(db.Text()) # Guild Roles + channels = db.Column(db.Text()) # Guild channels + owner_id = db.Column(db.String(255)) # Snowflake of the owner + icon = db.Column(db.String(255)) # The icon string, null if none - def __init__(self, guild_id): + def __init__(self, guild_id, name, roles, channels, owner_id, icon): self.guild_id = guild_id + self.name = name self.unauth_users = True # defaults to true + self.roles = roles + self.channels = channels + self.owner_id = owner_id + self.icon = icon def __repr__(self): - return ''.format(self.id, self.guild_id) \ No newline at end of file + return ''.format(self.id, self.guild_id) From 9c240939018d2e0ef90e18bc81e45146218ef55e Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sat, 6 May 2017 17:19:12 -0700 Subject: [PATCH 09/13] Fetching database for messages instead --- .gitignore | 4 +- discordbot/titanembeds/database/__init__.py | 29 ++++++++++- discordbot/titanembeds/database/messages.py | 4 +- webapp/titanembeds/blueprints/api/api.py | 11 ++--- webapp/titanembeds/database/__init__.py | 2 + .../titanembeds/database/custom_redislite.py | 35 ------------- webapp/titanembeds/database/guild_members.py | 28 +++++++++++ webapp/titanembeds/database/guilds.py | 12 ++++- webapp/titanembeds/database/messages.py | 49 +++++++++++++++++++ 9 files changed, 127 insertions(+), 47 deletions(-) delete mode 100644 webapp/titanembeds/database/custom_redislite.py create mode 100644 webapp/titanembeds/database/guild_members.py create mode 100644 webapp/titanembeds/database/messages.py diff --git a/.gitignore b/.gitignore index 1eb9de4..7f26ef5 100644 --- a/.gitignore +++ b/.gitignore @@ -92,5 +92,5 @@ ENV/ config.py redislite.db redislite.db.settings -tmp/* -!tmp/.gitinclude +webapp/tmp/* +!webapp/tmp/.gitinclude diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index af01c5a..ee2c08a 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -54,14 +54,38 @@ class DatabaseInterface(object): message.channel.id, message.id, message.content, + json.dumps(self.get_message_author(message)), str(message.timestamp), edit_ts, - json.dumps(message.mentions), + json.dumps(self.get_message_mentions(message.mentions)), json.dumps(message.attachments) ) session.add(msg) session.commit() + def get_message_author(self, message): + author = message.author + obj = { + "username": author.name, + "discriminator": author.discriminator, + "bot": author.bot, + "id": author.id, + "avatar": author.avatar + } + return obj + + def get_message_mentions(self, mentions): + ments = [] + for author in mentions: + ments.append({ + "username": author.name, + "discriminator": author.discriminator, + "bot": author.bot, + "id": author.id, + "avatar": author.avatar + }) + return ments + async def update_message(self, message): if message.server: async with threadpool(): @@ -73,8 +97,9 @@ class DatabaseInterface(object): if msg: msg.content = message.content msg.edited_timestamp = message.edited_timestamp - msg.mentions = json.dumps(message.mentions) + msg.mentions = json.dumps(self.get_message_mentions(message.mentions)) msg.attachments = json.dumps(message.attachments) + msg.author = json.dumps(self.get_message_author(message)) session.commit() async def delete_message(self, message): diff --git a/discordbot/titanembeds/database/messages.py b/discordbot/titanembeds/database/messages.py index d4bad0c..d7a9f20 100644 --- a/discordbot/titanembeds/database/messages.py +++ b/discordbot/titanembeds/database/messages.py @@ -7,16 +7,18 @@ class Messages(Base): channel_id = db.Column(db.String(255)) # Channel id message_id = db.Column(db.String(255)) # Message snowflake content = db.Column(db.Text()) # Message contents + author = db.Column(db.Text()) # Author json timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is created edited_timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is edited mentions = db.Column(db.Text()) # Mentions serialized attachments = db.Column(db.Text()) # serialized attachments - def __init__(self, guild_id, channel_id, message_id, content, timestamp, edited_timestamp, mentions, attachments): + def __init__(self, guild_id, channel_id, message_id, content, author, timestamp, edited_timestamp, mentions, attachments): self.guild_id = guild_id self.channel_id = channel_id self.message_id = message_id self.content = content + self.author = author self.timestamp = timestamp self.edited_timestamp = edited_timestamp self.mentions = mentions diff --git a/webapp/titanembeds/blueprints/api/api.py b/webapp/titanembeds/blueprints/api/api.py index 772d0c4..fe1d828 100644 --- a/webapp/titanembeds/blueprints/api/api.py +++ b/webapp/titanembeds/blueprints/api/api.py @@ -1,4 +1,4 @@ -from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, KeyValueProperties +from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, KeyValueProperties, get_channel_messages from titanembeds.decorators import valid_session_required, discord_users_only from titanembeds.utils import check_guild_existance, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild @@ -265,10 +265,9 @@ def fetch(): if not chan.get("read"): status_code = 401 else: - messages = discord_api.get_channel_messages(channel_id, after_snowflake) - status_code = messages['code'] - response = jsonify(messages=messages.get('content', messages), status=status) - response.status_code = status_code + messages = get_channel_messages(channel_id, after_snowflake) + response = jsonify(messages=messages, status=status) + response.status_code = 200 return response @api.route("/post", methods=["POST"]) @@ -382,4 +381,4 @@ def cleanup_keyval_db(): db.session.delete(m) db.session.commit() return ('', 204) - abort(401) \ No newline at end of file + abort(401) diff --git a/webapp/titanembeds/database/__init__.py b/webapp/titanembeds/database/__init__.py index 85fdafa..d62c160 100644 --- a/webapp/titanembeds/database/__init__.py +++ b/webapp/titanembeds/database/__init__.py @@ -6,4 +6,6 @@ from guilds import Guilds from unauthenticated_users import UnauthenticatedUsers from unauthenticated_bans import UnauthenticatedBans from authenticated_users import AuthenticatedUsers +from guild_members import GuildMembers from keyvalue_properties import KeyValueProperties, set_keyvalproperty, get_keyvalproperty, getexpir_keyvalproperty, setexpir_keyvalproperty, ifexists_keyvalproperty, delete_keyvalproperty +from messages import Messages, get_channel_messages diff --git a/webapp/titanembeds/database/custom_redislite.py b/webapp/titanembeds/database/custom_redislite.py deleted file mode 100644 index 4cade78..0000000 --- a/webapp/titanembeds/database/custom_redislite.py +++ /dev/null @@ -1,35 +0,0 @@ -import urlparse -from limits.storage import Storage -from redislite import Redis -import time - -class LimitsRedisLite(Storage): # For Python Limits - STORAGE_SCHEME = "redislite" - def __init__(self, uri, **options): - self.redis_instance = Redis(urlparse.urlparse(uri).netloc) - - def check(self): - return True - - def get_expiry(self, key): - return (self.redis_instance.ttl(key) or 0) + time.time() - - def incr(self, key, expiry, elastic_expiry=False): - if not self.redis_instance.exists(key): - self.redis_instance.set(key, 1, ex=expiry) - else: - oldexp = oldexp = self.get_expiry(key) - time.time() - if oldexp <= 0: - self.redis_instance.delete(key) - return self.incr(key, expiry, elastic_expiry) - self.redis_instance.set(key, int(self.redis_instance.get(key))+1, ex=int(round(oldexp))) - return int(self.get(key)) - - def get(self, key): - value = self.redis_instance.get(key) - if value: - return int(value) - return 0 - - def reset(self): - return self.redis_instance.flushdb() diff --git a/webapp/titanembeds/database/guild_members.py b/webapp/titanembeds/database/guild_members.py new file mode 100644 index 0000000..8488091 --- /dev/null +++ b/webapp/titanembeds/database/guild_members.py @@ -0,0 +1,28 @@ +from titanembeds.database import db + +class GuildMembers(db.Model): + __tablename__ = "guild_members" + id = db.Column(db.Integer, primary_key=True) # Auto incremented id + guild_id = db.Column(db.String(255)) # Discord guild id + user_id = db.Column(db.String(255)) # Discord user id + username = db.Column(db.String(255)) # Name + discriminator = db.Column(db.Integer) # User discriminator + nickname = db.Column(db.String(255)) # User nickname + avatar = db.Column(db.String(255)) # The avatar str of the user + active = db.Column(db.Boolean()) # If the user is a member of the guild + banned = db.Column(db.Boolean()) # If the user is banned in the guild + roles = db.Column(db.Text()) # Member roles + + def __init__(self, guild_id, user_id, username, discriminator, nickname, avatar, active, banned, roles): + self.guild_id = guild_id + self.user_id = user_id + self.username = username + self.discriminator = discriminator + self.nickname = nickname + self.avatar = avatar + self.active = active + self.banned = banned + self.roles = roles + + def __repr__(self): + return ''.format(self.id, self.guild_id, self.user_id, self.username, self.discriminator) diff --git a/webapp/titanembeds/database/guilds.py b/webapp/titanembeds/database/guilds.py index 858be37..e308dfc 100644 --- a/webapp/titanembeds/database/guilds.py +++ b/webapp/titanembeds/database/guilds.py @@ -4,11 +4,21 @@ class Guilds(db.Model): __tablename__ = "guilds" id = db.Column(db.Integer, primary_key=True) # Auto incremented id guild_id = db.Column(db.String(255)) # Discord guild id + name = db.Column(db.String(255)) # Name unauth_users = db.Column(db.Boolean()) # If allowed unauth users + roles = db.Column(db.Text()) # Guild Roles + channels = db.Column(db.Text()) # Guild channels + owner_id = db.Column(db.String(255)) # Snowflake of the owner + icon = db.Column(db.String(255)) # The icon string, null if none - def __init__(self, guild_id): + def __init__(self, guild_id, name, roles, channels, owner_id, icon): self.guild_id = guild_id + self.name = name self.unauth_users = True # defaults to true + self.roles = roles + self.channels = channels + self.owner_id = owner_id + self.icon = icon def __repr__(self): return ''.format(self.id, self.guild_id) diff --git a/webapp/titanembeds/database/messages.py b/webapp/titanembeds/database/messages.py new file mode 100644 index 0000000..66c7a76 --- /dev/null +++ b/webapp/titanembeds/database/messages.py @@ -0,0 +1,49 @@ +from titanembeds.database import db +from sqlalchemy import cast +import json + +class Messages(db.Model): + __tablename__ = "messages" + id = db.Column(db.Integer, primary_key=True) # Auto incremented id + guild_id = db.Column(db.String(255)) # Discord guild id + channel_id = db.Column(db.String(255)) # Channel id + message_id = db.Column(db.String(255)) # Message snowflake + content = db.Column(db.Text()) # Message contents + author = db.Column(db.Text()) # Author + timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is created + edited_timestamp = db.Column(db.TIMESTAMP) # Timestamp of when content is edited + mentions = db.Column(db.Text()) # Mentions serialized + attachments = db.Column(db.Text()) # serialized attachments + + def __init__(self, guild_id, channel_id, message_id, content, author, timestamp, edited_timestamp, mentions, attachments): + self.guild_id = guild_id + self.channel_id = channel_id + self.message_id = message_id + self.content = content + self.author = author + self.timestamp = timestamp + self.edited_timestamp = edited_timestamp + self.mentions = mentions + self.attachments = attachments + + def __repr__(self): + return ''.format(self.id, self.guild_id, self.guild_id, self.channel_id, self.message_id) + +def get_channel_messages(channel_id, after_snowflake=None): + if not after_snowflake: + q = db.session.query(Messages).filter(Messages.channel_id == channel_id).order_by(Messages.id.desc()).limit(50) + else: + q = db.session.query(Messages).filter(cast(Messages.channel_id, db.Integer) == int(channel_id)).filter(Messages.message_id > after_snowflake).order_by(Messages.id.desc()).limit(50) + msgs = [] + for x in q: + msgs.append({ + "attachments": json.loads(x.attachments), + "timestamp": x.timestamp, + "id": x.message_id, + "edited_timestamp": x.edited_timestamp, + "author": json.loads(x.author), + "content": x.content, + "channel_id": x.channel_id, + "mentions": json.loads(x.mentions) + }) + return msgs From fc169c20636aa61ed545428ec9ef45298b15dd0d Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sat, 6 May 2017 18:19:20 -0700 Subject: [PATCH 10/13] Transition over to use db instead of fetching discord REST --- discordbot/titanembeds/database/__init__.py | 1 + webapp/titanembeds/blueprints/api/api.py | 51 +++++++++++--------- webapp/titanembeds/database/__init__.py | 2 +- webapp/titanembeds/database/guild_members.py | 17 +++++++ 4 files changed, 47 insertions(+), 24 deletions(-) diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index ee2c08a..dc76813 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -145,6 +145,7 @@ class DatabaseInterface(object): "color": role.color.value, "hoist": role.hoist, "position": role.position, + "permissions": role.permissions.value }) return roles diff --git a/webapp/titanembeds/blueprints/api/api.py b/webapp/titanembeds/blueprints/api/api.py index fe1d828..42e349e 100644 --- a/webapp/titanembeds/blueprints/api/api.py +++ b/webapp/titanembeds/blueprints/api/api.py @@ -1,4 +1,4 @@ -from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, KeyValueProperties, get_channel_messages +from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, KeyValueProperties, GuildMembers, get_channel_messages, list_all_guild_members from titanembeds.decorators import valid_session_required, discord_users_only from titanembeds.utils import check_guild_existance, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild @@ -27,9 +27,8 @@ def checkUserRevoke(guild_id, user_key=None): banned = checkUserBanned(guild_id) if banned: return revoked - member = discord_api.get_guild_member_nocache(guild_id, session['user_id']) - if member['code'] == 200: - revoked = False + dbUser = GuildMembers.query.filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == session["user_id"]).first() + revoked = not dbUser.active return revoked def checkUserBanned(guild_id, ip_address=None): @@ -44,10 +43,8 @@ def checkUserBanned(guild_id, ip_address=None): banned = False else: banned = False - bans = discord_api.get_guild_bans(guild_id)['content'] - for user in bans: - if session['user_id'] == user['user']['id']: - return True + dbUser = GuildMembers.query.filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == session["user_id"]).first() + banned = dbUser.banned return banned def update_user_status(guild_id, username, user_key=None): @@ -97,7 +94,7 @@ def check_user_in_guild(guild_id): return guild_id in session['user_keys'] else: dbUser = db.session.query(AuthenticatedUsers).filter(and_(AuthenticatedUsers.guild_id == guild_id, AuthenticatedUsers.client_id == session['user_id'])).first() - return 200 == discord_api.get_guild_member_nocache(guild_id, session['user_id'])['code'] and dbUser is not None + return not checkUserRevoke(guild_id) and dbUser is not None def format_post_content(message): message = message.replace("<", "\<") @@ -114,20 +111,28 @@ def format_post_content(message): message = "**<{}#{}>** {}".format(session['username'], session['discriminator'], message) # I would like to do a @ mention, but i am worried about notif spam return message +def get_member_roles(guild_id, user_id): + q = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == user_id).first() + return json.loads(q.roles) + +def get_dbguild_channels(guild_id): + q = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first() + return json.loads(q.channels) + def get_guild_channels(guild_id): if user_unauthenticated(): member_roles = [guild_id] #equivilant to @everyone role else: - member = discord_api.get_guild_member(guild_id, session['user_id'])['content'] - member_roles = member['roles'] + member_roles = get_member_roles(guild_id, session['user_id']) if guild_id not in member_roles: member_roles.append(guild_id) - guild_channels = discord_api.get_guild_channels(guild_id)['content'] - guild_roles = discord_api.get_guild_roles(guild_id)["content"] - guild_owner = discord_api.get_guild(guild_id)['content']['owner_id'] + dbguild = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first() + guild_channels = json.loads(dbguild.channels) + guild_roles = json.loads(dbguild.roles) + guild_owner = json.loads(dbguild.owner_id) result_channels = [] for channel in guild_channels: - if channel['type'] == 0: + if channel['type'] == "text": result = {"channel": channel, "read": False, "write": False} if guild_owner == session['user_id']: result["read"] = True @@ -192,17 +197,17 @@ def get_guild_channels(guild_id): def filter_guild_channel(guild_id, channel_id): channels = get_guild_channels(guild_id) for chan in channels: - if chan["channel"]["id"] == guild_id: + if chan["channel"]["id"] == channel_id: return chan return None def get_online_discord_users(guild_id): embed = discord_api.get_widget(guild_id) - apimembers = discord_api.list_all_guild_members(guild_id) + apimembers = list_all_guild_members(guild_id) apimembers_filtered = {} for member in apimembers: apimembers_filtered[member["user"]["id"]] = member - guild_roles = discord_api.get_guild_roles(guild_id)["content"] + guild_roles = json.loads(db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first().roles) guildroles_filtered = {} for role in guild_roles: guildroles_filtered[role["id"]] = role @@ -235,12 +240,12 @@ def get_online_embed_users(guild_id): users['unauthenticated'].append(meta) for user in auths: client_id = user.client_id - u = discord_api.get_guild_member(guild_id, client_id)['content']['user'] + usrdb = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id).filter(GuildMembers.user_id == client_id).first() meta = { - 'id': u['id'], - 'username': u['username'], - 'discriminator': u['discriminator'], - 'avatar_url': generate_avatar_url(u['id'], u['avatar']), + 'id': usrdb.user_id, + 'username': usrdb.username, + 'discriminator': usrdb.discriminator, + 'avatar_url': generate_avatar_url(usrdb.user_id, usrdb.avatar), } users['authenticated'].append(meta) return users diff --git a/webapp/titanembeds/database/__init__.py b/webapp/titanembeds/database/__init__.py index d62c160..34644af 100644 --- a/webapp/titanembeds/database/__init__.py +++ b/webapp/titanembeds/database/__init__.py @@ -6,6 +6,6 @@ from guilds import Guilds from unauthenticated_users import UnauthenticatedUsers from unauthenticated_bans import UnauthenticatedBans from authenticated_users import AuthenticatedUsers -from guild_members import GuildMembers +from guild_members import GuildMembers, list_all_guild_members from keyvalue_properties import KeyValueProperties, set_keyvalproperty, get_keyvalproperty, getexpir_keyvalproperty, setexpir_keyvalproperty, ifexists_keyvalproperty, delete_keyvalproperty from messages import Messages, get_channel_messages diff --git a/webapp/titanembeds/database/guild_members.py b/webapp/titanembeds/database/guild_members.py index 8488091..4fd51a2 100644 --- a/webapp/titanembeds/database/guild_members.py +++ b/webapp/titanembeds/database/guild_members.py @@ -1,4 +1,5 @@ from titanembeds.database import db +import json class GuildMembers(db.Model): __tablename__ = "guild_members" @@ -26,3 +27,19 @@ class GuildMembers(db.Model): def __repr__(self): return ''.format(self.id, self.guild_id, self.user_id, self.username, self.discriminator) + +def list_all_guild_members(guild_id): + memlist = [] + members = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id).all() + for member in members: + memlist.append({ + "user": { + "username": member.username, + "discriminator": member.discriminator, + "id": member.user_id, + "avatar": member.avatar + }, + "roles": json.loads(member.roles), + "nickname": member.nickname, + }) + return memlist From 4fc0af2d80efa25abe017df0c67d49de702b7050 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sat, 6 May 2017 18:27:35 -0700 Subject: [PATCH 11/13] Strip discordrest from embed endpoint --- webapp/titanembeds/blueprints/embed/embed.py | 5 +++-- webapp/titanembeds/templates/embed.html.j2 | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/webapp/titanembeds/blueprints/embed/embed.py b/webapp/titanembeds/blueprints/embed/embed.py index b3c7360..dbb373b 100644 --- a/webapp/titanembeds/blueprints/embed/embed.py +++ b/webapp/titanembeds/blueprints/embed/embed.py @@ -1,6 +1,7 @@ from flask import Blueprint, render_template, abort, redirect, url_for, session -from titanembeds.utils import check_guild_existance, discord_api, guild_query_unauth_users_bool +from titanembeds.utils import check_guild_existance, guild_query_unauth_users_bool from titanembeds.oauth import generate_guild_icon_url, generate_avatar_url +from titanembeds.database import db, Guilds from config import config import random @@ -22,7 +23,7 @@ def get_logingreeting(): @embed.route("/") def guild_embed(guild_id): if check_guild_existance(guild_id): - guild = discord_api.get_guild(guild_id)['content'] + guild = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first() return render_template("embed.html.j2", login_greeting=get_logingreeting(), guild_id=guild_id, guild=guild, diff --git a/webapp/titanembeds/templates/embed.html.j2 b/webapp/titanembeds/templates/embed.html.j2 index 115e8cb..05f8188 100644 --- a/webapp/titanembeds/templates/embed.html.j2 +++ b/webapp/titanembeds/templates/embed.html.j2 @@ -10,7 +10,7 @@ - {{ guild['name'] }} - Embed - Titan Embeds for Discord + {{ guild.name }} - Embed - Titan Embeds for Discord {% include 'google_analytics.html.j2' %} @@ -32,16 +32,16 @@
  • - {% if guild['icon'] %} - + {% if guild.icon %} + {% endif %} - {{ guild['name'] }} + {{ guild.name }}
  • Actions
    • -
    • -
  • Open Server on Discordapp
    • +
    • +
  • Open Server on Discordapp
    • From 5b3bdaac38ae56fec836b7f4204582f9822f7959 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sat, 6 May 2017 18:36:45 -0700 Subject: [PATCH 12/13] Replace discordrest from user and utils --- webapp/titanembeds/blueprints/user/user.py | 17 ++++------------- webapp/titanembeds/utils.py | 4 ++-- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/webapp/titanembeds/blueprints/user/user.py b/webapp/titanembeds/blueprints/user/user.py index 82a0e96..8181b95 100644 --- a/webapp/titanembeds/blueprints/user/user.py +++ b/webapp/titanembeds/blueprints/user/user.py @@ -1,7 +1,6 @@ from flask import Blueprint, request, redirect, jsonify, abort, session, url_for, render_template from config import config from titanembeds.decorators import discord_users_only -from titanembeds.utils import discord_api from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans from titanembeds.oauth import authorize_url, token_url, make_authenticated_session, get_current_authenticated_user, get_user_managed_servers, check_user_can_administrate_guild, check_user_permission, generate_avatar_url, generate_guild_icon_url, generate_bot_invite_url import time @@ -74,16 +73,11 @@ def dashboard(): def administrate_guild(guild_id): if not check_user_can_administrate_guild(guild_id): return redirect(url_for("user.dashboard")) - guild = discord_api.get_guild(guild_id) - if guild['code'] != 200: + db_guild = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first() + if not db_guild: session["redirect"] = url_for("user.administrate_guild", guild_id=guild_id, _external=True) return redirect(generate_bot_invite_url(guild_id)) session["redirect"] = None - db_guild = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first() - if not db_guild: - db_guild = Guilds(guild_id) - db.session.add(db_guild) - db.session.commit() permissions=[] if check_user_permission(guild_id, 5): permissions.append("Manage Embed Settings") @@ -95,18 +89,15 @@ def administrate_guild(guild_id): all_bans = db.session.query(UnauthenticatedBans).filter(UnauthenticatedBans.guild_id == guild_id).all() users = prepare_guild_members_list(all_members, all_bans) dbguild_dict = {"unauth_users": db_guild.unauth_users} - return render_template("administrate_guild.html.j2", guild=guild['content'], dbguild=dbguild_dict, members=users, permissions=permissions) + return render_template("administrate_guild.html.j2", guild=db_guild, dbguild=dbguild_dict, members=users, permissions=permissions) @user.route("/administrate_guild/", methods=["POST"]) @discord_users_only() def update_administrate_guild(guild_id): if not check_user_can_administrate_guild(guild_id): abort(403) - guild = discord_api.get_guild(guild_id) - if guild['code'] != 200: - abort(guild['code']) db_guild = db.session.query(Guilds).filter(Guilds.guild_id == guild_id).first() - if db_guild is None: + if not db_guild: abort(400) db_guild.unauth_users = request.form.get("unauth_users", db_guild.unauth_users) in ["true", True] db.session.commit() diff --git a/webapp/titanembeds/utils.py b/webapp/titanembeds/utils.py index 49b0cdf..30fb376 100644 --- a/webapp/titanembeds/utils.py +++ b/webapp/titanembeds/utils.py @@ -67,8 +67,8 @@ def check_guild_existance(guild_id): dbGuild = Guilds.query.filter_by(guild_id=guild_id).first() if not dbGuild: return False - guild = discord_api.get_guild(guild_id) - return guild['code'] == 200 + else: + return True def guild_query_unauth_users_bool(guild_id): dbGuild = db.session.query(Guilds).filter(Guilds.guild_id==guild_id).first() From c584d8614a3b1ad10c049577e4209943f64e03e2 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sat, 6 May 2017 18:46:39 -0700 Subject: [PATCH 13/13] Update discordbot readme --- discordbot/README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/discordbot/README.md b/discordbot/README.md index 0472b08..4880b6a 100644 --- a/discordbot/README.md +++ b/discordbot/README.md @@ -1,6 +1,9 @@ # Titan - DiscordBot Portion The DiscordBot portion handles the communcation with Discord's websockets to provide real-time updates. The bot's primary role is to push content to the webapp's database to be retrieved at a later time. -It also includes misc. features to moderate guest users, etc. +It also includes misc. features to moderate guest users, etc. right in your discord server! # Installation -1. TODO, Sorry about that! (Though if you want to get ahead, install the latest discord.py pip package and that will require **Python 3.5**) \ No newline at end of file +1. Clone the repo (make sure you have **Python 3.5** installed on your system. This discordbot portion depends on that specifc Python version) +2. Install the pip requirements `pip install -r requirements.txt` +3. Clone `config.example.py` and rename it to `config.py`. Edit the file to your standards +4. Start the bot using `python run.py`