From 72649f069e9f539e5b4358f8c46e0980ad7e5b63 Mon Sep 17 00:00:00 2001 From: Jeremy Zhang Date: Sun, 20 Aug 2017 19:56:54 +0000 Subject: [PATCH] Basic message relaying via WS --- cloud9_install.sh | 2 +- discordbot/config.example.py | 2 + discordbot/titanembeds/bot.py | 3 + discordbot/titanembeds/database/__init__.py | 117 +++--------------- discordbot/titanembeds/socketio/__init__.py | 1 + .../titanembeds/socketio/socketiointerface.py | 26 ++++ discordbot/titanembeds/utils.py | 92 ++++++++++++++ requirements.txt | 2 + webapp/config.example.py | 2 + webapp/titanembeds/app.py | 8 +- .../titanembeds/blueprints/gateway/gateway.py | 20 ++- webapp/titanembeds/static/js/embed.js | 41 +++--- webapp/titanembeds/utils.py | 2 +- 13 files changed, 190 insertions(+), 128 deletions(-) create mode 100644 discordbot/titanembeds/socketio/__init__.py create mode 100644 discordbot/titanembeds/socketio/socketiointerface.py create mode 100644 discordbot/titanembeds/utils.py diff --git a/cloud9_install.sh b/cloud9_install.sh index e6fd011..88650f0 100644 --- a/cloud9_install.sh +++ b/cloud9_install.sh @@ -12,7 +12,7 @@ cp ~/workspace/webapp/alembic.example.ini ~/workspace/webapp/alembic.ini echo "[C9Setup] Installing Titan dependencies" cd ~/workspace/ sudo python3.5 -m pip install -r requirements.txt -sudo python3.5 -m pip install alembic pymysql gevent uwsgi +sudo python3.5 -m pip install alembic pymysql eventlet uwsgi echo "[C9Setup] Auto populating alembic.ini database url and titan database table" cd ~/workspace/webapp diff --git a/discordbot/config.example.py b/discordbot/config.example.py index 67e7003..e910ff8 100644 --- a/discordbot/config.example.py +++ b/discordbot/config.example.py @@ -3,5 +3,7 @@ config = { 'database-uri': "driver://username:password@host:port/database", + 'redis-uri': "redis://", + 'errorreporting-channelid': "", } \ No newline at end of file diff --git a/discordbot/titanembeds/bot.py b/discordbot/titanembeds/bot.py index a6327b3..a1bf2b9 100644 --- a/discordbot/titanembeds/bot.py +++ b/discordbot/titanembeds/bot.py @@ -1,6 +1,7 @@ from config import config from titanembeds.database import DatabaseInterface from titanembeds.commands import Commands +from titanembeds.socketio import SocketIOInterface import discord import aiohttp import asyncio @@ -17,6 +18,7 @@ class Titan(discord.Client): self.http.user_agent += ' TitanEmbeds-Bot' self.database = DatabaseInterface(self) self.command = Commands(self, self.database) + self.socketio = SocketIOInterface(self, config["redis-uri"]) self.database_connected = False self.loop.create_task(self.send_webserver_heartbeat()) @@ -123,6 +125,7 @@ class Titan(discord.Client): return await self.wait_until_dbonline() await self.database.push_message(message) + await self.socketio.on_message(message) msg_arr = message.content.split() # split the message if len(message.content.split()) > 1 and message.server: #making sure there is actually stuff in the message and have arguments and check if it is sent in server (not PM) diff --git a/discordbot/titanembeds/database/__init__.py b/discordbot/titanembeds/database/__init__.py index 3c22c11..d6dbf5e 100644 --- a/discordbot/titanembeds/database/__init__.py +++ b/discordbot/titanembeds/database/__init__.py @@ -18,6 +18,8 @@ from titanembeds.database.unauthenticated_users import UnauthenticatedUsers from titanembeds.database.unauthenticated_bans import UnauthenticatedBans from titanembeds.database.keyvalue_properties import KeyValueProperties +from titanembeds.utils import get_message_author, get_message_mentions, get_webhooks_list, get_emojis_list, get_roles_list, get_channels_list + class DatabaseInterface(object): # Courtesy of https://github.com/SunDwarf/Jokusoramame def __init__(self, bot): @@ -58,38 +60,15 @@ class DatabaseInterface(object): message.channel.id, message.id, message.content, - json.dumps(self.get_message_author(message)), + json.dumps(get_message_author(message)), str(message.timestamp), edit_ts, - json.dumps(self.get_message_mentions(message.mentions)), + json.dumps(get_message_mentions(message.mentions)), json.dumps(message.attachments) ) session.add(msg) session.commit() - def get_message_author(self, message): - author = message.author - obj = { - "username": author.name, - "discriminator": author.discriminator, - "bot": author.bot, - "id": author.id, - "avatar": author.avatar - } - return obj - - def get_message_mentions(self, mentions): - ments = [] - for author in mentions: - ments.append({ - "username": author.name, - "discriminator": author.discriminator, - "bot": author.bot, - "id": author.id, - "avatar": author.avatar - }) - return ments - async def update_message(self, message): if message.server: async with threadpool(): @@ -101,9 +80,9 @@ class DatabaseInterface(object): if msg: msg.content = message.content msg.edited_timestamp = message.edited_timestamp - msg.mentions = json.dumps(self.get_message_mentions(message.mentions)) + msg.mentions = json.dumps(get_message_mentions(message.mentions)) msg.attachments = json.dumps(message.attachments) - msg.author = json.dumps(self.get_message_author(message)) + msg.author = json.dumps(get_message_author(message)) session.commit() async def delete_message(self, message): @@ -127,91 +106,23 @@ class DatabaseInterface(object): gui = Guilds( guild.id, guild.name, - json.dumps(self.get_roles_list(guild.roles)), - json.dumps(self.get_channels_list(guild.channels)), - json.dumps(self.get_webhooks_list(server_webhooks)), - json.dumps(self.get_emojis_list(guild.emojis)), + json.dumps(get_roles_list(guild.roles)), + json.dumps(get_channels_list(guild.channels)), + json.dumps(get_webhooks_list(server_webhooks)), + json.dumps(get_emojis_list(guild.emojis)), guild.owner_id, guild.icon ) session.add(gui) else: gui.name = guild.name - gui.roles = json.dumps(self.get_roles_list(guild.roles)) - gui.channels = json.dumps(self.get_channels_list(guild.channels)) - gui.webhooks = json.dumps(self.get_webhooks_list(server_webhooks)) - gui.emojis = json.dumps(self.get_emojis_list(guild.emojis)) + gui.roles = json.dumps(get_roles_list(guild.roles)) + gui.channels = json.dumps(get_channels_list(guild.channels)) + gui.webhooks = json.dumps(get_webhooks_list(server_webhooks)) + gui.emojis = json.dumps(get_emojis_list(guild.emojis)) gui.owner_id = guild.owner_id gui.icon = guild.icon session.commit() - - def get_webhooks_list(self, guild_webhooks): - webhooks = [] - for webhook in guild_webhooks: - webhooks.append({ - "id": webhook.id, - "guild_id": webhook.server.id, - "channel_id": webhook.channel.id, - "name": webhook.name, - "token": webhook.token, - }) - return webhooks - - def get_emojis_list(self, guildemojis): - emojis = [] - for emote in guildemojis: - emojis.append({ - "id": emote.id, - "name": emote.name, - "require_colons": emote.require_colons, - "managed": emote.managed, - "roles": self.list_role_ids(emote.roles), - "url": emote.url - }) - return emojis - - def get_roles_list(self, guildroles): - roles = [] - for role in guildroles: - roles.append({ - "id": role.id, - "name": role.name, - "color": role.color.value, - "hoist": role.hoist, - "position": role.position, - "permissions": role.permissions.value - }) - return roles - - def get_channels_list(self, guildchannels): - channels = [] - for channel in guildchannels: - if str(channel.type) == "text": - overwrites = [] - for target, overwrite in channel.overwrites: - if isinstance(target, discord.Role): - type = "role" - else: - type = "member" - allow, deny = overwrite.pair() - allow = allow.value - deny = deny.value - overwrites.append({ - "id": target.id, - "type": type, - "allow": allow, - "deny": deny, - }) - - channels.append({ - "id": channel.id, - "name": channel.name, - "topic": channel.topic, - "position": channel.position, - "type": str(channel.type), - "permission_overwrites": overwrites - }) - return channels async def remove_unused_guilds(self, guilds): async with threadpool(): diff --git a/discordbot/titanembeds/socketio/__init__.py b/discordbot/titanembeds/socketio/__init__.py new file mode 100644 index 0000000..89ea000 --- /dev/null +++ b/discordbot/titanembeds/socketio/__init__.py @@ -0,0 +1 @@ +from .socketiointerface import SocketIOInterface \ No newline at end of file diff --git a/discordbot/titanembeds/socketio/socketiointerface.py b/discordbot/titanembeds/socketio/socketiointerface.py new file mode 100644 index 0000000..e7d3366 --- /dev/null +++ b/discordbot/titanembeds/socketio/socketiointerface.py @@ -0,0 +1,26 @@ +import socketio +from titanembeds.utils import get_message_author, get_message_mentions + +class SocketIOInterface: + def __init__(self, bot, redis_uri): + self.io = socketio.AsyncRedisManager(redis_uri, write_only=True, channel='flask-socketio') + self.bot = bot + + async def on_message(self, message): + if message.server: + edit_ts = message.edited_timestamp + if not edit_ts: + edit_ts = None + else: + edit_ts = str(edit_ts) + msg = { + "id": message.id, + "channel_id": message.channel.id, + "content": message.content, + "author": get_message_author(message), + "timestamp": str(message.timestamp), + "edited_timestamp": edit_ts, + "mentions": get_message_mentions(message.mentions), + "attachments": message.attachments, + } + await self.io.emit('MESSAGE_CREATE', data=msg, room=message.server.id, namespace='/gateway') \ No newline at end of file diff --git a/discordbot/titanembeds/utils.py b/discordbot/titanembeds/utils.py new file mode 100644 index 0000000..34a91c3 --- /dev/null +++ b/discordbot/titanembeds/utils.py @@ -0,0 +1,92 @@ +import discord + +def get_message_author(message): + author = message.author + obj = { + "username": author.name, + "discriminator": author.discriminator, + "bot": author.bot, + "id": author.id, + "avatar": author.avatar + } + return obj + +def get_message_mentions(mentions): + ments = [] + for author in mentions: + ments.append({ + "username": author.name, + "discriminator": author.discriminator, + "bot": author.bot, + "id": author.id, + "avatar": author.avatar + }) + return ments + +def get_webhooks_list(guild_webhooks): + webhooks = [] + for webhook in guild_webhooks: + webhooks.append({ + "id": webhook.id, + "guild_id": webhook.server.id, + "channel_id": webhook.channel.id, + "name": webhook.name, + "token": webhook.token, + }) + return webhooks + +def get_emojis_list(guildemojis): + emojis = [] + for emote in guildemojis: + emojis.append({ + "id": emote.id, + "name": emote.name, + "require_colons": emote.require_colons, + "managed": emote.managed, + "roles": list_role_ids(emote.roles), + "url": emote.url + }) + return emojis + +def get_roles_list(guildroles): + roles = [] + for role in guildroles: + roles.append({ + "id": role.id, + "name": role.name, + "color": role.color.value, + "hoist": role.hoist, + "position": role.position, + "permissions": role.permissions.value + }) + return roles + +def get_channels_list(guildchannels): + channels = [] + for channel in guildchannels: + if str(channel.type) == "text": + overwrites = [] + for target, overwrite in channel.overwrites: + if isinstance(target, discord.Role): + type = "role" + else: + type = "member" + allow, deny = overwrite.pair() + allow = allow.value + deny = deny.value + overwrites.append({ + "id": target.id, + "type": type, + "allow": allow, + "deny": deny, + }) + + channels.append({ + "id": channel.id, + "name": channel.name, + "topic": channel.topic, + "position": channel.position, + "type": str(channel.type), + "permission_overwrites": overwrites + }) + return channels \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 35b1a41..5a6d429 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,5 @@ flask_socketio paypalrestsdk https://github.com/TitanEmbeds/discord.py/archive/async.zip#egg=discord.py[voice] asyncio_extras +kombu +redis \ No newline at end of file diff --git a/webapp/config.example.py b/webapp/config.example.py index 4c8cab7..fe8ae22 100644 --- a/webapp/config.example.py +++ b/webapp/config.example.py @@ -13,4 +13,6 @@ config = { 'app-secret': "Type something random here, go wild.", 'database-uri': "driver://username:password@host:port/database", + 'redis-uri': "redis://", + 'websockets-mode': "LITTERALLY None or eventlet or gevent", } diff --git a/webapp/titanembeds/app.py b/webapp/titanembeds/app.py index f8bcf4e..f1d1c39 100644 --- a/webapp/titanembeds/app.py +++ b/webapp/titanembeds/app.py @@ -7,6 +7,12 @@ from .blueprints import api, user, admin, embed, gateway import os from titanembeds.database import get_administrators_list +if config.get("websockets-mode", None) == "eventlet": + import eventlet + eventlet.monkey_patch() +elif config.get("websockets-mode", None) == "gevent": + from gevent import monkey + monkey.patch_all() os.chdir(config['app-location']) app = Flask(__name__, static_folder="static") @@ -20,7 +26,7 @@ app.secret_key = config['app-secret'] db.init_app(app) rate_limiter.init_app(app) sslify = SSLify(app, permanent=True) -socketio.init_app(app) +socketio.init_app(app, message_queue=config["redis-uri"], path='gateway', async_mode=config.get("websockets-mode", None)) app.register_blueprint(api.api, url_prefix="/api", template_folder="/templates") app.register_blueprint(admin.admin, url_prefix="/admin", template_folder="/templates") diff --git a/webapp/titanembeds/blueprints/gateway/gateway.py b/webapp/titanembeds/blueprints/gateway/gateway.py index 0fc7c91..cc3ebfc 100644 --- a/webapp/titanembeds/blueprints/gateway/gateway.py +++ b/webapp/titanembeds/blueprints/gateway/gateway.py @@ -1,6 +1,22 @@ from titanembeds.utils import socketio -from flask_socketio import Namespace, emit +from flask_socketio import Namespace, emit, disconnect, join_room +import functools +from flask import request, session + +def authenticated_only(f): + @functools.wraps(f) + def wrapped(*args, **kwargs): + if False: + pass + #disconnect() + else: + return f(*args, **kwargs) + return wrapped class Gateway(Namespace): def on_connect(self): - emit('key', {'data': 'Connected', 'best_pone': "rainbow"}) \ No newline at end of file + emit('hello') + + def on_identify(self, data): + room = data["guild_id"] + join_room(room) \ No newline at end of file diff --git a/webapp/titanembeds/static/js/embed.js b/webapp/titanembeds/static/js/embed.js index 27cc1b5..0078fcc 100644 --- a/webapp/titanembeds/static/js/embed.js +++ b/webapp/titanembeds/static/js/embed.js @@ -249,12 +249,6 @@ } else { primeEmbed(); } - - /* SocketIO Test */ - var socket = io.connect(location.protocol + '//' + document.domain + ':' + location.port + "/gateway", {path: '/gateway'}); - socket.on('connect', function() { - console.log("Socket.IO Connected!"); - }); }); function changeTheme(theme=null, keep_custom_css=true) { @@ -756,20 +750,16 @@ } else { $("#administrate_link").hide(); } - if (times_fetched % 10 == 0 || priority_query_guild) { - var guild = query_guild(); - guild.done(function(guildobj) { - priority_query_guild = false; - fill_channels(guildobj.channels); - fill_discord_members(guildobj.discordmembers); - fill_authenticated_users(guildobj.embedmembers.authenticated); - fill_unauthenticated_users(guildobj.embedmembers.unauthenticated); - $("#instant-inv").attr("href", guildobj.instant_invite); - fetchtimeout = setTimeout(run_fetch_routine, 5000); - }); - } else { - fetchtimeout = setTimeout(run_fetch_routine, 5000); - } + var guild = query_guild(); + guild.done(function(guildobj) { + priority_query_guild = false; + fill_channels(guildobj.channels); + fill_discord_members(guildobj.discordmembers); + fill_authenticated_users(guildobj.embedmembers.authenticated); + fill_unauthenticated_users(guildobj.embedmembers.unauthenticated); + $("#instant-inv").attr("href", guildobj.instant_invite); + initiate_websockets(); + }); }); fet.fail(function(data) { if (data.status == 403) { @@ -953,4 +943,15 @@ // basically copied and pasted of browser ponies bookmarklet (function (srcs,cfg) { var cbcount = 1; var callback = function () { -- cbcount; if (cbcount === 0) { BrowserPonies.setBaseUrl(cfg.baseurl); if (!BrowserPoniesBaseConfig.loaded) { BrowserPonies.loadConfig(BrowserPoniesBaseConfig); BrowserPoniesBaseConfig.loaded = true; } BrowserPonies.loadConfig(cfg); if (!BrowserPonies.running()) BrowserPonies.start(); } }; if (typeof(BrowserPoniesConfig) === "undefined") { window.BrowserPoniesConfig = {}; } if (typeof(BrowserPoniesBaseConfig) === "undefined") { ++ cbcount; BrowserPoniesConfig.onbasecfg = callback; } if (typeof(BrowserPonies) === "undefined") { ++ cbcount; BrowserPoniesConfig.oninit = callback; } var node = (document.body || document.documentElement || document.getElementsByTagName('head')[0]); for (var id in srcs) { if (document.getElementById(id)) continue; if (node) { var s = document.createElement('script'); s.type = 'text/javascript'; s.id = id; s.src = srcs[id]; node.appendChild(s); } else { document.write('\u003cscript type="text/javscript" src="'+ srcs[id]+'" id="'+id+'"\u003e\u003c/script\u003e'); } } callback();})({"browser-ponies-script":"https://panzi.github.io/Browser-Ponies/browserponies.js","browser-ponies-config":"https://panzi.github.io/Browser-Ponies/basecfg.js"},{"baseurl":"https://panzi.github.io/Browser-Ponies/","fadeDuration":500,"volume":1,"fps":25,"speed":3,"audioEnabled":false,"showFps":false,"showLoadProgress":true,"speakProbability":0.1,"spawn":{"applejack":1,"fluttershy":1,"pinkie pie":1,"rainbow dash":1,"rarity":1,"twilight sparkle":1}}); }); + + function initiate_websockets() { + var socket = io.connect(location.protocol + '//' + document.domain + ':' + location.port + "/gateway", {path: '/gateway', transports: ['websocket']}); + socket.on('connect', function() { + socket.emit('identify', {"guild_id": guild_id}); + }); + + socket.on('MESSAGE_CREATE', function(msg) { + console.log(msg); + }); + } })(); diff --git a/webapp/titanembeds/utils.py b/webapp/titanembeds/utils.py index bfe4d23..7c5f7a4 100644 --- a/webapp/titanembeds/utils.py +++ b/webapp/titanembeds/utils.py @@ -90,4 +90,4 @@ def bot_alive(): return results rate_limiter = Limiter(key_func=get_client_ipaddr) # Default limit by ip address -socketio = SocketIO(path='gateway') \ No newline at end of file +socketio = SocketIO() \ No newline at end of file