Basic message relaying via WS

cloud9_install.sh

@@ -12,7 +12,7 @@ cp ~/workspace/webapp/alembic.example.ini ~/workspace/webapp/alembic.ini
 echo "[C9Setup] Installing Titan dependencies"
 cd ~/workspace/
 sudo python3.5 -m pip install -r requirements.txt
-sudo python3.5 -m pip install alembic pymysql gevent uwsgi
+sudo python3.5 -m pip install alembic pymysql eventlet uwsgi
 
 echo "[C9Setup] Auto populating alembic.ini database url and titan database table"
 cd ~/workspace/webapp

discordbot/config.example.py

@@ -3,5 +3,7 @@ config = {
     
     'database-uri': "driver://username:password@host:port/database",
     
+    'redis-uri': "redis://",
+    
     'errorreporting-channelid': "",
 }

discordbot/titanembeds/bot.py

@@ -1,6 +1,7 @@
 from config import config
 from titanembeds.database import DatabaseInterface
 from titanembeds.commands import Commands
+from titanembeds.socketio import SocketIOInterface
 import discord
 import aiohttp
 import asyncio
@@ -17,6 +18,7 @@ class Titan(discord.Client):
         self.http.user_agent += ' TitanEmbeds-Bot'
         self.database = DatabaseInterface(self)
         self.command = Commands(self, self.database)
+        self.socketio = SocketIOInterface(self, config["redis-uri"])
         
         self.database_connected = False
         self.loop.create_task(self.send_webserver_heartbeat())
@@ -123,6 +125,7 @@ class Titan(discord.Client):
             return
         await self.wait_until_dbonline()
         await self.database.push_message(message)
+        await self.socketio.on_message(message)
 
         msg_arr = message.content.split() # split the message
         if len(message.content.split()) > 1 and message.server: #making sure there is actually stuff in the message and have arguments and check if it is sent in server (not PM)

discordbot/titanembeds/database/__init__.py

@@ -18,6 +18,8 @@ from titanembeds.database.unauthenticated_users import UnauthenticatedUsers
 from titanembeds.database.unauthenticated_bans import UnauthenticatedBans
 from titanembeds.database.keyvalue_properties import KeyValueProperties
 
+from titanembeds.utils import get_message_author, get_message_mentions, get_webhooks_list, get_emojis_list, get_roles_list, get_channels_list
+
 class DatabaseInterface(object):
     # Courtesy of https://github.com/SunDwarf/Jokusoramame
     def __init__(self, bot):
@@ -58,38 +60,15 @@ class DatabaseInterface(object):
                         message.channel.id,
                         message.id,
                         message.content,
-                        json.dumps(self.get_message_author(message)),
+                        json.dumps(get_message_author(message)),
                         str(message.timestamp),
                         edit_ts,
-                        json.dumps(self.get_message_mentions(message.mentions)),
+                        json.dumps(get_message_mentions(message.mentions)),
                         json.dumps(message.attachments)
                     )
                     session.add(msg)
                     session.commit()
 
-    def get_message_author(self, message):
-        author = message.author
-        obj = {
-            "username": author.name,
-            "discriminator": author.discriminator,
-            "bot": author.bot,
-            "id": author.id,
-            "avatar": author.avatar
-        }
-        return obj
-
-    def get_message_mentions(self, mentions):
-        ments = []
-        for author in mentions:
-            ments.append({
-                "username": author.name,
-                "discriminator": author.discriminator,
-                "bot": author.bot,
-                "id": author.id,
-                "avatar": author.avatar
-            })
-        return ments
-
     async def update_message(self, message):
         if message.server:
             async with threadpool():
@@ -101,9 +80,9 @@ class DatabaseInterface(object):
                     if msg:
                         msg.content = message.content
                         msg.edited_timestamp = message.edited_timestamp
-                        msg.mentions = json.dumps(self.get_message_mentions(message.mentions))
+                        msg.mentions = json.dumps(get_message_mentions(message.mentions))
                         msg.attachments = json.dumps(message.attachments)
-                        msg.author = json.dumps(self.get_message_author(message))
+                        msg.author = json.dumps(get_message_author(message))
                         session.commit()
 
     async def delete_message(self, message):
@@ -127,92 +106,24 @@ class DatabaseInterface(object):
                     gui = Guilds(
                         guild.id,
                         guild.name,
-                        json.dumps(self.get_roles_list(guild.roles)),
+                        json.dumps(get_roles_list(guild.roles)),
-                        json.dumps(self.get_channels_list(guild.channels)),
+                        json.dumps(get_channels_list(guild.channels)),
-                        json.dumps(self.get_webhooks_list(server_webhooks)),
+                        json.dumps(get_webhooks_list(server_webhooks)),
-                        json.dumps(self.get_emojis_list(guild.emojis)),
+                        json.dumps(get_emojis_list(guild.emojis)),
                         guild.owner_id,
                         guild.icon
                     )
                     session.add(gui)
                 else:
                     gui.name = guild.name
-                    gui.roles = json.dumps(self.get_roles_list(guild.roles))
+                    gui.roles = json.dumps(get_roles_list(guild.roles))
-                    gui.channels = json.dumps(self.get_channels_list(guild.channels))
+                    gui.channels = json.dumps(get_channels_list(guild.channels))
-                    gui.webhooks = json.dumps(self.get_webhooks_list(server_webhooks))
+                    gui.webhooks = json.dumps(get_webhooks_list(server_webhooks))
-                    gui.emojis = json.dumps(self.get_emojis_list(guild.emojis))
+                    gui.emojis = json.dumps(get_emojis_list(guild.emojis))
                     gui.owner_id = guild.owner_id
                     gui.icon = guild.icon
                 session.commit()
 
-    def get_webhooks_list(self, guild_webhooks):
-        webhooks = []
-        for webhook in guild_webhooks:
-            webhooks.append({
-                "id": webhook.id,
-                "guild_id": webhook.server.id,
-                "channel_id": webhook.channel.id,
-                "name": webhook.name,
-                "token": webhook.token,
-            })
-        return webhooks
-    
-    def get_emojis_list(self, guildemojis):
-        emojis = []
-        for emote in guildemojis:
-            emojis.append({
-                "id": emote.id,
-                "name": emote.name,
-                "require_colons": emote.require_colons,
-                "managed": emote.managed,
-                "roles": self.list_role_ids(emote.roles),
-                "url": emote.url
-            })
-        return emojis
-
-    def get_roles_list(self, guildroles):
-        roles = []
-        for role in guildroles:
-            roles.append({
-                "id": role.id,
-                "name": role.name,
-                "color": role.color.value,
-                "hoist": role.hoist,
-                "position": role.position,
-                "permissions": role.permissions.value
-            })
-        return roles
-
-    def get_channels_list(self, guildchannels):
-        channels = []
-        for channel in guildchannels:
-            if str(channel.type) == "text":
-                overwrites = []
-                for target, overwrite in channel.overwrites:
-                    if isinstance(target, discord.Role):
-                        type = "role"
-                    else:
-                        type = "member"
-                    allow, deny = overwrite.pair()
-                    allow = allow.value
-                    deny = deny.value
-                    overwrites.append({
-                        "id": target.id,
-                        "type": type,
-                        "allow": allow,
-                        "deny": deny,
-                    })
-
-                channels.append({
-                    "id": channel.id,
-                    "name": channel.name,
-                    "topic": channel.topic,
-                    "position": channel.position,
-                    "type": str(channel.type),
-                    "permission_overwrites": overwrites
-                })
-        return channels
-
     async def remove_unused_guilds(self, guilds):
         async with threadpool():
             with self.get_session() as session:

discordbot/titanembeds/socketio/__init__.py

@@ -0,0 +1 @@
+from .socketiointerface import SocketIOInterface

discordbot/titanembeds/socketio/socketiointerface.py

@@ -0,0 +1,26 @@
+import socketio
+from titanembeds.utils import get_message_author, get_message_mentions
+
+class SocketIOInterface:
+    def __init__(self, bot, redis_uri):
+        self.io = socketio.AsyncRedisManager(redis_uri, write_only=True, channel='flask-socketio')
+        self.bot = bot
+    
+    async def on_message(self, message):
+        if message.server:
+            edit_ts = message.edited_timestamp
+            if not edit_ts:
+                edit_ts = None
+            else:
+                edit_ts = str(edit_ts)
+            msg = {
+                "id": message.id,
+                "channel_id": message.channel.id,
+                "content": message.content,
+                "author": get_message_author(message),
+                "timestamp": str(message.timestamp),
+                "edited_timestamp": edit_ts,
+                "mentions": get_message_mentions(message.mentions),
+                "attachments": message.attachments,
+            }
+            await self.io.emit('MESSAGE_CREATE', data=msg, room=message.server.id, namespace='/gateway')

discordbot/titanembeds/utils.py

@@ -0,0 +1,92 @@
+import discord
+
+def get_message_author(message):
+    author = message.author
+    obj = {
+        "username": author.name,
+        "discriminator": author.discriminator,
+        "bot": author.bot,
+        "id": author.id,
+        "avatar": author.avatar
+    }
+    return obj
+
+def get_message_mentions(mentions):
+    ments = []
+    for author in mentions:
+        ments.append({
+            "username": author.name,
+            "discriminator": author.discriminator,
+            "bot": author.bot,
+            "id": author.id,
+            "avatar": author.avatar
+        })
+    return ments
+
+def get_webhooks_list(guild_webhooks):
+    webhooks = []
+    for webhook in guild_webhooks:
+        webhooks.append({
+            "id": webhook.id,
+            "guild_id": webhook.server.id,
+            "channel_id": webhook.channel.id,
+            "name": webhook.name,
+            "token": webhook.token,
+        })
+    return webhooks
+
+def get_emojis_list(guildemojis):
+    emojis = []
+    for emote in guildemojis:
+        emojis.append({
+            "id": emote.id,
+            "name": emote.name,
+            "require_colons": emote.require_colons,
+            "managed": emote.managed,
+            "roles": list_role_ids(emote.roles),
+            "url": emote.url
+        })
+    return emojis
+
+def get_roles_list(guildroles):
+    roles = []
+    for role in guildroles:
+        roles.append({
+            "id": role.id,
+            "name": role.name,
+            "color": role.color.value,
+            "hoist": role.hoist,
+            "position": role.position,
+            "permissions": role.permissions.value
+        })
+    return roles
+
+def get_channels_list(guildchannels):
+    channels = []
+    for channel in guildchannels:
+        if str(channel.type) == "text":
+            overwrites = []
+            for target, overwrite in channel.overwrites:
+                if isinstance(target, discord.Role):
+                    type = "role"
+                else:
+                    type = "member"
+                allow, deny = overwrite.pair()
+                allow = allow.value
+                deny = deny.value
+                overwrites.append({
+                    "id": target.id,
+                    "type": type,
+                    "allow": allow,
+                    "deny": deny,
+                })
+
+            channels.append({
+                "id": channel.id,
+                "name": channel.name,
+                "topic": channel.topic,
+                "position": channel.position,
+                "type": str(channel.type),
+                "permission_overwrites": overwrites
+            })
+    return channels

requirements.txt

@@ -7,3 +7,5 @@ flask_socketio
 paypalrestsdk
 https://github.com/TitanEmbeds/discord.py/archive/async.zip#egg=discord.py[voice]
 asyncio_extras
+kombu
+redis

webapp/config.example.py

@@ -13,4 +13,6 @@ config = {
     'app-secret': "Type something random here, go wild.",
 
     'database-uri': "driver://username:password@host:port/database",
+    'redis-uri': "redis://",
+    'websockets-mode': "LITTERALLY None or eventlet or gevent",
 }

webapp/titanembeds/app.py

@@ -7,6 +7,12 @@ from .blueprints import api, user, admin, embed, gateway
 import os
 from titanembeds.database import get_administrators_list
 
+if config.get("websockets-mode", None) == "eventlet":
+    import eventlet
+    eventlet.monkey_patch()
+elif config.get("websockets-mode", None) == "gevent":
+    from gevent import monkey
+    monkey.patch_all()
 
 os.chdir(config['app-location'])
 app = Flask(__name__, static_folder="static")
@@ -20,7 +26,7 @@ app.secret_key = config['app-secret']
 db.init_app(app)
 rate_limiter.init_app(app)
 sslify = SSLify(app, permanent=True)
-socketio.init_app(app)
+socketio.init_app(app, message_queue=config["redis-uri"], path='gateway', async_mode=config.get("websockets-mode", None))
 
 app.register_blueprint(api.api, url_prefix="/api", template_folder="/templates")
 app.register_blueprint(admin.admin, url_prefix="/admin", template_folder="/templates")

webapp/titanembeds/blueprints/gateway/gateway.py

@@ -1,6 +1,22 @@
 from titanembeds.utils import socketio
-from flask_socketio import Namespace, emit
+from flask_socketio import Namespace, emit, disconnect, join_room
+import functools
+from flask import request, session
+
+def authenticated_only(f):
+    @functools.wraps(f)
+    def wrapped(*args, **kwargs):
+        if False:
+            pass
+            #disconnect()
+        else:
+            return f(*args, **kwargs)
+    return wrapped
 
 class Gateway(Namespace):
     def on_connect(self):
-        emit('key', {'data': 'Connected', 'best_pone': "rainbow"})
+        emit('hello')
+    
+    def on_identify(self, data):
+        room = data["guild_id"]
+        join_room(room)

webapp/titanembeds/static/js/embed.js

@@ -249,12 +249,6 @@
         } else {
             primeEmbed();
         }
-        
-        /* SocketIO Test */
-        var socket = io.connect(location.protocol + '//' + document.domain + ':' + location.port + "/gateway", {path: '/gateway'});
-        socket.on('connect', function() {
-            console.log("Socket.IO Connected!");
-        });
     });
     
     function changeTheme(theme=null, keep_custom_css=true) {
@@ -756,20 +750,16 @@
             } else {
                 $("#administrate_link").hide();
             }
-            if (times_fetched % 10 == 0 || priority_query_guild) {
+            var guild = query_guild();
-              var guild = query_guild();
+            guild.done(function(guildobj) {
-              guild.done(function(guildobj) {
+                priority_query_guild = false;
-                  priority_query_guild = false;
+                fill_channels(guildobj.channels);
-                  fill_channels(guildobj.channels);
+                fill_discord_members(guildobj.discordmembers);
-                  fill_discord_members(guildobj.discordmembers);
+                fill_authenticated_users(guildobj.embedmembers.authenticated);
-                  fill_authenticated_users(guildobj.embedmembers.authenticated);
+                fill_unauthenticated_users(guildobj.embedmembers.unauthenticated);
-                  fill_unauthenticated_users(guildobj.embedmembers.unauthenticated);
+                $("#instant-inv").attr("href", guildobj.instant_invite);
-                  $("#instant-inv").attr("href", guildobj.instant_invite);
+                initiate_websockets();
-                  fetchtimeout = setTimeout(run_fetch_routine, 5000);
+            });
-              });
-            } else {
-              fetchtimeout = setTimeout(run_fetch_routine, 5000);
-            }
         });
         fet.fail(function(data) {
             if (data.status == 403) {
@@ -953,4 +943,15 @@
         // basically copied and pasted of browser ponies bookmarklet
         (function (srcs,cfg) { var cbcount = 1; var callback = function () { -- cbcount; if (cbcount === 0) { BrowserPonies.setBaseUrl(cfg.baseurl); if (!BrowserPoniesBaseConfig.loaded) { BrowserPonies.loadConfig(BrowserPoniesBaseConfig); BrowserPoniesBaseConfig.loaded = true; } BrowserPonies.loadConfig(cfg); if (!BrowserPonies.running()) BrowserPonies.start(); } }; if (typeof(BrowserPoniesConfig) === "undefined") { window.BrowserPoniesConfig = {}; } if (typeof(BrowserPoniesBaseConfig) === "undefined") { ++ cbcount; BrowserPoniesConfig.onbasecfg = callback; } if (typeof(BrowserPonies) === "undefined") { ++ cbcount; BrowserPoniesConfig.oninit = callback; } var node = (document.body || document.documentElement || document.getElementsByTagName('head')[0]); for (var id in srcs) { if (document.getElementById(id)) continue; if (node) { var s = document.createElement('script'); s.type = 'text/javascript'; s.id = id; s.src = srcs[id]; node.appendChild(s); } else { document.write('\u003cscript type="text/javscript" src="'+ srcs[id]+'" id="'+id+'"\u003e\u003c/script\u003e'); } } callback();})({"browser-ponies-script":"https://panzi.github.io/Browser-Ponies/browserponies.js","browser-ponies-config":"https://panzi.github.io/Browser-Ponies/basecfg.js"},{"baseurl":"https://panzi.github.io/Browser-Ponies/","fadeDuration":500,"volume":1,"fps":25,"speed":3,"audioEnabled":false,"showFps":false,"showLoadProgress":true,"speakProbability":0.1,"spawn":{"applejack":1,"fluttershy":1,"pinkie pie":1,"rainbow dash":1,"rarity":1,"twilight sparkle":1}});
     });
+    
+    function initiate_websockets() {
+        var socket = io.connect(location.protocol + '//' + document.domain + ':' + location.port + "/gateway", {path: '/gateway', transports: ['websocket']});
+        socket.on('connect', function() {
+            socket.emit('identify', {"guild_id": guild_id});
+        });
+        
+        socket.on('MESSAGE_CREATE', function(msg) {
+            console.log(msg);
+        });
+    }
 })();

webapp/titanembeds/utils.py

@@ -90,4 +90,4 @@ def bot_alive():
     return results
 
 rate_limiter = Limiter(key_func=get_client_ipaddr) # Default limit by ip address
-socketio = SocketIO(path='gateway')
+socketio = SocketIO()