WS Realtime for message actions and Sidebar members list presence update

discordbot/titanembeds/bot.py

@@ -139,10 +139,12 @@ class Titan(discord.Client):
     async def on_message_edit(self, message_before, message_after):
         await self.wait_until_dbonline()
         await self.database.update_message(message_after)
+        await self.socketio.on_message_update(message_after)
 
     async def on_message_delete(self, message):
         await self.wait_until_dbonline()
         await self.database.delete_message(message)
+        await self.socketio.on_message_delete(message)
 
     async def on_server_join(self, guild):
         await self.wait_until_dbonline()
@@ -201,14 +203,17 @@ class Titan(discord.Client):
     async def on_member_join(self, member):
         await self.wait_until_dbonline()
         await self.database.update_guild_member(member, active=True, banned=False)
+        await self.socketio.on_guild_member_add(member)
 
     async def on_member_remove(self, member):
         await self.wait_until_dbonline()
         await self.database.update_guild_member(member, active=False, banned=False)
+        await self.socketio.on_guild_member_remove(member)
 
     async def on_member_update(self, memberbefore, memberafter):
         await self.wait_until_dbonline()
         await self.database.update_guild_member(memberafter)
+        await self.socketio.on_guild_member_update(memberafter)
 
     async def on_member_ban(self, member):
         await self.wait_until_dbonline()

discordbot/titanembeds/database/__init__.py

@@ -18,7 +18,7 @@ from titanembeds.database.unauthenticated_users import UnauthenticatedUsers
 from titanembeds.database.unauthenticated_bans import UnauthenticatedBans
 from titanembeds.database.keyvalue_properties import KeyValueProperties
 
-from titanembeds.utils import get_message_author, get_message_mentions, get_webhooks_list, get_emojis_list, get_roles_list, get_channels_list
+from titanembeds.utils import get_message_author, get_message_mentions, get_webhooks_list, get_emojis_list, get_roles_list, get_channels_list, list_role_ids
 
 class DatabaseInterface(object):
     # Courtesy of https://github.com/SunDwarf/Jokusoramame
@@ -167,7 +167,7 @@ class DatabaseInterface(object):
                         member.avatar,
                         active,
                         banned,
-                        json.dumps(self.list_role_ids(member.roles))
+                        json.dumps(list_role_ids(member.roles))
                     )
                     session.add(dbmember)
                 else:
@@ -177,7 +177,7 @@ class DatabaseInterface(object):
                     dbmember.discriminator = member.discriminator
                     dbmember.nickname = member.nick
                     dbmember.avatar = member.avatar
-                    dbmember.roles = json.dumps(self.list_role_ids(member.roles))
+                    dbmember.roles = json.dumps(list_role_ids(member.roles))
                 session.commit()
 
     async def unban_server_user(self, user, server):
@@ -205,12 +205,6 @@ class DatabaseInterface(object):
                 if changed:
                     session.commit()
 
-    def list_role_ids(self, usr_roles):
-        ids = []
-        for role in usr_roles:
-            ids.append(role.id)
-        return ids
-
     async def flag_unactive_bans(self, guild_id, guildbans):
         async with threadpool():
             with self.get_session() as session:

discordbot/titanembeds/socketio/socketiointerface.py

@@ -1,35 +1,93 @@
 import socketio
 from titanembeds.utils import get_message_author, get_message_mentions
+import time
+from email import utils as emailutils
 
 class SocketIOInterface:
     def __init__(self, bot, redis_uri):
         self.io = socketio.AsyncRedisManager(redis_uri, write_only=True, channel='flask-socketio')
         self.bot = bot
     
+    def format_datetime(self, datetimeobj):
+        return emailutils.formatdate(time.mktime(datetimeobj.timetuple())) # https://stackoverflow.com/questions/3453177/convert-python-datetime-to-rfc-2822
+    
+    def get_formatted_message(self, message):
+        edit_ts = message.edited_timestamp
+        if not edit_ts:
+            edit_ts = None
+        else:
+            edit_ts = self.format_datetime(edit_ts)
+        msg = {
+            "id": message.id,
+            "channel_id": message.channel.id,
+            "content": message.content,
+            "author": get_message_author(message),
+            "timestamp": self.format_datetime(message.timestamp),
+            "edited_timestamp": edit_ts,
+            "mentions": get_message_mentions(message.mentions),
+            "attachments": message.attachments,
+        }
+        nickname = None
+        if hasattr(message.author, 'nick') and message.author.nick:
+            nickname = message.author.nick
+        msg["author"]["nickname"] = nickname
+        for mention in msg["mentions"]:
+            mention["nickname"] = None
+            member = message.server.get_member(mention["id"])
+            if member:
+                mention["nickname"] = member.nick
+        return msg
+    
     async def on_message(self, message):
         if message.server:
-            edit_ts = message.edited_timestamp
+            msg = self.get_formatted_message(message)
-            if not edit_ts:
+            await self.io.emit('MESSAGE_CREATE', data=msg, room=str("CHANNEL_"+message.channel.id), namespace='/gateway')
-                edit_ts = None
+    
-            else:
+    async def on_message_delete(self, message):
-                edit_ts = str(edit_ts)
+        if message.server:
-            msg = {
+            msg = self.get_formatted_message(message)
-                "id": message.id,
+            await self.io.emit('MESSAGE_DELETE', data=msg, room=str("CHANNEL_"+message.channel.id), namespace='/gateway')
-                "channel_id": message.channel.id,
+    
-                "content": message.content,
+    async def on_message_update(self, message):
-                "author": get_message_author(message),
+        if message.server:
-                "timestamp": str(message.timestamp),
+            msg = self.get_formatted_message(message)
-                "edited_timestamp": edit_ts,
+            await self.io.emit('MESSAGE_UPDATE', data=msg, room=str("CHANNEL_"+message.channel.id), namespace='/gateway')
-                "mentions": get_message_mentions(message.mentions),
+    
-                "attachments": message.attachments,
+    def get_formatted_user(self, user):
+        userobj = {
+            "avatar": user.avatar,
+            "avatar_url": user.avatar_url,
+            "color": str(user.color)[1:],
+            "discriminator": user.discriminator,
+            "game": None,
+            "hoist-role": None,
+            "id": user.id,
+            "status": str(user.status),
+            "username": user.name,
+        }
+        if user.game:
+            userobj["game"] = {
+                "name": user.game.name
             }
-            nickname = None
+        roles = sorted(user.roles, key=lambda k: k.position, reverse=True)
-            if hasattr(message.author, 'nick') and message.author.nick:
+        for role in roles:
-                nickname = message.author.nick
+            if role.hoist:
-            msg["author"]["nickname"] = nickname
+                userobj["hoist-role"] = {
-            for mention in msg["mentions"]:
+                    "id": role.id,
-                mention["nickname"] = None
+                    "name": role.name,
-                member = message.server.get_member(mention["id"])
+                    "position": role.position,
-                if member:
+                }
-                    mention["nickname"] = member.nick
+                break
-            await self.io.emit('MESSAGE_CREATE', data=msg, room=str("CHANNEL_"+message.channel.id), namespace='/gateway')
+        return userobj
+
+    async def on_guild_member_add(self, member):
+        user = self.get_formatted_user(member)
+        await self.io.emit('GUILD_MEMBER_ADD', data=user, room=str("GUILD_"+member.server.id), namespace='/gateway')
+        
+    async def on_guild_member_remove(self, member):
+        user = self.get_formatted_user(member)
+        await self.io.emit('GUILD_MEMBER_REMOVE', data=user, room=str("GUILD_"+member.server.id), namespace='/gateway')
+        
+    async def on_guild_member_update(self, member):
+        user = self.get_formatted_user(member)
+        await self.io.emit('GUILD_MEMBER_UPDATE', data=user, room=str("GUILD_"+member.server.id), namespace='/gateway')

discordbot/titanembeds/utils.py

@@ -89,4 +89,10 @@ def get_channels_list(guildchannels):
                 "type": str(channel.type),
                 "permission_overwrites": overwrites
             })
-    return channels
+    return channels
+    
+def list_role_ids(usr_roles):
+    ids = []
+    for role in usr_roles:
+        ids.append(role.id)
+    return ids

webapp/titanembeds/blueprints/gateway/gateway.py

@@ -1,5 +1,6 @@
 from titanembeds.utils import socketio, guild_accepts_visitors, get_client_ipaddr
 from titanembeds.userbookkeeping import check_user_in_guild, get_guild_channels, update_user_status
+from titanembeds.database import db, GuildMembers
 from flask_socketio import Namespace, emit, disconnect, join_room
 import functools
 from flask import request, session
@@ -14,6 +15,7 @@ class Gateway(Namespace):
         if not guild_accepts_visitors(guild_id) and not check_user_in_guild(guild_id):
             disconnect()
             return
+        session["socket_guild_id"] = guild_id
         channels = []
         if guild_accepts_visitors(guild_id) and not check_user_in_guild(guild_id):
             channels = get_guild_channels(guild_id, force_everyone=True)
@@ -27,13 +29,34 @@ class Gateway(Namespace):
             join_room("IP_"+get_client_ipaddr())
         elif not session.get("unauthenticated", True):
             join_room("USER_"+session["user_id"])
+        visitor_mode = data["visitor_mode"]
+        if not visitor_mode:
+            if session["unauthenticated"]:
+                emit("embed_user_connect", {"unauthenticated": True, "username": session["username"], "discriminator": session["user_id"]}, room="GUILD_"+guild_id)
+            else:
+                nickname = db.session.query(GuildMembers).filter(GuildMembers.guild_id == guild_id, GuildMembers.user_id == session["user_id"]).first().nickname
+                emit("embed_user_connect", {"unauthenticated": False, "id": session["user_id"], "nickname": nickname, "discriminator": session["discriminator"], "avatar_url": session["avatar"]}, room="GUILD_"+guild_id)
         emit("identified")
     
+    def on_disconnect(self):
+        if "user_keys" not in session:
+            return
+        guild_id = session["socket_guild_id"]
+        msg = {}
+        if session["unauthenticated"]:
+            msg = {"unauthenticated": True, "username": session["username"], "discriminator": session["user_id"]}
+        else:
+            msg = {"unauthenticated": False, "id": session["user_id"]}
+        emit("embed_user_disconnect", msg, room="GUILD_"+guild_id)
+    
     def on_heartbeat(self, data):
         guild_id = data["guild_id"]
         visitor_mode = data["visitor_mode"]
         if not visitor_mode:
-            status = update_user_status(guild_id, session["username"], session["user_keys"][guild_id])
+            key = None
+            if session["unauthenticated"]:
+                key = session["user_keys"][guild_id]
+            status = update_user_status(guild_id, session["username"], key)
             if status["revoked"] or status["banned"]:
                 emit("revoke")
                 time.sleep(1000)

webapp/titanembeds/static/js/embed.js

@@ -24,6 +24,9 @@
     var current_username_discrim; // Current username/discrim pair, eg EndenDraogn#4151
     var visitor_mode = false; // Keep track of if using the visitor mode or authenticate mode
     var socket = null; // Socket.io object
+    var authenticated_users_list = []; // List of all authenticated users
+    var unauthenticated_users_list = []; // List of all guest users
+    var discord_users_list = []; // List of all discord users that are probably online
 
     function element_in_view(element, fullyInView) {
         var pageTop = $(window).scrollTop();
@@ -437,6 +440,7 @@
     }
 
     function fill_discord_members(discordmembers) {
+        discord_users_list = discordmembers;
         var template = $('#mustache_authedusers').html();
         Mustache.parse(template);
         $("#discord-members").empty();
@@ -518,6 +522,7 @@
               mention_member(event.data.member_id);
             });
         }
+        authenticated_users_list = users;
     }
 
     function fill_unauthenticated_users(users) {
@@ -530,6 +535,7 @@
             var rendered = Mustache.render(template, {"username": member.username, "discriminator": member.discriminator});
             $("#embed-unauth-users").append(rendered);
         }
+        unauthenticated_users_list = users;
     }
 
     function wait_for_discord_login() {
@@ -682,7 +688,7 @@
         return text;
     }
 
-    function fill_discord_messages(messages, jumpscroll) {
+    function fill_discord_messages(messages, jumpscroll, replace=null) {
         if (messages.length == 0) {
             return last_message_id;
         }
@@ -704,12 +710,19 @@
                 username = message.author.nickname;
             }
             var rendered = Mustache.render(template, {"id": message.id, "full_timestamp": message.formatted_timestamp, "time": message.formatted_time, "username": username, "discriminator": message.author.discriminator, "content": nl2br(message.content)});
-            $("#chatcontent").append(rendered);
+            if (replace == null) {
+                $("#chatcontent").append(rendered);
+                handle_last_message_mention();
+                $("#chatcontent p:last-child").find(".blockcode").find("br").remove(); // Remove excessive breaks in codeblocks
+            } else {
+                replace.html($(rendered).html());
+                replace.find(".blockcode").find("br").remove();
+            }
             last = message.id;
-            handle_last_message_mention();
-            $("#chatcontent p:last-child").find(".blockcode").find("br").remove(); // Remove excessive breaks in codeblocks
         }
-        $("html, body").animate({ scrollTop: $(document).height() }, "slow");
+        if (replace == null) {
+            $("html, body").animate({ scrollTop: $(document).height() }, "slow");
+        }
         $('#chatcontent').linkify({
             target: "_blank"
         });
@@ -924,7 +937,7 @@
         
         socket = io.connect(location.protocol + '//' + document.domain + ':' + location.port + "/gateway", {path: '/gateway', transports: ['websocket']});
         socket.on('connect', function () {
-            socket.emit('identify', {"guild_id": guild_id});
+            socket.emit('identify', {"guild_id": guild_id, "visitor_mode": visitor_mode});
         });
         
         socket.on("disconnect", function () {
@@ -938,6 +951,50 @@
             Materialize.toast('Authentication error! You have been disconnected by the server.', 10000);
         });
         
+        socket.on("embed_user_connect", function (msg) {
+            if (msg.unauthenticated) {
+                for (var i = 0; i < unauthenticated_users_list.length; i++) {
+                    var item = unauthenticated_users_list[i];
+                    if (item.username == msg.username && item.discriminator == msg.discriminator) {
+                        return;
+                    }
+                }
+                unauthenticated_users_list.push(msg);
+                fill_unauthenticated_users(unauthenticated_users_list);
+            } else {
+                for (var i = 0; i < authenticated_users_list.length; i++) {
+                    var item = authenticated_users_list[i];
+                    if (item.id == msg.id) {
+                        return;
+                    }
+                }
+                authenticated_users_list.push(msg);
+                fill_authenticated_users(authenticated_users_list);
+            }
+        });
+        
+        socket.on("embed_user_disconnect", function (msg) {
+            if (msg.unauthenticated) {
+                for (var i = 0; i < unauthenticated_users_list.length; i++) {
+                    var item = unauthenticated_users_list[i];
+                    if (item.username == msg.username && item.discriminator == msg.discriminator) {
+                        unauthenticated_users_list.splice(i, 1);
+                        fill_unauthenticated_users(unauthenticated_users_list);
+                        return;
+                    }
+                }
+            } else {
+                for (var i = 0; i < authenticated_users_list.length; i++) {
+                    var item = authenticated_users_list[i];
+                    if (item.id == msg.id) {
+                        authenticated_users_list.splice(i, 1);
+                        fill_authenticated_users(authenticated_users_list);
+                        return;
+                    }
+                }
+            }
+        });
+        
         socket.on("MESSAGE_CREATE", function (msg) {
             var thismsgchan = msg.channel_id;
             if (selected_channel != thismsgchan) {
@@ -946,6 +1003,57 @@
             var jumpscroll = element_in_view($('#discordmessage_'+last_message_id), true);
             last_message_id = fill_discord_messages([msg], jumpscroll);
         });
+        
+        socket.on("MESSAGE_DELETE", function (msg) {
+            var msgchan = msg.channel_id;
+            if (selected_channel != msgchan) {
+                return;
+            }
+            $("#discordmessage_"+msg.id).parent().remove();
+            last_message_id = $("#chatcontent").find("[id^=discordmessage_]").last().attr('id').substring(15);
+        });
+        
+        socket.on("MESSAGE_UPDATE", function (msg) {
+            var msgelem = $("#discordmessage_"+msg.id);
+            if (msgelem.length == 0) {
+                return;
+            }
+            var msgelem_parent = msgelem.parent();
+            fill_discord_messages([msg], false, msgelem_parent);
+        });
+        
+        socket.on("GUILD_MEMBER_ADD", function (usr) {
+            if (usr.status != "offline") {
+                discord_users_list.push(usr);
+                fill_discord_members(discord_users_list);
+            }
+        });
+        
+        socket.on("GUILD_MEMBER_UPDATE", function (usr) {
+            for (var i = 0; i < discord_users_list.length; i++) {
+                if (usr.id == discord_users_list[i].id) {
+                    if (usr.status == "offline") {
+                        discord_users_list.splice(i, 1);
+                        fill_discord_members(discord_users_list);
+                        return;
+                    } else {
+                        return;
+                    }
+                }
+            }
+            discord_users_list.push(usr);
+            fill_discord_members(discord_users_list);
+        });
+
+        socket.on("GUILD_MEMBER_REMOVE", function (usr) {
+            for (var i = 0; i < discord_users_list.length; i++) {
+                if (usr.id == discord_users_list[i].id) {
+                    discord_users_list.splice(i, 1);
+                    fill_discord_members(discord_users_list);
+                    return;
+                }
+            }
+        });
     }
     
     function send_socket_heartbeat() {