Move tracking online users to redis

discordbot/titanembeds/database/unauthenticated_users.py

@@ -12,8 +12,7 @@ class UnauthenticatedUsers(Base):
     discriminator = db.Column(db.Integer)           # The discriminator to distinguish unauth users with each other
     user_key = db.Column(db.Text())                 # The secret key used to identify the user holder
     ip_address = db.Column(db.String(255))          # The IP Address of the user
-    last_timestamp = db.Column(db.TIMESTAMP)        # The timestamp of when the user has last sent the heartbeat
     revoked = db.Column(db.Boolean())               # If the user's key has been revoked and a new one is required to be generated
 
     def __repr__(self):
-        return '<UnauthenticatedUsers {0} {1} {2} {3} {4} {5} {6} {7}>'.format(self.id, self.guild_id, self.username, self.discriminator, self.user_key, self.ip_address, self.last_timestamp, self.revoked)
+        return '<UnauthenticatedUsers {0} {1} {2} {3} {4} {5} {6}>'.format(self.id, self.guild_id, self.username, self.discriminator, self.user_key, self.ip_address, self.revoked)

webapp/alembic/versions/5642232899cf_removed_last_timestamp_columns.py

@@ -0,0 +1,30 @@
+"""Removed last timestamp columns
+
+Revision ID: 5642232899cf
+Revises: 0b7761d85555
+Create Date: 2018-01-21 03:03:32.359959
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '5642232899cf'
+down_revision = '0b7761d85555'
+branch_labels = None
+depends_on = None
+
+from alembic import op
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_column('authenticated_users', 'last_timestamp')
+    op.drop_column('unauthenticated_users', 'last_timestamp')
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column('unauthenticated_users', sa.Column('last_timestamp', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False))
+    op.add_column('authenticated_users', sa.Column('last_timestamp', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False))
+    # ### end Alembic commands ###

webapp/titanembeds/blueprints/admin/admin.py

@@ -3,6 +3,7 @@ from flask_socketio import emit
 from functools import wraps
 from titanembeds.database import db, get_administrators_list, Cosmetics, Guilds, UnauthenticatedUsers, UnauthenticatedBans, TitanTokens, TokenTransactions, get_titan_token, set_titan_token, list_disabled_guilds, DisabledGuilds, UserCSS, AuthenticatedUsers
 from titanembeds.oauth import generate_guild_icon_url
+from titanembeds.utils import get_online_embed_user_keys
 import datetime
 import json
 from sqlalchemy import func
@@ -22,9 +23,9 @@ def is_admin(f):
     return decorator(f)
 
 def get_online_users_count():
-    time_past = (datetime.datetime.now() - datetime.timedelta(seconds = 15)).strftime('%Y-%m-%d %H:%M:%S')
+    users = get_online_embed_user_keys()
-    unauths = db.session.query(func.count(UnauthenticatedUsers.id)).filter(UnauthenticatedUsers.last_timestamp > time_past, UnauthenticatedUsers.revoked == False).scalar()
+    auths = len(users["AuthenticatedUsers"])
-    auths = db.session.query(func.count(AuthenticatedUsers.id)).filter(AuthenticatedUsers.last_timestamp > time_past).scalar()
+    unauths = len(users["UnauthenticatedUsers"])
     return {"authenticated": auths, "guest": unauths, "total": auths + unauths}
 
 @admin.route("/")
@@ -114,14 +115,13 @@ def cosmetics_patch():
 def prepare_guild_members_list(members, bans):
     all_users = []
     ip_pool = []
-    members = sorted(members, key=lambda k: datetime.datetime.strptime(str(k.last_timestamp.replace(tzinfo=None, microsecond=0)), "%Y-%m-%d %H:%M:%S"), reverse=True)
+    members = sorted(members, key=lambda k: k.id, reverse=True)
     for member in members:
         user = {
             "id": member.id,
             "username": member.username,
             "discrim": member.discriminator,
             "ip": member.ip_address,
-            "last_visit": member.last_timestamp,
             "kicked": member.revoked,
             "banned": False,
             "banned_timestamp": None,
@@ -164,7 +164,7 @@ def administrate_guild(guild_id):
     permissions.append("Manage Embed Settings")
     permissions.append("Ban Members")
     permissions.append("Kick Members")
-    all_members = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.guild_id == guild_id).order_by(UnauthenticatedUsers.last_timestamp).all()
+    all_members = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.guild_id == guild_id).order_by(UnauthenticatedUsers.id).all()
     all_bans = db.session.query(UnauthenticatedBans).filter(UnauthenticatedBans.guild_id == guild_id).all()
     users = prepare_guild_members_list(all_members, all_bans)
     dbguild_dict = {

webapp/titanembeds/blueprints/api/api.py

@@ -1,6 +1,6 @@
 from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, Messages, get_channel_messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges
 from titanembeds.decorators import valid_session_required, discord_users_only, abort_if_guild_disabled
-from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles
+from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles, get_online_embed_user_keys
 from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild
 from flask import Blueprint, abort, jsonify, session, request, url_for
 from flask import current_app as app
@@ -111,9 +111,9 @@ def get_online_discord_users(guild_id, embed):
     return embed['members']
 
 def get_online_embed_users(guild_id):
-    time_past = (datetime.datetime.now() - datetime.timedelta(seconds = 15)).strftime('%Y-%m-%d %H:%M:%S')
+    usrs = get_online_embed_user_keys(guild_id)
-    unauths = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.last_timestamp > time_past, UnauthenticatedUsers.revoked == False, UnauthenticatedUsers.guild_id == guild_id).all()
+    unauths = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.user_key.in_(usrs["UnauthenticatedUsers"]), UnauthenticatedUsers.revoked == False, UnauthenticatedUsers.guild_id == guild_id).all() if usrs["UnauthenticatedUsers"] else []
-    auths = db.session.query(AuthenticatedUsers).filter(AuthenticatedUsers.last_timestamp > time_past, AuthenticatedUsers.guild_id == guild_id).all()
+    auths = db.session.query(AuthenticatedUsers).filter(AuthenticatedUsers.client_id.in_(usrs["AuthenticatedUsers"]), AuthenticatedUsers.guild_id == guild_id).all() if usrs["AuthenticatedUsers"] else []
     users = {'unauthenticated':[], 'authenticated':[]}
     for user in unauths:
         meta = {

webapp/titanembeds/blueprints/user/user.py

@@ -201,7 +201,7 @@ def administrate_guild(guild_id):
     if check_user_permission(guild_id, 1):
         permissions.append("Kick Members")
     cosmetics = db.session.query(Cosmetics).filter(Cosmetics.user_id == session['user_id']).first()
-    all_members = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.guild_id == guild_id).order_by(UnauthenticatedUsers.last_timestamp).all()
+    all_members = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.guild_id == guild_id).order_by(UnauthenticatedUsers.id).all()
     all_bans = db.session.query(UnauthenticatedBans).filter(UnauthenticatedBans.guild_id == guild_id).all()
     users = prepare_guild_members_list(all_members, all_bans)
     dbguild_dict = {
@@ -275,14 +275,13 @@ def add_bot(guild_id):
 def prepare_guild_members_list(members, bans):
     all_users = []
     ip_pool = []
-    members = sorted(members, key=lambda k: datetime.datetime.strptime(str(k.last_timestamp.replace(tzinfo=None, microsecond=0)), "%Y-%m-%d %H:%M:%S"), reverse=True)
+    members = sorted(members, key=lambda k: k.id, reverse=True)
     for member in members:
         user = {
             "id": member.id,
             "username": member.username,
             "discrim": member.discriminator,
             "ip": member.ip_address,
-            "last_visit": member.last_timestamp,
             "kicked": member.revoked,
             "banned": False,
             "banned_timestamp": None,

webapp/titanembeds/database/authenticated_users.py

@@ -7,14 +7,7 @@ class AuthenticatedUsers(db.Model):
     id = db.Column(db.Integer, primary_key=True)                    # Auto increment id
     guild_id = db.Column(db.String(255), nullable=False)            # Guild pretaining to the authenticated user
     client_id = db.Column(db.String(255), nullable=False)           # Client ID of the authenticated user
-    last_timestamp = db.Column(db.TIMESTAMP, nullable=False)        # The timestamp of when the user has last sent the heartbeat
 
     def __init__(self, guild_id, client_id):
         self.guild_id = guild_id
         self.client_id = client_id
-        self.last_timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
-
-    def bumpTimestamp(self):
-        self.last_timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
-        db.session.commit()
-        return self.last_timestamp

webapp/titanembeds/database/unauthenticated_users.py

@@ -12,7 +12,6 @@ class UnauthenticatedUsers(db.Model):
     discriminator = db.Column(db.Integer, nullable=False)           # The discriminator to distinguish unauth users with each other
     user_key = db.Column(db.Text(), nullable=False)                 # The secret key used to identify the user holder
     ip_address = db.Column(db.String(255), nullable=False)          # The IP Address of the user
-    last_timestamp = db.Column(db.TIMESTAMP, nullable=False)        # The timestamp of when the user has last sent the heartbeat
     revoked = db.Column(db.Boolean(), nullable=False)               # If the user's key has been revoked and a new one is required to be generated
 
     def __init__(self, guild_id, username, discriminator, ip_address):
@@ -21,11 +20,10 @@ class UnauthenticatedUsers(db.Model):
         self.discriminator = discriminator
         self.user_key = "".join(random.choice(string.ascii_letters) for _ in range(0, 32))
         self.ip_address = ip_address
-        self.last_timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
         self.revoked = False
 
     def __repr__(self):
-        return '<UnauthenticatedUsers {0} {1} {2} {3} {4} {5} {6} {7}>'.format(self.id, self.guild_id, self.username, self.discriminator, self.user_key, self.ip_address, self.last_timestamp, self.revoked)
+        return '<UnauthenticatedUsers {0} {1} {2} {3} {4} {5} {6}>'.format(self.id, self.guild_id, self.username, self.discriminator, self.user_key, self.ip_address, self.revoked)
 
     def isRevoked(self):
         return self.revoked
@@ -39,8 +37,3 @@ class UnauthenticatedUsers(db.Model):
         self.revoked = True
         db.session.commit()
         return self.revoked
-
-    def bumpTimestamp(self):
-        self.last_timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
-        db.session.commit()
-        return self.last_timestamp

webapp/titanembeds/templates/administrate_guild.html.j2

@@ -171,7 +171,6 @@
                         <th>Ban User</th>
                         <th>Username</th>
                         <th>Discrim</th>
-                        <th>Last Visit</th>
                         <th>IP Address Hash</th>
                         <th>Banned Timestamp</th>
                         <th>Banned by</th>
@@ -191,7 +190,6 @@
                         {% endif %}
                         <td>{{ member['username'] }}</td>
                         <td>{{ member['discrim'] }}</td>
-                        <td>{{ member['last_visit'] }}</td>
                         <td>{{ member['ip'] }}</td>
                         <td>{{ member['banned_timestamp'] }}</td>
                         <td>{{ member['banned_by'] }}</td>

webapp/titanembeds/utils.py

@@ -138,7 +138,7 @@ def update_user_status(guild_id, username, user_key=None):
             session['user_keys'].pop(guild_id, None)
             return status
         dbUser = UnauthenticatedUsers.query.filter(and_(UnauthenticatedUsers.guild_id == guild_id, UnauthenticatedUsers.user_key == user_key)).first()
-        dbUser.bumpTimestamp()
+        bump_user_presence_timestamp(guild_id, "UnauthenticatedUsers", user_key)
         if dbUser.username != username or dbUser.ip_address != ip_address:
             dbUser.username = username
             dbUser.ip_address = ip_address
@@ -161,10 +161,28 @@ def update_user_status(guild_id, username, user_key=None):
         dbMember = get_guild_member(guild_id, status["user_id"])
         if dbMember:
             status["nickname"] = dbMember.nickname
-        dbUser = db.session.query(AuthenticatedUsers).filter(and_(AuthenticatedUsers.guild_id == guild_id, AuthenticatedUsers.client_id == status['user_id'])).first()
+        bump_user_presence_timestamp(guild_id, "AuthenticatedUsers", status["user_id"])
-        dbUser.bumpTimestamp()
     return status
 
+def bump_user_presence_timestamp(guild_id, user_type, client_key):
+    redis_key = "MemberPresence/{}/{}/{}".format(guild_id, user_type, client_key)
+    redis_store.set(redis_key, "", 15)
+
+def get_online_embed_user_keys(guild_id="*", user_type=None):
+    if not user_type:
+        user_type = ["AuthenticatedUsers", "UnauthenticatedUsers"]
+    else:
+        user_type = [user_type]
+    usrs = {}
+    for utype in user_type:
+        usrs[utype] = []
+        keys = redis_store.keys("MemberPresence/{}/{}/*".format(guild_id, utype))
+        for key in keys:
+            key = str(key, "utf-8")
+            client_key = key.split("/")[-1]
+            usrs[utype].append(client_key)
+    return usrs
+
 def check_user_in_guild(guild_id):
     if user_unauthenticated():
         return guild_id in session.get("user_keys", {})