Yadciel/Titan
1
0
Fork 0
mirror of https://github.com/TitanEmbeds/Titan.git synced 2024-12-24 14:07:03 +01:00
Code Issues Packages Projects Releases Wiki Activity

Move tracking online users to redis

Browse Source
This commit is contained in:
Jeremy Zhang 2018-01-21 03:14:25 +00:00
parent 6b1c6179d1
commit 2c949d911b
9 changed files with 66 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
discordbot/titanembeds/database/unauthenticated_users.py
View File

@ -12,8 +12,7 @@ class UnauthenticatedUsers(Base):
discriminator = db.Column(db.Integer) # The discriminator to distinguish unauth users with each other
user_key = db.Column(db.Text()) # The secret key used to identify the user holder
ip_address = db.Column(db.String(255)) # The IP Address of the user
last_timestamp = db.Column(db.TIMESTAMP) # The timestamp of when the user has last sent the heartbeat
revoked = db.Column(db.Boolean()) # If the user's key has been revoked and a new one is required to be generated
def __repr__(self):
return '<UnauthenticatedUsers {0} {1} {2} {3} {4} {5} {6} {7}>'.format(self.id, self.guild_id, self.username, self.discriminator, self.user_key, self.ip_address, self.last_timestamp, self.revoked)
return '<UnauthenticatedUsers {0} {1} {2} {3} {4} {5} {6}>'.format(self.id, self.guild_id, self.username, self.discriminator, self.user_key, self.ip_address, self.revoked)

30
webapp/alembic/versions/5642232899cf_removed_last_timestamp_columns.py Normal file
View File

@ -0,0 +1,30 @@
"""Removed last timestamp columns
Revision ID: 5642232899cf
Revises: 0b7761d85555
Create Date: 2018-01-21 03:03:32.359959
"""
# revision identifiers, used by Alembic.
revision = '5642232899cf'
down_revision = '0b7761d85555'
branch_labels = None
depends_on = None
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql
def upgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.drop_column('authenticated_users', 'last_timestamp')
op.drop_column('unauthenticated_users', 'last_timestamp')
# ### end Alembic commands ###
def downgrade():
# ### commands auto generated by Alembic - please adjust! ###
op.add_column('unauthenticated_users', sa.Column('last_timestamp', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False))
op.add_column('authenticated_users', sa.Column('last_timestamp', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False))
# ### end Alembic commands ###

12
webapp/titanembeds/blueprints/admin/admin.py
View File

@ -3,6 +3,7 @@ from flask_socketio import emit
from functools import wraps
from titanembeds.database import db, get_administrators_list, Cosmetics, Guilds, UnauthenticatedUsers, UnauthenticatedBans, TitanTokens, TokenTransactions, get_titan_token, set_titan_token, list_disabled_guilds, DisabledGuilds, UserCSS, AuthenticatedUsers
from titanembeds.oauth import generate_guild_icon_url
from titanembeds.utils import get_online_embed_user_keys
import datetime
import json
from sqlalchemy import func
@ -22,9 +23,9 @@ def is_admin(f):
return decorator(f)
def get_online_users_count():
time_past = (datetime.datetime.now() - datetime.timedelta(seconds = 15)).strftime('%Y-%m-%d %H:%M:%S')
unauths = db.session.query(func.count(UnauthenticatedUsers.id)).filter(UnauthenticatedUsers.last_timestamp > time_past, UnauthenticatedUsers.revoked == False).scalar()
auths = db.session.query(func.count(AuthenticatedUsers.id)).filter(AuthenticatedUsers.last_timestamp > time_past).scalar()
users = get_online_embed_user_keys()
auths = len(users["AuthenticatedUsers"])
unauths = len(users["UnauthenticatedUsers"])
return {"authenticated": auths, "guest": unauths, "total": auths + unauths}
@admin.route("/")
@ -114,14 +115,13 @@ def cosmetics_patch():
def prepare_guild_members_list(members, bans):
all_users = []
ip_pool = []
members = sorted(members, key=lambda k: datetime.datetime.strptime(str(k.last_timestamp.replace(tzinfo=None, microsecond=0)), "%Y-%m-%d %H:%M:%S"), reverse=True)
members = sorted(members, key=lambda k: k.id, reverse=True)
for member in members:
user = {
"id": member.id,
"username": member.username,
"discrim": member.discriminator,
"ip": member.ip_address,
"last_visit": member.last_timestamp,
"kicked": member.revoked,
"banned": False,
"banned_timestamp": None,
@ -164,7 +164,7 @@ def administrate_guild(guild_id):
permissions.append("Manage Embed Settings")
permissions.append("Ban Members")
permissions.append("Kick Members")
all_members = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.guild_id == guild_id).order_by(UnauthenticatedUsers.last_timestamp).all()
all_members = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.guild_id == guild_id).order_by(UnauthenticatedUsers.id).all()
all_bans = db.session.query(UnauthenticatedBans).filter(UnauthenticatedBans.guild_id == guild_id).all()
users = prepare_guild_members_list(all_members, all_bans)
dbguild_dict = {

8
webapp/titanembeds/blueprints/api/api.py
View File

@ -1,6 +1,6 @@
from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, Messages, get_channel_messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges
from titanembeds.decorators import valid_session_required, discord_users_only, abort_if_guild_disabled
from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles
from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles, get_online_embed_user_keys
from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild
from flask import Blueprint, abort, jsonify, session, request, url_for
from flask import current_app as app
@ -111,9 +111,9 @@ def get_online_discord_users(guild_id, embed):
return embed['members']
def get_online_embed_users(guild_id):
time_past = (datetime.datetime.now() - datetime.timedelta(seconds = 15)).strftime('%Y-%m-%d %H:%M:%S')
unauths = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.last_timestamp > time_past, UnauthenticatedUsers.revoked == False, UnauthenticatedUsers.guild_id == guild_id).all()
auths = db.session.query(AuthenticatedUsers).filter(AuthenticatedUsers.last_timestamp > time_past, AuthenticatedUsers.guild_id == guild_id).all()
usrs = get_online_embed_user_keys(guild_id)
unauths = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.user_key.in_(usrs["UnauthenticatedUsers"]), UnauthenticatedUsers.revoked == False, UnauthenticatedUsers.guild_id == guild_id).all() if usrs["UnauthenticatedUsers"] else []
auths = db.session.query(AuthenticatedUsers).filter(AuthenticatedUsers.client_id.in_(usrs["AuthenticatedUsers"]), AuthenticatedUsers.guild_id == guild_id).all() if usrs["AuthenticatedUsers"] else []
users = {'unauthenticated':[], 'authenticated':[]}
for user in unauths:
meta = {

5
webapp/titanembeds/blueprints/user/user.py
View File

@ -201,7 +201,7 @@ def administrate_guild(guild_id):
if check_user_permission(guild_id, 1):
permissions.append("Kick Members")
cosmetics = db.session.query(Cosmetics).filter(Cosmetics.user_id == session['user_id']).first()
all_members = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.guild_id == guild_id).order_by(UnauthenticatedUsers.last_timestamp).all()
all_members = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.guild_id == guild_id).order_by(UnauthenticatedUsers.id).all()
all_bans = db.session.query(UnauthenticatedBans).filter(UnauthenticatedBans.guild_id == guild_id).all()
users = prepare_guild_members_list(all_members, all_bans)
dbguild_dict = {
@ -275,14 +275,13 @@ def add_bot(guild_id):
def prepare_guild_members_list(members, bans):
all_users = []
ip_pool = []
members = sorted(members, key=lambda k: datetime.datetime.strptime(str(k.last_timestamp.replace(tzinfo=None, microsecond=0)), "%Y-%m-%d %H:%M:%S"), reverse=True)
members = sorted(members, key=lambda k: k.id, reverse=True)
for member in members:
user = {
"id": member.id,
"username": member.username,
"discrim": member.discriminator,
"ip": member.ip_address,
"last_visit": member.last_timestamp,
"kicked": member.revoked,
"banned": False,
"banned_timestamp": None,

7
webapp/titanembeds/database/authenticated_users.py
View File

@ -7,14 +7,7 @@ class AuthenticatedUsers(db.Model):
id = db.Column(db.Integer, primary_key=True) # Auto increment id
guild_id = db.Column(db.String(255), nullable=False) # Guild pretaining to the authenticated user
client_id = db.Column(db.String(255), nullable=False) # Client ID of the authenticated user
last_timestamp = db.Column(db.TIMESTAMP, nullable=False) # The timestamp of when the user has last sent the heartbeat
def __init__(self, guild_id, client_id):
self.guild_id = guild_id
self.client_id = client_id
self.last_timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
def bumpTimestamp(self):
self.last_timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
db.session.commit()
return self.last_timestamp

9
webapp/titanembeds/database/unauthenticated_users.py
View File

@ -12,7 +12,6 @@ class UnauthenticatedUsers(db.Model):
discriminator = db.Column(db.Integer, nullable=False) # The discriminator to distinguish unauth users with each other
user_key = db.Column(db.Text(), nullable=False) # The secret key used to identify the user holder
ip_address = db.Column(db.String(255), nullable=False) # The IP Address of the user
last_timestamp = db.Column(db.TIMESTAMP, nullable=False) # The timestamp of when the user has last sent the heartbeat
revoked = db.Column(db.Boolean(), nullable=False) # If the user's key has been revoked and a new one is required to be generated
def __init__(self, guild_id, username, discriminator, ip_address):
@ -21,11 +20,10 @@ class UnauthenticatedUsers(db.Model):
self.discriminator = discriminator
self.user_key = "".join(random.choice(string.ascii_letters) for _ in range(0, 32))
self.ip_address = ip_address
self.last_timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
self.revoked = False
def __repr__(self):
return '<UnauthenticatedUsers {0} {1} {2} {3} {4} {5} {6} {7}>'.format(self.id, self.guild_id, self.username, self.discriminator, self.user_key, self.ip_address, self.last_timestamp, self.revoked)
return '<UnauthenticatedUsers {0} {1} {2} {3} {4} {5} {6}>'.format(self.id, self.guild_id, self.username, self.discriminator, self.user_key, self.ip_address, self.revoked)
def isRevoked(self):
return self.revoked
@ -39,8 +37,3 @@ class UnauthenticatedUsers(db.Model):
self.revoked = True
db.session.commit()
return self.revoked
def bumpTimestamp(self):
self.last_timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S')
db.session.commit()
return self.last_timestamp

2
webapp/titanembeds/templates/administrate_guild.html.j2
View File

@ -171,7 +171,6 @@
<th>Ban User</th>
<th>Username</th>
<th>Discrim</th>
<th>Last Visit</th>
<th>IP Address Hash</th>
<th>Banned Timestamp</th>
<th>Banned by</th>
@ -191,7 +190,6 @@
{% endif %}
<td>{{ member['username'] }}</td>
<td>{{ member['discrim'] }}</td>
<td>{{ member['last_visit'] }}</td>
<td>{{ member['ip'] }}</td>
<td>{{ member['banned_timestamp'] }}</td>
<td>{{ member['banned_by'] }}</td>

24
webapp/titanembeds/utils.py
View File

@ -138,7 +138,7 @@ def update_user_status(guild_id, username, user_key=None):
session['user_keys'].pop(guild_id, None)
return status
dbUser = UnauthenticatedUsers.query.filter(and_(UnauthenticatedUsers.guild_id == guild_id, UnauthenticatedUsers.user_key == user_key)).first()
dbUser.bumpTimestamp()
bump_user_presence_timestamp(guild_id, "UnauthenticatedUsers", user_key)
if dbUser.username != username or dbUser.ip_address != ip_address:
dbUser.username = username
dbUser.ip_address = ip_address
@ -161,10 +161,28 @@ def update_user_status(guild_id, username, user_key=None):
dbMember = get_guild_member(guild_id, status["user_id"])
if dbMember:
status["nickname"] = dbMember.nickname
dbUser = db.session.query(AuthenticatedUsers).filter(and_(AuthenticatedUsers.guild_id == guild_id, AuthenticatedUsers.client_id == status['user_id'])).first()
dbUser.bumpTimestamp()
bump_user_presence_timestamp(guild_id, "AuthenticatedUsers", status["user_id"])
return status
def bump_user_presence_timestamp(guild_id, user_type, client_key):
redis_key = "MemberPresence/{}/{}/{}".format(guild_id, user_type, client_key)
redis_store.set(redis_key, "", 15)
def get_online_embed_user_keys(guild_id="*", user_type=None):
if not user_type:
user_type = ["AuthenticatedUsers", "UnauthenticatedUsers"]
else:
user_type = [user_type]
usrs = {}
for utype in user_type:
usrs[utype] = []
keys = redis_store.keys("MemberPresence/{}/{}/*".format(guild_id, utype))
for key in keys:
key = str(key, "utf-8")
client_key = key.split("/")[-1]
usrs[utype].append(client_key)
return usrs
def check_user_in_guild(guild_id):
if user_unauthenticated():
return guild_id in session.get("user_keys", {})