diff --git a/discordbot/titanembeds/database/unauthenticated_users.py b/discordbot/titanembeds/database/unauthenticated_users.py index 6abb1d7..931b118 100644 --- a/discordbot/titanembeds/database/unauthenticated_users.py +++ b/discordbot/titanembeds/database/unauthenticated_users.py @@ -12,8 +12,7 @@ class UnauthenticatedUsers(Base): discriminator = db.Column(db.Integer) # The discriminator to distinguish unauth users with each other user_key = db.Column(db.Text()) # The secret key used to identify the user holder ip_address = db.Column(db.String(255)) # The IP Address of the user - last_timestamp = db.Column(db.TIMESTAMP) # The timestamp of when the user has last sent the heartbeat revoked = db.Column(db.Boolean()) # If the user's key has been revoked and a new one is required to be generated def __repr__(self): - return ''.format(self.id, self.guild_id, self.username, self.discriminator, self.user_key, self.ip_address, self.last_timestamp, self.revoked) + return ''.format(self.id, self.guild_id, self.username, self.discriminator, self.user_key, self.ip_address, self.revoked) diff --git a/webapp/alembic/versions/5642232899cf_removed_last_timestamp_columns.py b/webapp/alembic/versions/5642232899cf_removed_last_timestamp_columns.py new file mode 100644 index 0000000..7ea45cb --- /dev/null +++ b/webapp/alembic/versions/5642232899cf_removed_last_timestamp_columns.py @@ -0,0 +1,30 @@ +"""Removed last timestamp columns + +Revision ID: 5642232899cf +Revises: 0b7761d85555 +Create Date: 2018-01-21 03:03:32.359959 + +""" + +# revision identifiers, used by Alembic. +revision = '5642232899cf' +down_revision = '0b7761d85555' +branch_labels = None +depends_on = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column('authenticated_users', 'last_timestamp') + op.drop_column('unauthenticated_users', 'last_timestamp') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column('unauthenticated_users', sa.Column('last_timestamp', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False)) + op.add_column('authenticated_users', sa.Column('last_timestamp', postgresql.TIMESTAMP(), server_default=sa.text('now()'), autoincrement=False, nullable=False)) + # ### end Alembic commands ### diff --git a/webapp/titanembeds/blueprints/admin/admin.py b/webapp/titanembeds/blueprints/admin/admin.py index e30b30b..ad01f01 100644 --- a/webapp/titanembeds/blueprints/admin/admin.py +++ b/webapp/titanembeds/blueprints/admin/admin.py @@ -3,6 +3,7 @@ from flask_socketio import emit from functools import wraps from titanembeds.database import db, get_administrators_list, Cosmetics, Guilds, UnauthenticatedUsers, UnauthenticatedBans, TitanTokens, TokenTransactions, get_titan_token, set_titan_token, list_disabled_guilds, DisabledGuilds, UserCSS, AuthenticatedUsers from titanembeds.oauth import generate_guild_icon_url +from titanembeds.utils import get_online_embed_user_keys import datetime import json from sqlalchemy import func @@ -22,9 +23,9 @@ def is_admin(f): return decorator(f) def get_online_users_count(): - time_past = (datetime.datetime.now() - datetime.timedelta(seconds = 15)).strftime('%Y-%m-%d %H:%M:%S') - unauths = db.session.query(func.count(UnauthenticatedUsers.id)).filter(UnauthenticatedUsers.last_timestamp > time_past, UnauthenticatedUsers.revoked == False).scalar() - auths = db.session.query(func.count(AuthenticatedUsers.id)).filter(AuthenticatedUsers.last_timestamp > time_past).scalar() + users = get_online_embed_user_keys() + auths = len(users["AuthenticatedUsers"]) + unauths = len(users["UnauthenticatedUsers"]) return {"authenticated": auths, "guest": unauths, "total": auths + unauths} @admin.route("/") @@ -114,14 +115,13 @@ def cosmetics_patch(): def prepare_guild_members_list(members, bans): all_users = [] ip_pool = [] - members = sorted(members, key=lambda k: datetime.datetime.strptime(str(k.last_timestamp.replace(tzinfo=None, microsecond=0)), "%Y-%m-%d %H:%M:%S"), reverse=True) + members = sorted(members, key=lambda k: k.id, reverse=True) for member in members: user = { "id": member.id, "username": member.username, "discrim": member.discriminator, "ip": member.ip_address, - "last_visit": member.last_timestamp, "kicked": member.revoked, "banned": False, "banned_timestamp": None, @@ -164,7 +164,7 @@ def administrate_guild(guild_id): permissions.append("Manage Embed Settings") permissions.append("Ban Members") permissions.append("Kick Members") - all_members = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.guild_id == guild_id).order_by(UnauthenticatedUsers.last_timestamp).all() + all_members = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.guild_id == guild_id).order_by(UnauthenticatedUsers.id).all() all_bans = db.session.query(UnauthenticatedBans).filter(UnauthenticatedBans.guild_id == guild_id).all() users = prepare_guild_members_list(all_members, all_bans) dbguild_dict = { diff --git a/webapp/titanembeds/blueprints/api/api.py b/webapp/titanembeds/blueprints/api/api.py index 23fbca1..adaf415 100644 --- a/webapp/titanembeds/blueprints/api/api.py +++ b/webapp/titanembeds/blueprints/api/api.py @@ -1,6 +1,6 @@ from titanembeds.database import db, Guilds, UnauthenticatedUsers, UnauthenticatedBans, AuthenticatedUsers, GuildMembers, Messages, get_channel_messages, list_all_guild_members, get_guild_member, get_administrators_list, get_badges from titanembeds.decorators import valid_session_required, discord_users_only, abort_if_guild_disabled -from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles +from titanembeds.utils import check_guild_existance, guild_accepts_visitors, guild_query_unauth_users_bool, get_client_ipaddr, discord_api, rate_limiter, channel_ratelimit_key, guild_ratelimit_key, user_unauthenticated, checkUserRevoke, checkUserBanned, update_user_status, check_user_in_guild, get_guild_channels, guild_webhooks_enabled, guild_unauthcaptcha_enabled, get_member_roles, get_online_embed_user_keys from titanembeds.oauth import user_has_permission, generate_avatar_url, check_user_can_administrate_guild from flask import Blueprint, abort, jsonify, session, request, url_for from flask import current_app as app @@ -111,9 +111,9 @@ def get_online_discord_users(guild_id, embed): return embed['members'] def get_online_embed_users(guild_id): - time_past = (datetime.datetime.now() - datetime.timedelta(seconds = 15)).strftime('%Y-%m-%d %H:%M:%S') - unauths = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.last_timestamp > time_past, UnauthenticatedUsers.revoked == False, UnauthenticatedUsers.guild_id == guild_id).all() - auths = db.session.query(AuthenticatedUsers).filter(AuthenticatedUsers.last_timestamp > time_past, AuthenticatedUsers.guild_id == guild_id).all() + usrs = get_online_embed_user_keys(guild_id) + unauths = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.user_key.in_(usrs["UnauthenticatedUsers"]), UnauthenticatedUsers.revoked == False, UnauthenticatedUsers.guild_id == guild_id).all() if usrs["UnauthenticatedUsers"] else [] + auths = db.session.query(AuthenticatedUsers).filter(AuthenticatedUsers.client_id.in_(usrs["AuthenticatedUsers"]), AuthenticatedUsers.guild_id == guild_id).all() if usrs["AuthenticatedUsers"] else [] users = {'unauthenticated':[], 'authenticated':[]} for user in unauths: meta = { diff --git a/webapp/titanembeds/blueprints/user/user.py b/webapp/titanembeds/blueprints/user/user.py index f5ef15f..3e6f198 100644 --- a/webapp/titanembeds/blueprints/user/user.py +++ b/webapp/titanembeds/blueprints/user/user.py @@ -201,7 +201,7 @@ def administrate_guild(guild_id): if check_user_permission(guild_id, 1): permissions.append("Kick Members") cosmetics = db.session.query(Cosmetics).filter(Cosmetics.user_id == session['user_id']).first() - all_members = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.guild_id == guild_id).order_by(UnauthenticatedUsers.last_timestamp).all() + all_members = db.session.query(UnauthenticatedUsers).filter(UnauthenticatedUsers.guild_id == guild_id).order_by(UnauthenticatedUsers.id).all() all_bans = db.session.query(UnauthenticatedBans).filter(UnauthenticatedBans.guild_id == guild_id).all() users = prepare_guild_members_list(all_members, all_bans) dbguild_dict = { @@ -275,14 +275,13 @@ def add_bot(guild_id): def prepare_guild_members_list(members, bans): all_users = [] ip_pool = [] - members = sorted(members, key=lambda k: datetime.datetime.strptime(str(k.last_timestamp.replace(tzinfo=None, microsecond=0)), "%Y-%m-%d %H:%M:%S"), reverse=True) + members = sorted(members, key=lambda k: k.id, reverse=True) for member in members: user = { "id": member.id, "username": member.username, "discrim": member.discriminator, "ip": member.ip_address, - "last_visit": member.last_timestamp, "kicked": member.revoked, "banned": False, "banned_timestamp": None, diff --git a/webapp/titanembeds/database/authenticated_users.py b/webapp/titanembeds/database/authenticated_users.py index fd9f923..01b0df3 100644 --- a/webapp/titanembeds/database/authenticated_users.py +++ b/webapp/titanembeds/database/authenticated_users.py @@ -7,14 +7,7 @@ class AuthenticatedUsers(db.Model): id = db.Column(db.Integer, primary_key=True) # Auto increment id guild_id = db.Column(db.String(255), nullable=False) # Guild pretaining to the authenticated user client_id = db.Column(db.String(255), nullable=False) # Client ID of the authenticated user - last_timestamp = db.Column(db.TIMESTAMP, nullable=False) # The timestamp of when the user has last sent the heartbeat def __init__(self, guild_id, client_id): self.guild_id = guild_id - self.client_id = client_id - self.last_timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') - - def bumpTimestamp(self): - self.last_timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') - db.session.commit() - return self.last_timestamp + self.client_id = client_id \ No newline at end of file diff --git a/webapp/titanembeds/database/unauthenticated_users.py b/webapp/titanembeds/database/unauthenticated_users.py index ed4867b..75470a3 100644 --- a/webapp/titanembeds/database/unauthenticated_users.py +++ b/webapp/titanembeds/database/unauthenticated_users.py @@ -12,7 +12,6 @@ class UnauthenticatedUsers(db.Model): discriminator = db.Column(db.Integer, nullable=False) # The discriminator to distinguish unauth users with each other user_key = db.Column(db.Text(), nullable=False) # The secret key used to identify the user holder ip_address = db.Column(db.String(255), nullable=False) # The IP Address of the user - last_timestamp = db.Column(db.TIMESTAMP, nullable=False) # The timestamp of when the user has last sent the heartbeat revoked = db.Column(db.Boolean(), nullable=False) # If the user's key has been revoked and a new one is required to be generated def __init__(self, guild_id, username, discriminator, ip_address): @@ -21,11 +20,10 @@ class UnauthenticatedUsers(db.Model): self.discriminator = discriminator self.user_key = "".join(random.choice(string.ascii_letters) for _ in range(0, 32)) self.ip_address = ip_address - self.last_timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') self.revoked = False def __repr__(self): - return ''.format(self.id, self.guild_id, self.username, self.discriminator, self.user_key, self.ip_address, self.last_timestamp, self.revoked) + return ''.format(self.id, self.guild_id, self.username, self.discriminator, self.user_key, self.ip_address, self.revoked) def isRevoked(self): return self.revoked @@ -39,8 +37,3 @@ class UnauthenticatedUsers(db.Model): self.revoked = True db.session.commit() return self.revoked - - def bumpTimestamp(self): - self.last_timestamp = datetime.datetime.fromtimestamp(time.time()).strftime('%Y-%m-%d %H:%M:%S') - db.session.commit() - return self.last_timestamp diff --git a/webapp/titanembeds/templates/administrate_guild.html.j2 b/webapp/titanembeds/templates/administrate_guild.html.j2 index bf972bf..3ffe2cf 100644 --- a/webapp/titanembeds/templates/administrate_guild.html.j2 +++ b/webapp/titanembeds/templates/administrate_guild.html.j2 @@ -171,7 +171,6 @@ Ban User Username Discrim - Last Visit IP Address Hash Banned Timestamp Banned by @@ -191,7 +190,6 @@ {% endif %} {{ member['username'] }} {{ member['discrim'] }} - {{ member['last_visit'] }} {{ member['ip'] }} {{ member['banned_timestamp'] }} {{ member['banned_by'] }} diff --git a/webapp/titanembeds/utils.py b/webapp/titanembeds/utils.py index 39e0ff1..fd1bb9c 100644 --- a/webapp/titanembeds/utils.py +++ b/webapp/titanembeds/utils.py @@ -138,7 +138,7 @@ def update_user_status(guild_id, username, user_key=None): session['user_keys'].pop(guild_id, None) return status dbUser = UnauthenticatedUsers.query.filter(and_(UnauthenticatedUsers.guild_id == guild_id, UnauthenticatedUsers.user_key == user_key)).first() - dbUser.bumpTimestamp() + bump_user_presence_timestamp(guild_id, "UnauthenticatedUsers", user_key) if dbUser.username != username or dbUser.ip_address != ip_address: dbUser.username = username dbUser.ip_address = ip_address @@ -161,10 +161,28 @@ def update_user_status(guild_id, username, user_key=None): dbMember = get_guild_member(guild_id, status["user_id"]) if dbMember: status["nickname"] = dbMember.nickname - dbUser = db.session.query(AuthenticatedUsers).filter(and_(AuthenticatedUsers.guild_id == guild_id, AuthenticatedUsers.client_id == status['user_id'])).first() - dbUser.bumpTimestamp() + bump_user_presence_timestamp(guild_id, "AuthenticatedUsers", status["user_id"]) return status +def bump_user_presence_timestamp(guild_id, user_type, client_key): + redis_key = "MemberPresence/{}/{}/{}".format(guild_id, user_type, client_key) + redis_store.set(redis_key, "", 15) + +def get_online_embed_user_keys(guild_id="*", user_type=None): + if not user_type: + user_type = ["AuthenticatedUsers", "UnauthenticatedUsers"] + else: + user_type = [user_type] + usrs = {} + for utype in user_type: + usrs[utype] = [] + keys = redis_store.keys("MemberPresence/{}/{}/*".format(guild_id, utype)) + for key in keys: + key = str(key, "utf-8") + client_key = key.split("/")[-1] + usrs[utype].append(client_key) + return usrs + def check_user_in_guild(guild_id): if user_unauthenticated(): return guild_id in session.get("user_keys", {})