mirror of
https://github.com/TitanEmbeds/Titan.git
synced 2025-06-16 11:25:22 +02:00
Use redis to store discord api related objects, so far messages only
Keeping the table for now in case the redis method does not work as hoped
This commit is contained in:
Jeremy Zhang
@ -1,5 +1,6 @@
|
||||
from config import config
|
||||
from titanembeds.database import DatabaseInterface
|
||||
from titanembeds.redisqueue import RedisQueue
|
||||
from titanembeds.commands import Commands
|
||||
from titanembeds.socketio import SocketIOInterface
|
||||
from titanembeds.poststats import DiscordBotsOrg, BotsDiscordPw
|
||||
@ -21,6 +22,7 @@ class Titan(discord.AutoShardedClient):
|
||||
self.aiosession = aiohttp.ClientSession(loop=self.loop)
|
||||
self.http.user_agent += ' TitanEmbeds-Bot'
|
||||
self.database = DatabaseInterface(self)
|
||||
self.redisqueue = RedisQueue(self, config["redis-uri"])
|
||||
self.command = Commands(self, self.database)
|
||||
self.socketio = SocketIOInterface(self, config["redis-uri"])
|
||||
|
||||
@ -57,6 +59,8 @@ class Titan(discord.AutoShardedClient):
|
||||
|
||||
async def start(self):
|
||||
await self.database.connect(config["database-uri"])
|
||||
await self.redisqueue.connect()
|
||||
self.loop.create_task(self.redisqueue.subscribe())
|
||||
await super().start(config["bot-token"])
|
||||
|
||||
async def on_ready(self):
|
||||
@ -78,6 +82,7 @@ class Titan(discord.AutoShardedClient):
|
||||
async def on_message(self, message):
|
||||
await self.socketio.on_message(message)
|
||||
await self.database.push_message(message)
|
||||
await self.redisqueue.push_message(message)
|
||||
|
||||
msg_arr = message.content.split() # split the message
|
||||
if len(message.content.split()) > 1 and message.guild: #making sure there is actually stuff in the message and have arguments and check if it is sent in server (not PM)
|
||||
@ -92,11 +97,13 @@ class Titan(discord.AutoShardedClient):
|
||||
|
||||
async def on_message_edit(self, message_before, message_after):
|
||||
await self.database.update_message(message_after)
|
||||
await self.redisqueue.update_message(message_after)
|
||||
await self.socketio.on_message_update(message_after)
|
||||
|
||||
async def on_message_delete(self, message):
|
||||
self.delete_list.append(message.id)
|
||||
await self.database.delete_message(message)
|
||||
await self.redisqueue.delete_message(message)
|
||||
await self.socketio.on_message_delete(message)
|
||||
|
||||
async def on_guild_join(self, guild):
|
||||
|
||||
110
discordbot/titanembeds/redisqueue.py
Normal file
110
discordbot/titanembeds/redisqueue.py
Normal file
@ -0,0 +1,110 @@
|
||||
from titanembeds.utils import get_formatted_message
|
||||
from urllib.parse import urlparse
|
||||
import asyncio_redis
|
||||
import json
|
||||
import discord
|
||||
import asyncio
|
||||
import traceback
|
||||
import sys
|
||||
import re
|
||||
|
||||
class RedisQueue:
|
||||
def __init__(self, bot, redis_uri):
|
||||
self.bot = bot
|
||||
self.redis_uri = redis_uri
|
||||
|
||||
async def connect(self):
|
||||
url_parsed = urlparse(self.redis_uri)
|
||||
url_path = 0
|
||||
if url_parsed.path and len(url_parsed.path) > 2:
|
||||
url_path = int(url_parsed.path[1:])
|
||||
self.sub_connection = await asyncio_redis.Connection.create(
|
||||
host = url_parsed.hostname or "localhost",
|
||||
port = url_parsed.port or 6379,
|
||||
password = url_parsed.password,
|
||||
db = url_path
|
||||
)
|
||||
self.connection = await asyncio_redis.Pool.create(
|
||||
host = url_parsed.hostname or "localhost",
|
||||
port = url_parsed.port or 6379,
|
||||
password = url_parsed.password,
|
||||
db = url_path,
|
||||
poolsize = 10
|
||||
)
|
||||
|
||||
async def subscribe(self):
|
||||
await self.bot.wait_until_ready()
|
||||
subscriber = await self.sub_connection.start_subscribe()
|
||||
await subscriber.subscribe(["discord-api-req"])
|
||||
while True:
|
||||
reply = await subscriber.next_published()
|
||||
request = json.loads(reply.value)
|
||||
resource = request["resource"]
|
||||
self.dispatch(resource, request["key"], request["params"])
|
||||
|
||||
def dispatch(self, event, key, params):
|
||||
method = "on_" + event
|
||||
if hasattr(self, method):
|
||||
self.bot.loop.create_task(self._run_event(method, key, params))
|
||||
|
||||
async def _run_event(self, event, key, params):
|
||||
try:
|
||||
await getattr(self, event)(key, params)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except Exception:
|
||||
try:
|
||||
await self.on_error(event)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
async def on_error(self, event_method):
|
||||
print('Ignoring exception in {}'.format(event_method), file=sys.stderr)
|
||||
traceback.print_exc()
|
||||
|
||||
async def set_scan_json(self, key, dict_key, dict_value_pattern):
|
||||
unformatted_item = None
|
||||
formatted_item = None
|
||||
exists = await self.connection.exists(key)
|
||||
if exists:
|
||||
members = await self.connection.smembers(key)
|
||||
for member in members:
|
||||
the_member = await member
|
||||
parsed = json.loads(the_member)
|
||||
if re.match(str(dict_value_pattern), str(parsed[dict_key])):
|
||||
unformatted_item = the_member
|
||||
formatted_item = parsed
|
||||
break
|
||||
return (unformatted_item, formatted_item)
|
||||
|
||||
async def on_get_channel_messages(self, key, params):
|
||||
channel = self.bot.get_channel(int(params["channel_id"]))
|
||||
if not channel or not isinstance(channel, discord.channel.TextChannel):
|
||||
return
|
||||
await self.connection.delete([key])
|
||||
messages = []
|
||||
async for message in channel.history(limit=50):
|
||||
formatted = get_formatted_message(message)
|
||||
messages.append(json.dumps(formatted))
|
||||
await self.connection.sadd(key, messages)
|
||||
|
||||
async def push_message(self, message):
|
||||
if message.guild:
|
||||
key = "Queue/channels/{}/messages".format(message.channel.id)
|
||||
exists = await self.connection.exists(key)
|
||||
if exists:
|
||||
message = get_formatted_message(message)
|
||||
await self.connection.sadd(key, [json.dumps(message)])
|
||||
|
||||
async def delete_message(self, message):
|
||||
if message.guild:
|
||||
key = "Queue/channels/{}/messages".format(message.channel.id)
|
||||
exists = await self.connection.exists(key)
|
||||
if exists:
|
||||
unformatted_item, formatted_item = await self.set_scan_json(key, "id", message.id)
|
||||
if formatted_item:
|
||||
await self.connection.srem(key, [unformatted_item])
|
||||
|
||||
async def update_message(self, message):
|
||||
await self.delete_message(message)
|
||||
await self.push_message(message)
|
||||
@ -1,7 +1,5 @@
|
||||
import socketio
|
||||
from titanembeds.utils import get_message_author, get_message_mentions, get_roles_list, get_attachments_list, get_embeds_list
|
||||
import time
|
||||
from email import utils as emailutils
|
||||
from titanembeds.utils import get_message_author, get_message_mentions, get_roles_list, get_attachments_list, get_embeds_list, get_formatted_message, get_formatted_user, get_formatted_emojis, get_formatted_guild, get_formatted_channel, get_formatted_role
|
||||
import discord
|
||||
|
||||
class SocketIOInterface:
|
||||
@ -9,178 +7,67 @@ class SocketIOInterface:
|
||||
self.io = socketio.AsyncRedisManager(redis_uri, write_only=True, channel='flask-socketio')
|
||||
self.bot = bot
|
||||
|
||||
def format_datetime(self, datetimeobj):
|
||||
return emailutils.formatdate(time.mktime(datetimeobj.timetuple())) # https://stackoverflow.com/questions/3453177/convert-python-datetime-to-rfc-2822
|
||||
|
||||
def get_formatted_message(self, message):
|
||||
edit_ts = message.edited_at
|
||||
if not edit_ts:
|
||||
edit_ts = None
|
||||
else:
|
||||
edit_ts = self.format_datetime(edit_ts)
|
||||
msg = {
|
||||
"id": str(message.id),
|
||||
"channel_id": str(message.channel.id),
|
||||
"content": message.content,
|
||||
"author": get_message_author(message),
|
||||
"timestamp": self.format_datetime(message.created_at),
|
||||
"edited_timestamp": edit_ts,
|
||||
}
|
||||
if hasattr(message, "mentions"):
|
||||
msg["mentions"] = get_message_mentions(message.mentions)
|
||||
if hasattr(message, "attachments"):
|
||||
msg["attachments"] = get_attachments_list(message.attachments)
|
||||
if hasattr(message, "embeds"):
|
||||
msg["embeds"] = get_embeds_list(message.embeds)
|
||||
if hasattr(message, "author"):
|
||||
nickname = None
|
||||
if hasattr(message.author, 'nick') and message.author.nick:
|
||||
nickname = message.author.nick
|
||||
msg["author"]["nickname"] = nickname
|
||||
if hasattr(message, "mentions"):
|
||||
for mention in msg["mentions"]:
|
||||
mention["nickname"] = None
|
||||
member = message.guild.get_member(mention["id"])
|
||||
if member:
|
||||
mention["nickname"] = member.nick
|
||||
return msg
|
||||
|
||||
async def on_message(self, message):
|
||||
if message.guild:
|
||||
msg = self.get_formatted_message(message)
|
||||
msg = get_formatted_message(message)
|
||||
await self.io.emit('MESSAGE_CREATE', data=msg, room=str("CHANNEL_"+str(message.channel.id)), namespace='/gateway')
|
||||
|
||||
async def on_message_delete(self, message):
|
||||
if message.guild:
|
||||
msg = self.get_formatted_message(message)
|
||||
msg = get_formatted_message(message)
|
||||
await self.io.emit('MESSAGE_DELETE', data=msg, room=str("CHANNEL_"+str(message.channel.id)), namespace='/gateway')
|
||||
|
||||
async def on_message_update(self, message):
|
||||
if message.guild:
|
||||
msg = self.get_formatted_message(message)
|
||||
msg = get_formatted_message(message)
|
||||
await self.io.emit('MESSAGE_UPDATE', data=msg, room=str("CHANNEL_"+str(message.channel.id)), namespace='/gateway')
|
||||
|
||||
def get_formatted_user(self, user):
|
||||
userobj = {
|
||||
"avatar": user.avatar,
|
||||
"avatar_url": user.avatar_url,
|
||||
"color": str(user.color)[1:],
|
||||
"discriminator": user.discriminator,
|
||||
"game": None,
|
||||
"hoist-role": None,
|
||||
"id": str(user.id),
|
||||
"status": str(user.status),
|
||||
"username": user.name,
|
||||
"nick": None,
|
||||
}
|
||||
if userobj["color"] == "000000":
|
||||
userobj["color"] = None
|
||||
# if userobj["avatar_url"][len(userobj["avatar_url"])-15:] != ".jpg":
|
||||
# userobj["avatar_url"] = userobj["avatar_url"][:len(userobj["avatar_url"])-14] + ".jpg"
|
||||
if user.nick:
|
||||
userobj["nick"] = user.nick
|
||||
if hasattr(user, "activity") and user.activity:
|
||||
userobj["activity"] = {
|
||||
"name": user.activity.name
|
||||
}
|
||||
roles = sorted(user.roles, key=lambda k: k.position, reverse=True)
|
||||
for role in roles:
|
||||
if role.hoist:
|
||||
userobj["hoist-role"] = {
|
||||
"id": str(role.id),
|
||||
"name": role.name,
|
||||
"position": role.position,
|
||||
}
|
||||
break
|
||||
return userobj
|
||||
|
||||
async def on_guild_member_add(self, member):
|
||||
user = self.get_formatted_user(member)
|
||||
user = get_formatted_user(member)
|
||||
await self.io.emit('GUILD_MEMBER_ADD', data=user, room=str("GUILD_"+str(member.guild.id)), namespace='/gateway')
|
||||
|
||||
async def on_guild_member_remove(self, member):
|
||||
user = self.get_formatted_user(member)
|
||||
user = get_formatted_user(member)
|
||||
await self.io.emit('GUILD_MEMBER_REMOVE', data=user, room=str("GUILD_"+str(member.guild.id)), namespace='/gateway')
|
||||
|
||||
async def on_guild_member_update(self, member):
|
||||
user = self.get_formatted_user(member)
|
||||
user = get_formatted_user(member)
|
||||
await self.io.emit('GUILD_MEMBER_UPDATE', data=user, room=str("GUILD_"+str(member.guild.id)), namespace='/gateway')
|
||||
|
||||
def get_formatted_emojis(self, emojis):
|
||||
emotes = []
|
||||
for emo in emojis:
|
||||
emotes.append({
|
||||
"id": str(emo.id),
|
||||
"managed": emo.managed,
|
||||
"name": emo.name,
|
||||
"require_colons": emo.require_colons,
|
||||
"roles": get_roles_list(emo.roles),
|
||||
"url": emo.url,
|
||||
})
|
||||
return emotes
|
||||
|
||||
async def on_guild_emojis_update(self, emojis):
|
||||
emotes = self.get_formatted_emojis(emojis)
|
||||
emotes = get_formatted_emojis(emojis)
|
||||
await self.io.emit('GUILD_EMOJIS_UPDATE', data=emotes, room=str("GUILD_"+str(emojis[0].guild.id)), namespace='/gateway')
|
||||
|
||||
def get_formatted_guild(self, guild):
|
||||
guil = {
|
||||
"id": str(guild.id),
|
||||
"name": guild.name,
|
||||
"icon": guild.icon,
|
||||
"icon_url": guild.icon_url,
|
||||
}
|
||||
return guil
|
||||
|
||||
async def on_guild_update(self, guild):
|
||||
guildobj = self.get_formatted_guild(guild)
|
||||
guildobj = get_formatted_guild(guild)
|
||||
await self.io.emit('GUILD_UPDATE', data=guildobj, room=str("GUILD_"+str(guild.id)), namespace='/gateway')
|
||||
|
||||
def get_formatted_channel(self, channel):
|
||||
chan = {
|
||||
"id": str(channel.id),
|
||||
"guild_id": str(channel.guild.id),
|
||||
}
|
||||
return chan
|
||||
|
||||
async def on_channel_delete(self, channel):
|
||||
if str(channel.type) != "text":
|
||||
return
|
||||
chan = self.get_formatted_channel(channel)
|
||||
chan = get_formatted_channel(channel)
|
||||
await self.io.emit('CHANNEL_DELETE', data=chan, room=str("GUILD_"+str(channel.guild.id)), namespace='/gateway')
|
||||
|
||||
async def on_channel_create(self, channel):
|
||||
if str(channel.type) != "text":
|
||||
return
|
||||
chan = self.get_formatted_channel(channel)
|
||||
chan = get_formatted_channel(channel)
|
||||
await self.io.emit('CHANNEL_CREATE', data=chan, room=str("GUILD_"+str(channel.guild.id)), namespace='/gateway')
|
||||
|
||||
async def on_channel_update(self, channel):
|
||||
if not isinstance(channel, discord.channel.TextChannel) and not isinstance(channel, discord.channel.CategoryChannel):
|
||||
return
|
||||
chan = self.get_formatted_channel(channel)
|
||||
chan = get_formatted_channel(channel)
|
||||
await self.io.emit('CHANNEL_UPDATE', data=chan, room=str("GUILD_"+str(channel.guild.id)), namespace='/gateway')
|
||||
|
||||
def get_formatted_role(self, role):
|
||||
rol = {
|
||||
"id": str(role.id),
|
||||
"guild_id": str(role.guild.id),
|
||||
"name": role.name,
|
||||
"color": role.color.value,
|
||||
"hoist": role.hoist,
|
||||
"position": role.position,
|
||||
"permissions": role.permissions.value,
|
||||
}
|
||||
return rol
|
||||
|
||||
async def on_guild_role_create(self, role):
|
||||
rol = self.get_formatted_role(role)
|
||||
rol = get_formatted_role(role)
|
||||
await self.io.emit('GUILD_ROLE_CREATE', data=rol, room=str("GUILD_"+str(role.guild.id)), namespace='/gateway')
|
||||
|
||||
async def on_guild_role_update(self, role):
|
||||
rol = self.get_formatted_role(role)
|
||||
rol = get_formatted_role(role)
|
||||
await self.io.emit('GUILD_ROLE_UPDATE', data=rol, room=str("GUILD_"+str(role.guild.id)), namespace='/gateway')
|
||||
|
||||
async def on_guild_role_delete(self, role):
|
||||
rol = self.get_formatted_role(role)
|
||||
rol = get_formatted_role(role)
|
||||
await self.io.emit('GUILD_ROLE_DELETE', data=rol, room=str("GUILD_"+str(role.guild.id)), namespace='/gateway')
|
||||
@ -1,4 +1,76 @@
|
||||
import discord
|
||||
import time
|
||||
from email import utils as emailutils
|
||||
|
||||
def format_datetime(datetimeobj):
|
||||
return emailutils.formatdate(time.mktime(datetimeobj.timetuple())) # https://stackoverflow.com/questions/3453177/convert-python-datetime-to-rfc-2822
|
||||
|
||||
def get_formatted_message(message):
|
||||
edit_ts = message.edited_at
|
||||
if not edit_ts:
|
||||
edit_ts = None
|
||||
else:
|
||||
edit_ts = format_datetime(edit_ts)
|
||||
msg = {
|
||||
"id": str(message.id),
|
||||
"channel_id": str(message.channel.id),
|
||||
"content": message.content,
|
||||
"author": get_message_author(message),
|
||||
"timestamp": format_datetime(message.created_at),
|
||||
"edited_timestamp": edit_ts,
|
||||
}
|
||||
if hasattr(message, "mentions"):
|
||||
msg["mentions"] = get_message_mentions(message.mentions)
|
||||
if hasattr(message, "attachments"):
|
||||
msg["attachments"] = get_attachments_list(message.attachments)
|
||||
if hasattr(message, "embeds"):
|
||||
msg["embeds"] = get_embeds_list(message.embeds)
|
||||
if hasattr(message, "author"):
|
||||
nickname = None
|
||||
if hasattr(message.author, 'nick') and message.author.nick:
|
||||
nickname = message.author.nick
|
||||
msg["author"]["nickname"] = nickname
|
||||
if hasattr(message, "mentions"):
|
||||
for mention in msg["mentions"]:
|
||||
mention["nickname"] = None
|
||||
member = message.guild.get_member(mention["id"])
|
||||
if member:
|
||||
mention["nickname"] = member.nick
|
||||
return msg
|
||||
|
||||
def get_formatted_user(user):
|
||||
userobj = {
|
||||
"avatar": user.avatar,
|
||||
"avatar_url": user.avatar_url,
|
||||
"color": str(user.color)[1:],
|
||||
"discriminator": user.discriminator,
|
||||
"game": None,
|
||||
"hoist-role": None,
|
||||
"id": str(user.id),
|
||||
"status": str(user.status),
|
||||
"username": user.name,
|
||||
"nick": None,
|
||||
}
|
||||
if userobj["color"] == "000000":
|
||||
userobj["color"] = None
|
||||
# if userobj["avatar_url"][len(userobj["avatar_url"])-15:] != ".jpg":
|
||||
# userobj["avatar_url"] = userobj["avatar_url"][:len(userobj["avatar_url"])-14] + ".jpg"
|
||||
if user.nick:
|
||||
userobj["nick"] = user.nick
|
||||
if hasattr(user, "activity") and user.activity:
|
||||
userobj["activity"] = {
|
||||
"name": user.activity.name
|
||||
}
|
||||
roles = sorted(user.roles, key=lambda k: k.position, reverse=True)
|
||||
for role in roles:
|
||||
if role.hoist:
|
||||
userobj["hoist-role"] = {
|
||||
"id": str(role.id),
|
||||
"name": role.name,
|
||||
"position": role.position,
|
||||
}
|
||||
break
|
||||
return userobj
|
||||
|
||||
def get_message_author(message):
|
||||
if not hasattr(message, "author"):
|
||||
@ -12,6 +84,47 @@ def get_message_author(message):
|
||||
"avatar": author.avatar
|
||||
}
|
||||
return obj
|
||||
|
||||
def get_formatted_emojis(emojis):
|
||||
emotes = []
|
||||
for emo in emojis:
|
||||
emotes.append({
|
||||
"id": str(emo.id),
|
||||
"managed": emo.managed,
|
||||
"name": emo.name,
|
||||
"require_colons": emo.require_colons,
|
||||
"roles": get_roles_list(emo.roles),
|
||||
"url": emo.url,
|
||||
})
|
||||
return emotes
|
||||
|
||||
def get_formatted_guild(guild):
|
||||
guil = {
|
||||
"id": str(guild.id),
|
||||
"name": guild.name,
|
||||
"icon": guild.icon,
|
||||
"icon_url": guild.icon_url,
|
||||
}
|
||||
return guil
|
||||
|
||||
def get_formatted_channel(channel):
|
||||
chan = {
|
||||
"id": str(channel.id),
|
||||
"guild_id": str(channel.guild.id),
|
||||
}
|
||||
return chan
|
||||
|
||||
def get_formatted_role(role):
|
||||
rol = {
|
||||
"id": str(role.id),
|
||||
"guild_id": str(role.guild.id),
|
||||
"name": role.name,
|
||||
"color": role.color.value,
|
||||
"hoist": role.hoist,
|
||||
"position": role.position,
|
||||
"permissions": role.permissions.value,
|
||||
}
|
||||
return rol
|
||||
|
||||
def get_message_mentions(mentions):
|
||||
ments = []
|
||||
|
||||
Reference in New Issue
Block a user