Implemented memcached caching

This commit is contained in:
Profitroll 2025-02-09 23:00:18 +01:00
parent d402c520a5
commit b3a7e3623a
Signed by: profitroll
GPG Key ID: FA35CAB49DACD3B2
20 changed files with 460 additions and 101 deletions

0
classes/__init__.py Normal file
View File

3
classes/cache/__init__.py vendored Normal file
View File

@ -0,0 +1,3 @@
from .holo_cache import HoloCache
from .holo_cache_memcached import HoloCacheMemcached
from .holo_cache_redis import HoloCacheRedis

44
classes/cache/holo_cache.py vendored Normal file
View File

@ -0,0 +1,44 @@
from abc import ABC, abstractmethod
from typing import Any, Dict
import pymemcache
import redis
class HoloCache(ABC):
client: pymemcache.Client | redis.Redis
@classmethod
@abstractmethod
def from_config(cls, engine_config: Dict[str, Any]) -> Any:
pass
@abstractmethod
def get_json(self, key: str) -> Any | None:
# TODO This method must also carry out ObjectId conversion!
pass
@abstractmethod
def get_string(self, key: str) -> str | None:
pass
@abstractmethod
def get_object(self, key: str) -> Any | None:
pass
@abstractmethod
def set_json(self, key: str, value: Any) -> None:
# TODO This method must also carry out ObjectId conversion!
pass
@abstractmethod
def set_string(self, key: str, value: str) -> None:
pass
@abstractmethod
def set_object(self, key: str, value: Any) -> None:
pass
@abstractmethod
def delete(self, key: str) -> None:
pass

106
classes/cache/holo_cache_memcached.py vendored Normal file
View File

@ -0,0 +1,106 @@
import logging
from logging import Logger
from typing import Dict, Any
from bson import ObjectId
from pymemcache import Client
from ujson import dumps, loads
from . import HoloCache
logger: Logger = logging.getLogger(__name__)
class HoloCacheMemcached(HoloCache):
client: Client
def __init__(self, client: Client):
self.client = client
@classmethod
def from_config(cls, engine_config: Dict[str, Any]) -> "HoloCacheMemcached":
if "uri" not in engine_config:
raise KeyError(
"Cache configuration is invalid. Please check if all keys are set (engine: memcached)"
)
return cls(Client(engine_config["uri"], default_noreply=True))
def get_json(self, key: str) -> Any | None:
try:
result: Any | None = self.client.get(key, None)
logger.debug(
"Got json cache key '%s'%s",
key,
"" if result is not None else " (not found)",
)
except Exception as exc:
logger.error("Could not get json cache key '%s' due to: %s", key, exc)
return None
return None if result is None else self._string_to_json(result)
def get_string(self, key: str) -> str | None:
try:
result: str | None = self.client.get(key, None)
logger.debug(
"Got string cache key '%s'%s",
key,
"" if result is not None else " (not found)",
)
return result
except Exception as exc:
logger.error("Could not get string cache key '%s' due to: %s", key, exc)
return None
def get_object(self, key: str) -> Any | None:
# TODO Implement binary deserialization
raise NotImplementedError()
def set_json(self, key: str, value: Any) -> None:
try:
self.client.set(key, self._json_to_string(value))
logger.debug("Set json cache key '%s'", key)
except Exception as exc:
logger.error("Could not set json cache key '%s' due to: %s", key, exc)
return None
def set_string(self, key: str, value: str) -> None:
try:
self.client.set(key, value)
logger.debug("Set string cache key '%s'", key)
except Exception as exc:
logger.error("Could not set string cache key '%s' due to: %s", key, exc)
return None
def set_object(self, key: str, value: Any) -> None:
# TODO Implement binary serialization
raise NotImplementedError()
def delete(self, key: str) -> None:
try:
self.client.delete(key)
logger.debug("Deleted cache key '%s'", key)
except Exception as exc:
logger.error("Could not delete cache key '%s' due to: %s", key, exc)
@staticmethod
def _json_to_string(json_object: Any) -> str:
if isinstance(json_object, dict) and "_id" in json_object:
json_object["_id"] = str(json_object["_id"])
return dumps(
json_object, ensure_ascii=False, indent=0, escape_forward_slashes=False
)
@staticmethod
def _string_to_json(json_string: str) -> Any:
json_object: Any = loads(json_string)
if isinstance(json_object, dict) and "_id" in json_object:
json_object["_id"] = ObjectId(json_object["_id"])
return json_object

34
classes/cache/holo_cache_redis.py vendored Normal file
View File

@ -0,0 +1,34 @@
from typing import Dict, Any
from redis import Redis
from classes.cache import HoloCache
class HoloCacheRedis(HoloCache):
client: Redis
@classmethod
def from_config(cls, engine_config: Dict[str, Any]) -> Any:
raise NotImplementedError()
def get_json(self, key: str) -> Any | None:
raise NotImplementedError()
def get_string(self, key: str) -> str | None:
raise NotImplementedError()
def get_object(self, key: str) -> Any | None:
raise NotImplementedError()
def set_json(self, key: str, value: Any) -> None:
raise NotImplementedError()
def set_string(self, key: str, value: str) -> None:
raise NotImplementedError()
def set_object(self, key: str, value: Any) -> None:
raise NotImplementedError()
def delete(self, key: str) -> None:
raise NotImplementedError()

View File

@ -1,6 +1,21 @@
import logging
from logging import Logger
from libbot.pycord.classes import PycordBot
from classes.cache.holo_cache_memcached import HoloCacheMemcached
from classes.cache.holo_cache_redis import HoloCacheRedis
from modules.cache_utils import create_cache_client
logger: Logger = logging.getLogger(__name__)
class HoloBot(PycordBot):
cache: HoloCacheMemcached | HoloCacheRedis | None = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def _set_cache_engine(self) -> None:
if "cache" in self.config and self.config["cache"]["type"] is not None:
self.cache = create_cache_client(self.config, self.config["cache"]["type"])

View File

@ -6,7 +6,9 @@ from bson import ObjectId
from discord import User, Member
from libbot.utils import config_get
from pymongo.results import InsertOneResult
from typing_extensions import deprecated
from classes.cache import HoloCache
from errors import UserNotFoundError
from modules.database import col_warnings, col_users
@ -29,33 +31,41 @@ class HoloUser:
@classmethod
async def from_user(
cls, user: User | Member, allow_creation: bool = True
cls,
user: User | Member,
allow_creation: bool = True,
cache: HoloCache | None = None,
) -> "HoloUser":
"""Get an object that has a proper binding between Discord ID and database
### Args:
* `user` (User | Member): Object from which an ID can be extracted
* `allow_creation` (bool, optional): Whether to allow creation of a new user record if none found. Defaults to True.
* `cache` (HoloCache | None, optional): Cache engine to get the cache from
### Raises:
* `UserNotFoundError`: User with such ID does not seem to exist in database
"""
db_entry: Dict[str, Any] | None = await col_users.find_one({"user": user.id})
if cache is not None:
cached_entry: Dict[str, Any] | None = cache.get_json(f"user_{user.id}")
if cached_entry is not None:
return cls(**cached_entry)
db_entry: Dict[str, Any] | None = await col_users.find_one({"id": user.id})
if db_entry is None:
if not allow_creation:
raise UserNotFoundError(user=user, user_id=user.id)
db_entry = {
"user": user.id,
"custom_role": None,
"custom_channel": None,
}
db_entry = HoloUser.get_defaults(user.id)
insert_result: InsertOneResult = await col_users.insert_one(db_entry)
db_entry["_id"] = insert_result.inserted_id()
db_entry["_id"] = insert_result.inserted_id
db_entry["id"] = db_entry.pop("user")
if cache is not None:
cache.set_json(f"user_{user.id}", db_entry)
return cls(**db_entry)
@ -63,23 +73,28 @@ class HoloUser:
async def from_id(cls, user_id: int) -> "HoloUser":
return NotImplemented
# TODO Deprecate and remove warnings
@deprecated("Warnings are deprecated")
async def get_warnings(self) -> int:
"""Get number of warnings user has
### Returns:
* `int`: Number of warnings
"""
warns: Dict[str, Any] | None = await col_warnings.find_one({"user": self.id})
warns: Dict[str, Any] | None = await col_warnings.find_one({"id": self.id})
return 0 if warns is None else warns["warns"]
async def warn(self, count: int = 1, reason: str = "Not provided") -> None:
# TODO Deprecate and remove warnings
@deprecated("Warnings are deprecated")
async def warn(self, count: int = 1, reason: str = "Reason not provided") -> None:
"""Warn and add count to warns number
### Args:
* `count` (int, optional): Count of warnings to be added. Defaults to 1.
* `reason` (int, optional): Count of warnings to be added. Defaults to 1.
"""
warns: Dict[str, Any] | None = await col_warnings.find_one({"user": self.id})
warns: Dict[str, Any] | None = await col_warnings.find_one({"id": self.id})
if warns is not None:
await col_warnings.update_one(
@ -87,16 +102,17 @@ class HoloUser:
{"$set": {"warns": warns["warns"] + count}},
)
else:
await col_warnings.insert_one(document={"user": self.id, "warns": count})
await col_warnings.insert_one(document={"id": self.id, "warns": count})
logger.info("User %s was warned %s times due to: %s", self.id, count, reason)
async def _set(self, key: str, value: Any) -> None:
async def _set(self, key: str, value: Any, cache: HoloCache | None = None) -> None:
"""Set attribute data and save it into the database
### Args:
* `key` (str): Attribute to be changed
* `value` (Any): Value to set
* `cache` (HoloCache | None, optional): Cache engine to write the update into
"""
if not hasattr(self, key):
raise AttributeError()
@ -107,40 +123,99 @@ class HoloUser:
{"_id": self._id}, {"$set": {key: value}}, upsert=True
)
logger.info("Set attribute %s of user %s to %s", key, self.id, value)
self._update_cache(cache)
async def _remove(self, key: str) -> None:
logger.info("Set attribute '%s' of user %s to '%s'", key, self.id, value)
async def _remove(self, key: str, cache: HoloCache | None = None) -> None:
"""Remove attribute data and save it into the database
### Args:
* `key` (str): Attribute to be removed
* `cache` (HoloCache | None, optional): Cache engine to write the update into
"""
if not hasattr(self, key):
raise AttributeError()
setattr(self, key, None)
default_value: Any = HoloUser.get_default_value(key)
setattr(self, key, default_value)
await col_users.update_one(
{"_id": self._id}, {"$unset": {key: None}}, upsert=True
{"_id": self._id}, {"$set": {key: default_value}}, upsert=True
)
logger.info("Removed attribute %s of user %s", key, self.id)
self._update_cache(cache)
async def set_custom_channel(self, channel_id: int) -> None:
await self._set("custom_channel", channel_id)
logger.info("Removed attribute '%s' of user %s", key, self.id)
async def set_custom_role(self, role_id: int) -> None:
await self._set("custom_role", role_id)
def _get_cache_key(self) -> str:
return f"user_{self.id}"
async def remove_custom_channel(self) -> None:
await self._remove("custom_channel")
def _update_cache(self, cache: HoloCache | None = None) -> None:
if cache is None:
return
async def remove_custom_role(self) -> None:
await self._remove("custom_role")
user_dict: Dict[str, Any] = self._to_dict()
async def purge(self) -> None:
"""Completely remove user data from database. Will not remove transactions logs and warnings."""
if user_dict is not None:
cache.set_json(self._get_cache_key(), user_dict)
else:
self._delete_cache(cache)
def _delete_cache(self, cache: HoloCache | None = None) -> None:
if cache is None:
return
cache.delete(self._get_cache_key())
@staticmethod
def get_defaults(user_id: int | None = None) -> Dict[str, Any]:
return {
"id": user_id,
"custom_role": None,
"custom_channel": None,
}
@staticmethod
def get_default_value(key: str) -> Any:
if key not in HoloUser.get_defaults():
raise KeyError(f"There's no default value for key '{key}' in HoloUser")
return HoloUser.get_defaults()[key]
def _to_dict(self) -> Dict[str, Any]:
return {
"_id": self._id,
"id": self.id,
"custom_role": self.custom_role,
"custom_channel": self.custom_channel,
}
async def set_custom_channel(
self, channel_id: int, cache: HoloCache | None = None
) -> None:
await self._set("custom_channel", channel_id, cache=cache)
async def set_custom_role(
self, role_id: int, cache: HoloCache | None = None
) -> None:
await self._set("custom_role", role_id, cache=cache)
async def remove_custom_channel(self, cache: HoloCache | None = None) -> None:
await self._remove("custom_channel", cache=cache)
async def remove_custom_role(self, cache: HoloCache | None = None) -> None:
await self._remove("custom_role", cache=cache)
async def purge(self, cache: HoloCache | None = None) -> None:
"""Completely remove user data from database. Will not remove transactions logs and warnings.
### Args:
* `cache` (HoloCache | None, optional): Cache engine to write the update into
"""
await col_users.delete_one({"_id": self._id})
self._delete_cache(cache)
@staticmethod
async def is_moderator(member: User | Member) -> bool:

View File

@ -56,7 +56,7 @@ class Analytics(commands.Cog):
# Insert entry into the database
await col_analytics.insert_one(
{
"user": message.author.id,
"user_id": message.author.id,
"channel": message.channel.id,
"content": message.content,
"stickers": stickers,

View File

@ -47,7 +47,9 @@ class CustomChannels(commands.Cog):
Command to create a custom channel for a user.
"""
holo_user_ctx: HoloUser = await HoloUser.from_user(ctx.user)
holo_user_ctx: HoloUser = await HoloUser.from_user(
ctx.user, cache=self.client.cache
)
# Return if the user is using the command outside of a guild
if not hasattr(ctx.author, "guild"):
@ -100,7 +102,9 @@ class CustomChannels(commands.Cog):
manage_channels=True,
)
await holo_user_ctx.set_custom_channel(created_channel.id)
await holo_user_ctx.set_custom_channel(
created_channel.id, cache=self.client.cache
)
await ctx.respond(
embed=Embed(
@ -136,7 +140,9 @@ class CustomChannels(commands.Cog):
Command to change properties of a custom channel.
"""
holo_user_ctx: HoloUser = await HoloUser.from_user(ctx.user)
holo_user_ctx: HoloUser = await HoloUser.from_user(
ctx.user, cache=self.client.cache
)
custom_channel: TextChannel | None = ds_utils.get(
ctx.guild.channels, id=holo_user_ctx.custom_channel
@ -182,7 +188,9 @@ class CustomChannels(commands.Cog):
"""Command /customchannel remove [<confirm>]
Command to remove a custom channel. Requires additional confirmation."""
holo_user_ctx: HoloUser = await HoloUser.from_user(ctx.user)
holo_user_ctx: HoloUser = await HoloUser.from_user(
ctx.user, cache=self.client.cache
)
# Return if the user does not have a custom channel
if holo_user_ctx.custom_channel is None:
@ -211,7 +219,7 @@ class CustomChannels(commands.Cog):
color=Color.FAIL,
)
)
await holo_user_ctx.remove_custom_channel()
await holo_user_ctx.remove_custom_channel(cache=self.client.cache)
return
# Return if the confirmation is missing
@ -227,7 +235,7 @@ class CustomChannels(commands.Cog):
await custom_channel.delete(reason="Власник запросив видалення")
await holo_user_ctx.remove_custom_channel()
await holo_user_ctx.remove_custom_channel(cache=self.client.cache)
try:
await ctx.respond(

View File

@ -14,7 +14,6 @@ from libbot.utils import config_get, json_write
from classes.holo_bot import HoloBot
from classes.holo_user import HoloUser
from enums import Color
from modules.database import col_users
from modules.utils_sync import guild_name
logger: Logger = logging.getLogger(__name__)
@ -90,7 +89,7 @@ class Data(commands.Cog):
{
"id": member.id,
"nick": member.nick,
"username": f"{member.name}#{member.discriminator}",
"username": member.name,
"bot": member.bot,
}
)
@ -101,6 +100,7 @@ class Data(commands.Cog):
await ctx.respond(file=File(Path(f"tmp/{uuid}"), filename="users.json"))
# TODO Deprecate this command
@data.command(
name="migrate",
description="Мігрувати всіх користувачів до бази",
@ -164,20 +164,7 @@ class Data(commands.Cog):
if member.bot:
continue
if (await col_users.find_one({"user": member.id})) is None:
user: Dict[str, Any] = {}
defaults: Dict[str, Any] = await config_get("user", "defaults")
user["user"] = member.id
for key in defaults:
user[key] = defaults[key]
await col_users.insert_one(document=user)
logging.info(
"Added DB record for user %s during migration", member.id
)
await HoloUser.from_user(member, cache=self.client.cache)
await ctx.respond(
embed=Embed(

View File

@ -1,6 +1,5 @@
import logging
from logging import Logger
from typing import Dict, Any
from discord import Member, Message, TextChannel, MessageType
from discord import utils as ds_utils
@ -8,6 +7,7 @@ from discord.ext import commands
from libbot.utils import config_get
from classes.holo_bot import HoloBot
from classes.holo_user import HoloUser
from modules.database import col_users
logger: Logger = logging.getLogger(__name__)
@ -25,16 +25,7 @@ class Logger(commands.Cog):
and (message.author.bot is False)
and (message.author.system is False)
):
if (await col_users.find_one({"user": message.author.id})) is None:
user: Dict[str, Any] = {}
defaults: Dict[str, Any] = await config_get("user", "defaults")
user["user"] = message.author.id
for key in defaults:
user[key] = defaults[key]
await col_users.insert_one(document=user)
await HoloUser.from_user(message.author, cache=self.client.cache)
if (
(message.type == MessageType.thread_created)
@ -69,30 +60,21 @@ class Logger(commands.Cog):
id=await config_get("rules", "channels", "text"),
)
if welcome_chan is None:
logger.warning("Could not find a welcome channel by its id")
if (
(member != self.client.user)
and (member.bot is False)
and (member.system is False)
):
await welcome_chan.send(
content=(await config_get("welcome", "messages")).format(
mention=member.mention, rules=rules_chan.mention
if welcome_chan is not None and rules_chan is not None:
await welcome_chan.send(
content=(await config_get("welcome", "messages")).format(
mention=member.mention, rules=rules_chan.mention
)
)
)
else:
logger.warning("Could not find a welcome and/or rules channel by id")
if (await col_users.find_one({"user": member.id})) is None:
user: Dict[str, Any] = {}
defaults: Dict[str, Any] = await config_get("user", "defaults")
user["user"] = member.id
for key in defaults:
user[key] = defaults[key]
await col_users.insert_one(document=user)
await HoloUser.from_user(member, cache=self.client.cache)
def setup(client: HoloBot) -> None:

View File

@ -22,16 +22,20 @@
"port": 27017,
"name": "holo_discord"
},
"cache": {
"type": null,
"memcached": {
"uri": "127.0.0.1:11211"
},
"redis": {
"uri": "127.0.0.1:6379"
}
},
"logging": {
"size": 512,
"location": "logs"
},
"defaults": {
"user": {
"custom_role": null,
"custom_channel": null
}
},
"defaults": {},
"categories": {
"custom_channels": 0
},

15
main.py
View File

@ -12,8 +12,15 @@ from classes.holo_bot import HoloBot
from modules.migrator import migrate_database
from modules.scheduler import scheduler
if not Path("config.json").exists():
print(
"Config file is missing: Make sure the configuration file 'config.json' is in place.",
flush=True,
)
sys.exit()
logging.basicConfig(
level=logging.INFO,
level=logging.INFO if not config_get("debug") else logging.DEBUG,
format="%(name)s.%(funcName)s | %(levelname)s | %(message)s",
datefmt="[%X]",
)
@ -41,12 +48,6 @@ with contextlib.suppress(ImportError):
def main() -> None:
if not Path("config.json").exists():
logger.error(
"Config file is missing: Make sure the configuration file 'config.json' is in place."
)
sys.exit()
# Perform migration if command line argument was provided
if args.migrate:
logger.info("Performing migrations...")

View File

@ -0,0 +1,63 @@
import logging
from logging import Logger
from libbot.utils import config_set, config_delete
from mongodb_migrations.base import BaseMigration
logger: Logger = logging.getLogger(__name__)
class Migration(BaseMigration):
def upgrade(self):
try:
config_set(
"cache",
{
"type": None,
"memcached": {"uri": "127.0.0.1:11211"},
"redis": {"uri": "127.0.0.1:6379"},
},
*[],
)
except Exception as exc:
logger.error(
"Could not upgrade the config during migration '%s' due to: %s",
__name__,
exc,
)
self.db.users.update_many(
{"user": {"$exists": True}},
{"$rename": {"user": "id"}},
)
self.db.analytics.update_many(
{"user": {"$exists": True}},
{"$rename": {"user": "user_id"}},
)
self.db.warnings.update_many(
{"user": {"$exists": True}},
{"$rename": {"user": "user_id"}},
)
def downgrade(self):
try:
config_delete("cache", *[])
except Exception as exc:
logger.error(
"Could not downgrade the config during migration '%s' due to: %s",
__name__,
exc,
)
self.db.users.update_many(
{"id": {"$exists": True}},
{"$rename": {"id": "user"}},
)
self.db.analytics.update_many(
{"user": {"$exists": True}},
{"$rename": {"user_id": "user"}},
)
self.db.warnings.update_many(
{"user": {"$exists": True}},
{"$rename": {"user_id": "user"}},
)

29
modules/cache_utils.py Normal file
View File

@ -0,0 +1,29 @@
from typing import Dict, Any, Literal
from classes.cache.holo_cache_memcached import HoloCacheMemcached
from classes.cache.holo_cache_redis import HoloCacheRedis
def create_cache_client(
config: Dict[str, Any],
engine: Literal["memcached", "redis"] | None = None,
) -> HoloCacheMemcached | HoloCacheRedis:
if engine not in ["memcached", "redis"] or engine is None:
raise KeyError(
f"Incorrect cache engine provided. Expected 'memcached' or 'redis', got '{engine}'"
)
if "cache" not in config or engine not in config["cache"]:
raise KeyError(
f"Cache configuration is invalid. Please check if all keys are set (engine: '{engine}')"
)
match engine:
case "memcached":
return HoloCacheMemcached.from_config(config["cache"][engine])
case "redis":
return HoloCacheRedis.from_config(config["cache"][engine])
case _:
raise KeyError(
f"Cache implementation for the engine '{engine}' is not present."
)

View File

@ -38,3 +38,6 @@ sync_db: Database = db_client_sync.get_database(name=db_config["name"])
sync_col_users: Collection = sync_db.get_collection("users")
sync_col_warnings: Collection = sync_db.get_collection("warnings")
sync_col_analytics: Collection = sync_db.get_collection("analytics")
# Update indexes
sync_col_users.create_index(["id"], unique=True)

View File

@ -7,5 +7,7 @@ apscheduler>=3.10.0
async_pymongo==0.1.11
libbot[speed,pycord]==4.0.2
mongodb-migrations==1.3.1
pymemcache~=4.0.0
redis~=5.2.1
ujson~=5.10.0
WaifuPicsPython==0.2.0

View File

@ -1,14 +1,14 @@
{
"$jsonSchema": {
"required": [
"user",
"user_id",
"channel",
"content",
"stickers",
"attachments"
],
"properties": {
"user": {
"user_id": {
"bsonType": "long",
"description": "Discord ID of user"
},
@ -17,7 +17,10 @@
"description": "Discord ID of a channel"
},
"content": {
"bsonType": ["null", "string"],
"bsonType": [
"null",
"string"
],
"description": "Text of the message"
},
"stickers": {
@ -40,7 +43,7 @@
"format": {
"bsonType": "array"
},
"user": {
"user_id": {
"bsonType": "string"
}
}

View File

@ -1,12 +1,12 @@
{
"$jsonSchema": {
"required": [
"user",
"id",
"custom_role",
"custom_channel"
],
"properties": {
"user": {
"id": {
"bsonType": "long",
"description": "Discord ID of user"
},

View File

@ -1,11 +1,11 @@
{
"$jsonSchema": {
"required": [
"user",
"user_id",
"warnings"
],
"properties": {
"user": {
"user_id": {
"bsonType": "long",
"description": "Discord ID of user"
},