HoloUA/Discord
HoloUA/Discord
1
0
Fork 0
Code Issues Pull Requests Releases 8 Activity

Caching will now be used from libbot

Browse Source
This commit is contained in:
Profitroll 2025-02-16 17:44:13 +01:00
parent afbb36e182
commit 95aecd3c99
Signed by: profitroll
GPG Key ID: FA35CAB49DACD3B2
10 changed files with 21 additions and 312 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
classes/cache/__init__.py vendored
View File

@ -1,3 +0,0 @@
from .holo_cache import HoloCache
from .holo_cache_memcached import HoloCacheMemcached
from .holo_cache_redis import HoloCacheRedis

44
classes/cache/holo_cache.py vendored
View File

@ -1,44 +0,0 @@
from abc import ABC, abstractmethod
from typing import Any, Dict
import pymemcache
import redis
class HoloCache(ABC):
client: pymemcache.Client | redis.Redis
@classmethod
@abstractmethod
def from_config(cls, engine_config: Dict[str, Any]) -> Any:
pass
@abstractmethod
def get_json(self, key: str) -> Any | None:
# TODO This method must also carry out ObjectId conversion!
pass
@abstractmethod
def get_string(self, key: str) -> str | None:
pass
@abstractmethod
def get_object(self, key: str) -> Any | None:
pass
@abstractmethod
def set_json(self, key: str, value: Any) -> None:
# TODO This method must also carry out ObjectId conversion!
pass
@abstractmethod
def set_string(self, key: str, value: str) -> None:
pass
@abstractmethod
def set_object(self, key: str, value: Any) -> None:
pass
@abstractmethod
def delete(self, key: str) -> None:
pass

89
classes/cache/holo_cache_memcached.py vendored
View File

@ -1,89 +0,0 @@
import logging
from logging import Logger
from typing import Dict, Any
from pymemcache import Client
from modules.cache_utils import string_to_json, json_to_string
from . import HoloCache
logger: Logger = logging.getLogger(__name__)
class HoloCacheMemcached(HoloCache):
client: Client
def __init__(self, client: Client):
self.client = client
logger.info("Initialized Memcached for caching")
@classmethod
def from_config(cls, engine_config: Dict[str, Any]) -> "HoloCacheMemcached":
if "uri" not in engine_config:
raise KeyError(
"Cache configuration is invalid. Please check if all keys are set (engine: memcached)"
)
return cls(Client(engine_config["uri"], default_noreply=True))
def get_json(self, key: str) -> Any | None:
try:
result: Any | None = self.client.get(key, None)
logger.debug(
"Got json cache key '%s'%s",
key,
"" if result is not None else " (not found)",
)
except Exception as exc:
logger.error("Could not get json cache key '%s' due to: %s", key, exc)
return None
return None if result is None else string_to_json(result)
def get_string(self, key: str) -> str | None:
try:
result: str | None = self.client.get(key, None)
logger.debug(
"Got string cache key '%s'%s",
key,
"" if result is not None else " (not found)",
)
return result
except Exception as exc:
logger.error("Could not get string cache key '%s' due to: %s", key, exc)
return None
# TODO Implement binary deserialization
def get_object(self, key: str) -> Any | None:
raise NotImplementedError()
def set_json(self, key: str, value: Any) -> None:
try:
self.client.set(key, json_to_string(value))
logger.debug("Set json cache key '%s'", key)
except Exception as exc:
logger.error("Could not set json cache key '%s' due to: %s", key, exc)
return None
def set_string(self, key: str, value: str) -> None:
try:
self.client.set(key, value)
logger.debug("Set string cache key '%s'", key)
except Exception as exc:
logger.error("Could not set string cache key '%s' due to: %s", key, exc)
return None
# TODO Implement binary serialization
def set_object(self, key: str, value: Any) -> None:
raise NotImplementedError()
def delete(self, key: str) -> None:
try:
self.client.delete(key)
logger.debug("Deleted cache key '%s'", key)
except Exception as exc:
logger.error("Could not delete cache key '%s' due to: %s", key, exc)

89
classes/cache/holo_cache_redis.py vendored
View File

@ -1,89 +0,0 @@
import logging
from logging import Logger
from typing import Dict, Any
from redis import Redis
from classes.cache import HoloCache
from modules.cache_utils import string_to_json, json_to_string
logger: Logger = logging.getLogger(__name__)
class HoloCacheRedis(HoloCache):
client: Redis
def __init__(self, client: Redis):
self.client = client
logger.info("Initialized Redis for caching")
@classmethod
def from_config(cls, engine_config: Dict[str, Any]) -> Any:
if "uri" not in engine_config:
raise KeyError(
"Cache configuration is invalid. Please check if all keys are set (engine: memcached)"
)
return cls(Redis.from_url(engine_config["uri"]))
def get_json(self, key: str) -> Any | None:
try:
result: Any | None = self.client.get(key)
logger.debug(
"Got json cache key '%s'%s",
key,
"" if result is not None else " (not found)",
)
except Exception as exc:
logger.error("Could not get json cache key '%s' due to: %s", key, exc)
return None
return None if result is None else string_to_json(result)
def get_string(self, key: str) -> str | None:
try:
result: str | None = self.client.get(key)
logger.debug(
"Got string cache key '%s'%s",
key,
"" if result is not None else " (not found)",
)
return result
except Exception as exc:
logger.error("Could not get string cache key '%s' due to: %s", key, exc)
return None
# TODO Implement binary deserialization
def get_object(self, key: str) -> Any | None:
raise NotImplementedError()
def set_json(self, key: str, value: Any) -> None:
try:
self.client.set(key, json_to_string(value))
logger.debug("Set json cache key '%s'", key)
except Exception as exc:
logger.error("Could not set json cache key '%s' due to: %s", key, exc)
return None
def set_string(self, key: str, value: str) -> None:
try:
self.client.set(key, value)
logger.debug("Set string cache key '%s'", key)
except Exception as exc:
logger.error("Could not set string cache key '%s' due to: %s", key, exc)
return None
# TODO Implement binary serialization
def set_object(self, key: str, value: Any) -> None:
raise NotImplementedError()
def delete(self, key: str) -> None:
try:
self.client.delete(key)
logger.debug("Deleted cache key '%s'", key)
except Exception as exc:
logger.error("Could not delete cache key '%s' due to: %s", key, exc)

8
classes/holo_bot.py
View File

@ -1,17 +1,15 @@
import logging
from logging import Logger
from libbot.cache.classes import CacheMemcached, CacheRedis
from libbot.cache.manager import create_cache_client
from libbot.pycord.classes import PycordBot
from classes.cache.holo_cache_memcached import HoloCacheMemcached
from classes.cache.holo_cache_redis import HoloCacheRedis
from modules.cache_manager import create_cache_client
logger: Logger = logging.getLogger(__name__)
class HoloBot(PycordBot):
cache: HoloCacheMemcached | HoloCacheRedis | None = None
cache: CacheMemcached | CacheRedis | None = None
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)

32
classes/holo_user.py
View File

@ -1,13 +1,13 @@
import logging
from logging import Logger
from typing import Any, Dict
from typing import Any, Dict, Optional
from bson import ObjectId
from discord import User, Member
from libbot.cache.classes import Cache
from libbot.utils import config_get
from pymongo.results import InsertOneResult
from classes.cache import HoloCache
from errors import UserNotFoundError
from modules.database import col_users
@ -33,14 +33,14 @@ class HoloUser:
cls,
user: User | Member,
allow_creation: bool = True,
cache: HoloCache | None = None,
cache: Optional[Cache] = None,
) -> "HoloUser":
"""Get an object that has a proper binding between Discord ID and database
### Args:
* `user` (User | Member): Object from which an ID can be extracted
* `allow_creation` (bool, optional): Whether to allow creation of a new user record if none found. Defaults to True.
* `cache` (HoloCache | None, optional): Cache engine to get the cache from
* `cache` (Cache, optional): Cache engine to get the cache from
### Raises:
* `UserNotFoundError`: User with such ID does not seem to exist in database
@ -72,13 +72,13 @@ class HoloUser:
async def from_id(cls, user_id: int) -> "HoloUser":
raise NotImplementedError()
async def _set(self, key: str, value: Any, cache: HoloCache | None = None) -> None:
async def _set(self, key: str, value: Any, cache: Optional[Cache] = None) -> None:
"""Set attribute data and save it into the database
### Args:
* `key` (str): Attribute to be changed
* `value` (Any): Value to set
* `cache` (HoloCache | None, optional): Cache engine to write the update into
* `cache` (Cache, optional): Cache engine to write the update into
"""
if not hasattr(self, key):
raise AttributeError()
@ -93,12 +93,12 @@ class HoloUser:
logger.info("Set attribute '%s' of user %s to '%s'", key, self.id, value)
async def _remove(self, key: str, cache: HoloCache | None = None) -> None:
async def _remove(self, key: str, cache: Optional[Cache] = None) -> None:
"""Remove attribute data and save it into the database
### Args:
* `key` (str): Attribute to be removed
* `cache` (HoloCache | None, optional): Cache engine to write the update into
* `cache` (Cache, optional): Cache engine to write the update into
"""
if not hasattr(self, key):
raise AttributeError()
@ -118,7 +118,7 @@ class HoloUser:
def _get_cache_key(self) -> str:
return f"user_{self.id}"
def _update_cache(self, cache: HoloCache | None = None) -> None:
def _update_cache(self, cache: Optional[Cache] = None) -> None:
if cache is None:
return
@ -129,7 +129,7 @@ class HoloUser:
else:
self._delete_cache(cache)
def _delete_cache(self, cache: HoloCache | None = None) -> None:
def _delete_cache(self, cache: Optional[Cache] = None) -> None:
if cache is None:
return
@ -159,26 +159,26 @@ class HoloUser:
}
async def set_custom_channel(
self, channel_id: int, cache: HoloCache | None = None
self, channel_id: int, cache: Optional[Cache] = None
) -> None:
await self._set("custom_channel", channel_id, cache=cache)
async def set_custom_role(
self, role_id: int, cache: HoloCache | None = None
self, role_id: int, cache: Optional[Cache] = None
) -> None:
await self._set("custom_role", role_id, cache=cache)
async def remove_custom_channel(self, cache: HoloCache | None = None) -> None:
async def remove_custom_channel(self, cache: Optional[Cache] = None) -> None:
await self._remove("custom_channel", cache=cache)
async def remove_custom_role(self, cache: HoloCache | None = None) -> None:
async def remove_custom_role(self, cache: Optional[Cache] = None) -> None:
await self._remove("custom_role", cache=cache)
async def purge(self, cache: HoloCache | None = None) -> None:
async def purge(self, cache: Optional[Cache] = None) -> None:
"""Completely remove user data from database. Only removes the user record from users collection.
### Args:
* `cache` (HoloCache | None, optional): Cache engine to write the update into
* `cache` (Cache, optional): Cache engine to write the update into
"""
await col_users.delete_one({"_id": self._id})
self._delete_cache(cache)

29
modules/cache_manager.py
View File

@ -1,29 +0,0 @@
from typing import Dict, Any, Literal
from classes.cache.holo_cache_memcached import HoloCacheMemcached
from classes.cache.holo_cache_redis import HoloCacheRedis
def create_cache_client(
config: Dict[str, Any],
engine: Literal["memcached", "redis"] | None = None,
) -> HoloCacheMemcached | HoloCacheRedis:
if engine not in ["memcached", "redis"] or engine is None:
raise KeyError(
f"Incorrect cache engine provided. Expected 'memcached' or 'redis', got '{engine}'"
)
if "cache" not in config or engine not in config["cache"]:
raise KeyError(
f"Cache configuration is invalid. Please check if all keys are set (engine: '{engine}')"
)
match engine:
case "memcached":
return HoloCacheMemcached.from_config(config["cache"][engine])
case "redis":
return HoloCacheRedis.from_config(config["cache"][engine])
case _:
raise KeyError(
f"Cache implementation for the engine '{engine}' is not present."
)

25
modules/cache_utils.py
View File

@ -1,25 +0,0 @@
from copy import deepcopy
from typing import Any
from bson import ObjectId
from ujson import dumps, loads
def json_to_string(json_object: Any) -> str:
json_object_copy: Any = deepcopy(json_object)
if isinstance(json_object_copy, dict) and "_id" in json_object_copy:
json_object_copy["_id"] = str(json_object_copy["_id"])
return dumps(
json_object_copy, ensure_ascii=False, indent=0, escape_forward_slashes=False
)
def string_to_json(json_string: str) -> Any:
json_object: Any = loads(json_string)
if "_id" in json_object:
json_object["_id"] = ObjectId(json_object["_id"])
return json_object

12
modules/database.py
View File

@ -2,9 +2,6 @@ from typing import Dict, Any
from async_pymongo import AsyncClient, AsyncCollection, AsyncDatabase
from libbot.utils import config_get
from pymongo import MongoClient
from pymongo.synchronous.collection import Collection
from pymongo.synchronous.database import Database
db_config: Dict[str, Any] = config_get("database")
@ -23,7 +20,6 @@ con_string: str = (
)
db_client: AsyncClient = AsyncClient(con_string)
db_client_sync: MongoClient = MongoClient(con_string)
# Async declarations per default
db: AsyncDatabase = db_client.get_database(name=db_config["name"])
@ -31,11 +27,5 @@ db: AsyncDatabase = db_client.get_database(name=db_config["name"])
col_users: AsyncCollection = db.get_collection("users")
col_analytics: AsyncCollection = db.get_collection("analytics")
# Sync declarations as a fallback
sync_db: Database = db_client_sync.get_database(name=db_config["name"])
sync_col_users: Collection = sync_db.get_collection("users")
sync_col_analytics: Collection = sync_db.get_collection("analytics")
# Update indexes
sync_col_users.create_index(["id"], unique=True)
db.dispatch.get_collection("users").create_index("id", unique=True)

2
requirements.txt
View File

@ -5,7 +5,7 @@ requests>=2.32.2
aiofiles~=24.1.0
apscheduler>=3.10.0
async_pymongo==0.1.11
libbot[speed,pycord]==4.0.2
libbot[speed,pycord,cache]==4.1.0
mongodb-migrations==1.3.1
pymemcache~=4.0.0
redis~=5.2.1