From 226721bb62e7a67807d4ccb17033e25f26879492 Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 9 Feb 2025 20:57:22 +0100 Subject: [PATCH 1/3] Updated username to replace discriminator with a username for data export --- cogs/data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cogs/data.py b/cogs/data.py index b5af890..e892039 100644 --- a/cogs/data.py +++ b/cogs/data.py @@ -90,13 +90,14 @@ class Data(commands.Cog): { "id": member.id, "nick": member.nick, - "username": f"{member.name}#{member.discriminator}", + "username": f"{member.name}", "bot": member.bot, } ) # Temporary file must be written synchronously, # otherwise it will not be there when ctx.respond() is be called + # TODO Find a way to give this file to Pycord without FS operations json_write(users, Path(f"tmp/{uuid}")) await ctx.respond(file=File(Path(f"tmp/{uuid}"), filename="users.json")) From dae89fd1ac12562a5b0ec3a3ad3229b0141e7869 Mon Sep 17 00:00:00 2001 From: kku Date: Mon, 10 Feb 2025 12:50:56 +0100 Subject: [PATCH 2/3] Removed warnings functionality --- classes/holo_user.py | 40 +++------------------------------------- modules/database.py | 2 -- validation/warnings.json | 18 ------------------ 3 files changed, 3 insertions(+), 57 deletions(-) delete mode 100644 validation/warnings.json diff --git a/classes/holo_user.py b/classes/holo_user.py index a4c2506..2d08564 100644 --- a/classes/holo_user.py +++ b/classes/holo_user.py @@ -6,11 +6,10 @@ from bson import ObjectId from discord import User, Member from libbot.utils import config_get from pymongo.results import InsertOneResult -from typing_extensions import deprecated from classes.cache import HoloCache from errors import UserNotFoundError -from modules.database import col_warnings, col_users +from modules.database import col_users logger: Logger = logging.getLogger(__name__) @@ -71,40 +70,7 @@ class HoloUser: @classmethod async def from_id(cls, user_id: int) -> "HoloUser": - return NotImplemented - - # TODO Deprecate and remove warnings - @deprecated("Warnings are deprecated") - async def get_warnings(self) -> int: - """Get number of warnings user has - - ### Returns: - * `int`: Number of warnings - """ - warns: Dict[str, Any] | None = await col_warnings.find_one({"id": self.id}) - - return 0 if warns is None else warns["warns"] - - # TODO Deprecate and remove warnings - @deprecated("Warnings are deprecated") - async def warn(self, count: int = 1, reason: str = "Reason not provided") -> None: - """Warn and add count to warns number - - ### Args: - * `count` (int, optional): Count of warnings to be added. Defaults to 1. - * `reason` (int, optional): Count of warnings to be added. Defaults to 1. - """ - warns: Dict[str, Any] | None = await col_warnings.find_one({"id": self.id}) - - if warns is not None: - await col_warnings.update_one( - {"_id": self._id}, - {"$set": {"warns": warns["warns"] + count}}, - ) - else: - await col_warnings.insert_one(document={"id": self.id, "warns": count}) - - logger.info("User %s was warned %s times due to: %s", self.id, count, reason) + raise NotImplementedError() async def _set(self, key: str, value: Any, cache: HoloCache | None = None) -> None: """Set attribute data and save it into the database @@ -209,7 +175,7 @@ class HoloUser: await self._remove("custom_role", cache=cache) async def purge(self, cache: HoloCache | None = None) -> None: - """Completely remove user data from database. Will not remove transactions logs and warnings. + """Completely remove user data from database. Only removes the user record from users collection. ### Args: * `cache` (HoloCache | None, optional): Cache engine to write the update into diff --git a/modules/database.py b/modules/database.py index fb62da4..97bf49a 100644 --- a/modules/database.py +++ b/modules/database.py @@ -29,14 +29,12 @@ db_client_sync: MongoClient = MongoClient(con_string) db: AsyncDatabase = db_client.get_database(name=db_config["name"]) col_users: AsyncCollection = db.get_collection("users") -col_warnings: AsyncCollection = db.get_collection("warnings") col_analytics: AsyncCollection = db.get_collection("analytics") # Sync declarations as a fallback sync_db: Database = db_client_sync.get_database(name=db_config["name"]) sync_col_users: Collection = sync_db.get_collection("users") -sync_col_warnings: Collection = sync_db.get_collection("warnings") sync_col_analytics: Collection = sync_db.get_collection("analytics") # Update indexes diff --git a/validation/warnings.json b/validation/warnings.json deleted file mode 100644 index 23596ea..0000000 --- a/validation/warnings.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "$jsonSchema": { - "required": [ - "user_id", - "warnings" - ], - "properties": { - "user_id": { - "bsonType": "long", - "description": "Discord ID of user" - }, - "warnings": { - "bsonType": "int", - "description": "Number of warnings on count" - } - } - } -} \ No newline at end of file From 95aecd3c991f5de407d473227eaf1fbc24db618d Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 16 Feb 2025 17:44:13 +0100 Subject: [PATCH 3/3] Caching will now be used from libbot --- classes/cache/__init__.py | 3 - classes/cache/holo_cache.py | 44 ------------- classes/cache/holo_cache_memcached.py | 89 --------------------------- classes/cache/holo_cache_redis.py | 89 --------------------------- classes/holo_bot.py | 8 +-- classes/holo_user.py | 32 +++++----- modules/cache_manager.py | 29 --------- modules/cache_utils.py | 25 -------- modules/database.py | 12 +--- requirements.txt | 2 +- 10 files changed, 21 insertions(+), 312 deletions(-) delete mode 100644 classes/cache/__init__.py delete mode 100644 classes/cache/holo_cache.py delete mode 100644 classes/cache/holo_cache_memcached.py delete mode 100644 classes/cache/holo_cache_redis.py delete mode 100644 modules/cache_manager.py delete mode 100644 modules/cache_utils.py diff --git a/classes/cache/__init__.py b/classes/cache/__init__.py deleted file mode 100644 index 05d7db8..0000000 --- a/classes/cache/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .holo_cache import HoloCache -from .holo_cache_memcached import HoloCacheMemcached -from .holo_cache_redis import HoloCacheRedis diff --git a/classes/cache/holo_cache.py b/classes/cache/holo_cache.py deleted file mode 100644 index 5969365..0000000 --- a/classes/cache/holo_cache.py +++ /dev/null @@ -1,44 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, Dict - -import pymemcache -import redis - - -class HoloCache(ABC): - client: pymemcache.Client | redis.Redis - - @classmethod - @abstractmethod - def from_config(cls, engine_config: Dict[str, Any]) -> Any: - pass - - @abstractmethod - def get_json(self, key: str) -> Any | None: - # TODO This method must also carry out ObjectId conversion! - pass - - @abstractmethod - def get_string(self, key: str) -> str | None: - pass - - @abstractmethod - def get_object(self, key: str) -> Any | None: - pass - - @abstractmethod - def set_json(self, key: str, value: Any) -> None: - # TODO This method must also carry out ObjectId conversion! - pass - - @abstractmethod - def set_string(self, key: str, value: str) -> None: - pass - - @abstractmethod - def set_object(self, key: str, value: Any) -> None: - pass - - @abstractmethod - def delete(self, key: str) -> None: - pass diff --git a/classes/cache/holo_cache_memcached.py b/classes/cache/holo_cache_memcached.py deleted file mode 100644 index fe53e65..0000000 --- a/classes/cache/holo_cache_memcached.py +++ /dev/null @@ -1,89 +0,0 @@ -import logging -from logging import Logger -from typing import Dict, Any - -from pymemcache import Client - -from modules.cache_utils import string_to_json, json_to_string -from . import HoloCache - -logger: Logger = logging.getLogger(__name__) - - -class HoloCacheMemcached(HoloCache): - client: Client - - def __init__(self, client: Client): - self.client = client - - logger.info("Initialized Memcached for caching") - - @classmethod - def from_config(cls, engine_config: Dict[str, Any]) -> "HoloCacheMemcached": - if "uri" not in engine_config: - raise KeyError( - "Cache configuration is invalid. Please check if all keys are set (engine: memcached)" - ) - - return cls(Client(engine_config["uri"], default_noreply=True)) - - def get_json(self, key: str) -> Any | None: - try: - result: Any | None = self.client.get(key, None) - - logger.debug( - "Got json cache key '%s'%s", - key, - "" if result is not None else " (not found)", - ) - except Exception as exc: - logger.error("Could not get json cache key '%s' due to: %s", key, exc) - return None - - return None if result is None else string_to_json(result) - - def get_string(self, key: str) -> str | None: - try: - result: str | None = self.client.get(key, None) - - logger.debug( - "Got string cache key '%s'%s", - key, - "" if result is not None else " (not found)", - ) - - return result - except Exception as exc: - logger.error("Could not get string cache key '%s' due to: %s", key, exc) - return None - - # TODO Implement binary deserialization - def get_object(self, key: str) -> Any | None: - raise NotImplementedError() - - def set_json(self, key: str, value: Any) -> None: - try: - self.client.set(key, json_to_string(value)) - logger.debug("Set json cache key '%s'", key) - except Exception as exc: - logger.error("Could not set json cache key '%s' due to: %s", key, exc) - return None - - def set_string(self, key: str, value: str) -> None: - try: - self.client.set(key, value) - logger.debug("Set string cache key '%s'", key) - except Exception as exc: - logger.error("Could not set string cache key '%s' due to: %s", key, exc) - return None - - # TODO Implement binary serialization - def set_object(self, key: str, value: Any) -> None: - raise NotImplementedError() - - def delete(self, key: str) -> None: - try: - self.client.delete(key) - logger.debug("Deleted cache key '%s'", key) - except Exception as exc: - logger.error("Could not delete cache key '%s' due to: %s", key, exc) diff --git a/classes/cache/holo_cache_redis.py b/classes/cache/holo_cache_redis.py deleted file mode 100644 index 7d6a125..0000000 --- a/classes/cache/holo_cache_redis.py +++ /dev/null @@ -1,89 +0,0 @@ -import logging -from logging import Logger -from typing import Dict, Any - -from redis import Redis - -from classes.cache import HoloCache -from modules.cache_utils import string_to_json, json_to_string - -logger: Logger = logging.getLogger(__name__) - - -class HoloCacheRedis(HoloCache): - client: Redis - - def __init__(self, client: Redis): - self.client = client - - logger.info("Initialized Redis for caching") - - @classmethod - def from_config(cls, engine_config: Dict[str, Any]) -> Any: - if "uri" not in engine_config: - raise KeyError( - "Cache configuration is invalid. Please check if all keys are set (engine: memcached)" - ) - - return cls(Redis.from_url(engine_config["uri"])) - - def get_json(self, key: str) -> Any | None: - try: - result: Any | None = self.client.get(key) - - logger.debug( - "Got json cache key '%s'%s", - key, - "" if result is not None else " (not found)", - ) - except Exception as exc: - logger.error("Could not get json cache key '%s' due to: %s", key, exc) - return None - - return None if result is None else string_to_json(result) - - def get_string(self, key: str) -> str | None: - try: - result: str | None = self.client.get(key) - - logger.debug( - "Got string cache key '%s'%s", - key, - "" if result is not None else " (not found)", - ) - - return result - except Exception as exc: - logger.error("Could not get string cache key '%s' due to: %s", key, exc) - return None - - # TODO Implement binary deserialization - def get_object(self, key: str) -> Any | None: - raise NotImplementedError() - - def set_json(self, key: str, value: Any) -> None: - try: - self.client.set(key, json_to_string(value)) - logger.debug("Set json cache key '%s'", key) - except Exception as exc: - logger.error("Could not set json cache key '%s' due to: %s", key, exc) - return None - - def set_string(self, key: str, value: str) -> None: - try: - self.client.set(key, value) - logger.debug("Set string cache key '%s'", key) - except Exception as exc: - logger.error("Could not set string cache key '%s' due to: %s", key, exc) - return None - - # TODO Implement binary serialization - def set_object(self, key: str, value: Any) -> None: - raise NotImplementedError() - - def delete(self, key: str) -> None: - try: - self.client.delete(key) - logger.debug("Deleted cache key '%s'", key) - except Exception as exc: - logger.error("Could not delete cache key '%s' due to: %s", key, exc) diff --git a/classes/holo_bot.py b/classes/holo_bot.py index 881e92d..aad37a6 100644 --- a/classes/holo_bot.py +++ b/classes/holo_bot.py @@ -1,17 +1,15 @@ import logging from logging import Logger +from libbot.cache.classes import CacheMemcached, CacheRedis +from libbot.cache.manager import create_cache_client from libbot.pycord.classes import PycordBot -from classes.cache.holo_cache_memcached import HoloCacheMemcached -from classes.cache.holo_cache_redis import HoloCacheRedis -from modules.cache_manager import create_cache_client - logger: Logger = logging.getLogger(__name__) class HoloBot(PycordBot): - cache: HoloCacheMemcached | HoloCacheRedis | None = None + cache: CacheMemcached | CacheRedis | None = None def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/classes/holo_user.py b/classes/holo_user.py index 2d08564..1562e83 100644 --- a/classes/holo_user.py +++ b/classes/holo_user.py @@ -1,13 +1,13 @@ import logging from logging import Logger -from typing import Any, Dict +from typing import Any, Dict, Optional from bson import ObjectId from discord import User, Member +from libbot.cache.classes import Cache from libbot.utils import config_get from pymongo.results import InsertOneResult -from classes.cache import HoloCache from errors import UserNotFoundError from modules.database import col_users @@ -33,14 +33,14 @@ class HoloUser: cls, user: User | Member, allow_creation: bool = True, - cache: HoloCache | None = None, + cache: Optional[Cache] = None, ) -> "HoloUser": """Get an object that has a proper binding between Discord ID and database ### Args: * `user` (User | Member): Object from which an ID can be extracted * `allow_creation` (bool, optional): Whether to allow creation of a new user record if none found. Defaults to True. - * `cache` (HoloCache | None, optional): Cache engine to get the cache from + * `cache` (Cache, optional): Cache engine to get the cache from ### Raises: * `UserNotFoundError`: User with such ID does not seem to exist in database @@ -72,13 +72,13 @@ class HoloUser: async def from_id(cls, user_id: int) -> "HoloUser": raise NotImplementedError() - async def _set(self, key: str, value: Any, cache: HoloCache | None = None) -> None: + async def _set(self, key: str, value: Any, cache: Optional[Cache] = None) -> None: """Set attribute data and save it into the database ### Args: * `key` (str): Attribute to be changed * `value` (Any): Value to set - * `cache` (HoloCache | None, optional): Cache engine to write the update into + * `cache` (Cache, optional): Cache engine to write the update into """ if not hasattr(self, key): raise AttributeError() @@ -93,12 +93,12 @@ class HoloUser: logger.info("Set attribute '%s' of user %s to '%s'", key, self.id, value) - async def _remove(self, key: str, cache: HoloCache | None = None) -> None: + async def _remove(self, key: str, cache: Optional[Cache] = None) -> None: """Remove attribute data and save it into the database ### Args: * `key` (str): Attribute to be removed - * `cache` (HoloCache | None, optional): Cache engine to write the update into + * `cache` (Cache, optional): Cache engine to write the update into """ if not hasattr(self, key): raise AttributeError() @@ -118,7 +118,7 @@ class HoloUser: def _get_cache_key(self) -> str: return f"user_{self.id}" - def _update_cache(self, cache: HoloCache | None = None) -> None: + def _update_cache(self, cache: Optional[Cache] = None) -> None: if cache is None: return @@ -129,7 +129,7 @@ class HoloUser: else: self._delete_cache(cache) - def _delete_cache(self, cache: HoloCache | None = None) -> None: + def _delete_cache(self, cache: Optional[Cache] = None) -> None: if cache is None: return @@ -159,26 +159,26 @@ class HoloUser: } async def set_custom_channel( - self, channel_id: int, cache: HoloCache | None = None + self, channel_id: int, cache: Optional[Cache] = None ) -> None: await self._set("custom_channel", channel_id, cache=cache) async def set_custom_role( - self, role_id: int, cache: HoloCache | None = None + self, role_id: int, cache: Optional[Cache] = None ) -> None: await self._set("custom_role", role_id, cache=cache) - async def remove_custom_channel(self, cache: HoloCache | None = None) -> None: + async def remove_custom_channel(self, cache: Optional[Cache] = None) -> None: await self._remove("custom_channel", cache=cache) - async def remove_custom_role(self, cache: HoloCache | None = None) -> None: + async def remove_custom_role(self, cache: Optional[Cache] = None) -> None: await self._remove("custom_role", cache=cache) - async def purge(self, cache: HoloCache | None = None) -> None: + async def purge(self, cache: Optional[Cache] = None) -> None: """Completely remove user data from database. Only removes the user record from users collection. ### Args: - * `cache` (HoloCache | None, optional): Cache engine to write the update into + * `cache` (Cache, optional): Cache engine to write the update into """ await col_users.delete_one({"_id": self._id}) self._delete_cache(cache) diff --git a/modules/cache_manager.py b/modules/cache_manager.py deleted file mode 100644 index 02d01fb..0000000 --- a/modules/cache_manager.py +++ /dev/null @@ -1,29 +0,0 @@ -from typing import Dict, Any, Literal - -from classes.cache.holo_cache_memcached import HoloCacheMemcached -from classes.cache.holo_cache_redis import HoloCacheRedis - - -def create_cache_client( - config: Dict[str, Any], - engine: Literal["memcached", "redis"] | None = None, -) -> HoloCacheMemcached | HoloCacheRedis: - if engine not in ["memcached", "redis"] or engine is None: - raise KeyError( - f"Incorrect cache engine provided. Expected 'memcached' or 'redis', got '{engine}'" - ) - - if "cache" not in config or engine not in config["cache"]: - raise KeyError( - f"Cache configuration is invalid. Please check if all keys are set (engine: '{engine}')" - ) - - match engine: - case "memcached": - return HoloCacheMemcached.from_config(config["cache"][engine]) - case "redis": - return HoloCacheRedis.from_config(config["cache"][engine]) - case _: - raise KeyError( - f"Cache implementation for the engine '{engine}' is not present." - ) diff --git a/modules/cache_utils.py b/modules/cache_utils.py deleted file mode 100644 index 2f95204..0000000 --- a/modules/cache_utils.py +++ /dev/null @@ -1,25 +0,0 @@ -from copy import deepcopy -from typing import Any - -from bson import ObjectId -from ujson import dumps, loads - - -def json_to_string(json_object: Any) -> str: - json_object_copy: Any = deepcopy(json_object) - - if isinstance(json_object_copy, dict) and "_id" in json_object_copy: - json_object_copy["_id"] = str(json_object_copy["_id"]) - - return dumps( - json_object_copy, ensure_ascii=False, indent=0, escape_forward_slashes=False - ) - - -def string_to_json(json_string: str) -> Any: - json_object: Any = loads(json_string) - - if "_id" in json_object: - json_object["_id"] = ObjectId(json_object["_id"]) - - return json_object diff --git a/modules/database.py b/modules/database.py index 97bf49a..fadd535 100644 --- a/modules/database.py +++ b/modules/database.py @@ -2,9 +2,6 @@ from typing import Dict, Any from async_pymongo import AsyncClient, AsyncCollection, AsyncDatabase from libbot.utils import config_get -from pymongo import MongoClient -from pymongo.synchronous.collection import Collection -from pymongo.synchronous.database import Database db_config: Dict[str, Any] = config_get("database") @@ -23,7 +20,6 @@ con_string: str = ( ) db_client: AsyncClient = AsyncClient(con_string) -db_client_sync: MongoClient = MongoClient(con_string) # Async declarations per default db: AsyncDatabase = db_client.get_database(name=db_config["name"]) @@ -31,11 +27,5 @@ db: AsyncDatabase = db_client.get_database(name=db_config["name"]) col_users: AsyncCollection = db.get_collection("users") col_analytics: AsyncCollection = db.get_collection("analytics") -# Sync declarations as a fallback -sync_db: Database = db_client_sync.get_database(name=db_config["name"]) - -sync_col_users: Collection = sync_db.get_collection("users") -sync_col_analytics: Collection = sync_db.get_collection("analytics") - # Update indexes -sync_col_users.create_index(["id"], unique=True) +db.dispatch.get_collection("users").create_index("id", unique=True) diff --git a/requirements.txt b/requirements.txt index f3937a3..0cd2539 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ requests>=2.32.2 aiofiles~=24.1.0 apscheduler>=3.10.0 async_pymongo==0.1.11 -libbot[speed,pycord]==4.0.2 +libbot[speed,pycord,cache]==4.1.0 mongodb-migrations==1.3.1 pymemcache~=4.0.0 redis~=5.2.1