From a1bfbb537a9b7f1ec42819dbb08fe6844847a76e Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 16 Feb 2025 20:38:41 +0100 Subject: [PATCH] Replaced built-in caching with the one from libbot --- classes/cache/__init__.py | 3 -- classes/cache/cache.py | 44 ---------------- classes/cache/cache_memcached.py | 89 -------------------------------- classes/cache/cache_redis.py | 89 -------------------------------- classes/pycord_bot.py | 2 +- classes/pycord_user.py | 2 +- modules/cache_manager.py | 29 ----------- modules/cache_utils.py | 25 --------- requirements.txt | 6 +-- 9 files changed, 4 insertions(+), 285 deletions(-) delete mode 100644 classes/cache/__init__.py delete mode 100644 classes/cache/cache.py delete mode 100644 classes/cache/cache_memcached.py delete mode 100644 classes/cache/cache_redis.py delete mode 100644 modules/cache_manager.py delete mode 100644 modules/cache_utils.py diff --git a/classes/cache/__init__.py b/classes/cache/__init__.py deleted file mode 100644 index dce54a8..0000000 --- a/classes/cache/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .cache import Cache -from .cache_memcached import CacheMemcached -from .cache_redis import CacheRedis diff --git a/classes/cache/cache.py b/classes/cache/cache.py deleted file mode 100644 index 8b0f617..0000000 --- a/classes/cache/cache.py +++ /dev/null @@ -1,44 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Any, Dict - -import pymemcache -import redis - - -class Cache(ABC): - client: pymemcache.Client | redis.Redis - - @classmethod - @abstractmethod - def from_config(cls, engine_config: Dict[str, Any]) -> Any: - pass - - @abstractmethod - def get_json(self, key: str) -> Any | None: - # TODO This method must also carry out ObjectId conversion! - pass - - @abstractmethod - def get_string(self, key: str) -> str | None: - pass - - @abstractmethod - def get_object(self, key: str) -> Any | None: - pass - - @abstractmethod - def set_json(self, key: str, value: Any) -> None: - # TODO This method must also carry out ObjectId conversion! - pass - - @abstractmethod - def set_string(self, key: str, value: str) -> None: - pass - - @abstractmethod - def set_object(self, key: str, value: Any) -> None: - pass - - @abstractmethod - def delete(self, key: str) -> None: - pass diff --git a/classes/cache/cache_memcached.py b/classes/cache/cache_memcached.py deleted file mode 100644 index 81e91f9..0000000 --- a/classes/cache/cache_memcached.py +++ /dev/null @@ -1,89 +0,0 @@ -import logging -from logging import Logger -from typing import Dict, Any - -from pymemcache import Client - -from modules.cache_utils import string_to_json, json_to_string -from . import Cache - -logger: Logger = logging.getLogger(__name__) - - -class CacheMemcached(Cache): - client: Client - - def __init__(self, client: Client): - self.client = client - - logger.info("Initialized Memcached for caching") - - @classmethod - def from_config(cls, engine_config: Dict[str, Any]) -> "CacheMemcached": - if "uri" not in engine_config: - raise KeyError( - "Cache configuration is invalid. Please check if all keys are set (engine: memcached)" - ) - - return cls(Client(engine_config["uri"], default_noreply=True)) - - def get_json(self, key: str) -> Any | None: - try: - result: Any | None = self.client.get(key, None) - - logger.debug( - "Got json cache key '%s'%s", - key, - "" if result is not None else " (not found)", - ) - except Exception as exc: - logger.error("Could not get json cache key '%s' due to: %s", key, exc) - return None - - return None if result is None else string_to_json(result) - - def get_string(self, key: str) -> str | None: - try: - result: str | None = self.client.get(key, None) - - logger.debug( - "Got string cache key '%s'%s", - key, - "" if result is not None else " (not found)", - ) - - return result - except Exception as exc: - logger.error("Could not get string cache key '%s' due to: %s", key, exc) - return None - - # TODO Implement binary deserialization - def get_object(self, key: str) -> Any | None: - raise NotImplementedError() - - def set_json(self, key: str, value: Any) -> None: - try: - self.client.set(key, json_to_string(value)) - logger.debug("Set json cache key '%s'", key) - except Exception as exc: - logger.error("Could not set json cache key '%s' due to: %s", key, exc) - return None - - def set_string(self, key: str, value: str) -> None: - try: - self.client.set(key, value) - logger.debug("Set string cache key '%s'", key) - except Exception as exc: - logger.error("Could not set string cache key '%s' due to: %s", key, exc) - return None - - # TODO Implement binary serialization - def set_object(self, key: str, value: Any) -> None: - raise NotImplementedError() - - def delete(self, key: str) -> None: - try: - self.client.delete(key) - logger.debug("Deleted cache key '%s'", key) - except Exception as exc: - logger.error("Could not delete cache key '%s' due to: %s", key, exc) diff --git a/classes/cache/cache_redis.py b/classes/cache/cache_redis.py deleted file mode 100644 index 83fc022..0000000 --- a/classes/cache/cache_redis.py +++ /dev/null @@ -1,89 +0,0 @@ -import logging -from logging import Logger -from typing import Dict, Any - -from redis import Redis - -from classes.cache import Cache -from modules.cache_utils import string_to_json, json_to_string - -logger: Logger = logging.getLogger(__name__) - - -class CacheRedis(Cache): - client: Redis - - def __init__(self, client: Redis): - self.client = client - - logger.info("Initialized Redis for caching") - - @classmethod - def from_config(cls, engine_config: Dict[str, Any]) -> Any: - if "uri" not in engine_config: - raise KeyError( - "Cache configuration is invalid. Please check if all keys are set (engine: memcached)" - ) - - return cls(Redis.from_url(engine_config["uri"])) - - def get_json(self, key: str) -> Any | None: - try: - result: Any | None = self.client.get(key) - - logger.debug( - "Got json cache key '%s'%s", - key, - "" if result is not None else " (not found)", - ) - except Exception as exc: - logger.error("Could not get json cache key '%s' due to: %s", key, exc) - return None - - return None if result is None else string_to_json(result) - - def get_string(self, key: str) -> str | None: - try: - result: str | None = self.client.get(key) - - logger.debug( - "Got string cache key '%s'%s", - key, - "" if result is not None else " (not found)", - ) - - return result - except Exception as exc: - logger.error("Could not get string cache key '%s' due to: %s", key, exc) - return None - - # TODO Implement binary deserialization - def get_object(self, key: str) -> Any | None: - raise NotImplementedError() - - def set_json(self, key: str, value: Any) -> None: - try: - self.client.set(key, json_to_string(value)) - logger.debug("Set json cache key '%s'", key) - except Exception as exc: - logger.error("Could not set json cache key '%s' due to: %s", key, exc) - return None - - def set_string(self, key: str, value: str) -> None: - try: - self.client.set(key, value) - logger.debug("Set string cache key '%s'", key) - except Exception as exc: - logger.error("Could not set string cache key '%s' due to: %s", key, exc) - return None - - # TODO Implement binary serialization - def set_object(self, key: str, value: Any) -> None: - raise NotImplementedError() - - def delete(self, key: str) -> None: - try: - self.client.delete(key) - logger.debug("Deleted cache key '%s'", key) - except Exception as exc: - logger.error("Could not delete cache key '%s' due to: %s", key, exc) diff --git a/classes/pycord_bot.py b/classes/pycord_bot.py index 9e7717b..ed85b9e 100644 --- a/classes/pycord_bot.py +++ b/classes/pycord_bot.py @@ -2,10 +2,10 @@ from typing import Any from aiohttp import ClientSession from discord import User +from libbot.cache.manager import create_cache_client from libbot.pycord.classes import PycordBot as LibPycordBot from classes import PycordUser -from modules.cache_manager import create_cache_client # from modules.tracking.dhl import update_tracks_dhl diff --git a/classes/pycord_user.py b/classes/pycord_user.py index 9bf9252..34e5fb2 100644 --- a/classes/pycord_user.py +++ b/classes/pycord_user.py @@ -3,9 +3,9 @@ from dataclasses import dataclass from typing import Dict, Any, Optional from bson import ObjectId +from libbot.cache.classes import Cache from pymongo.results import InsertOneResult -from classes.cache import Cache from classes.errors.pycord_user import UserNotFoundError from modules.database import col_users diff --git a/modules/cache_manager.py b/modules/cache_manager.py deleted file mode 100644 index c852f8e..0000000 --- a/modules/cache_manager.py +++ /dev/null @@ -1,29 +0,0 @@ -from typing import Dict, Any, Literal - -from classes.cache.cache_memcached import CacheMemcached -from classes.cache.cache_redis import CacheRedis - - -def create_cache_client( - config: Dict[str, Any], - engine: Literal["memcached", "redis"] | None = None, -) -> CacheMemcached | CacheRedis: - if engine not in ["memcached", "redis"] or engine is None: - raise KeyError( - f"Incorrect cache engine provided. Expected 'memcached' or 'redis', got '{engine}'" - ) - - if "cache" not in config or engine not in config["cache"]: - raise KeyError( - f"Cache configuration is invalid. Please check if all keys are set (engine: '{engine}')" - ) - - match engine: - case "memcached": - return CacheMemcached.from_config(config["cache"][engine]) - case "redis": - return CacheRedis.from_config(config["cache"][engine]) - case _: - raise KeyError( - f"Cache implementation for the engine '{engine}' is not present." - ) diff --git a/modules/cache_utils.py b/modules/cache_utils.py deleted file mode 100644 index 2f95204..0000000 --- a/modules/cache_utils.py +++ /dev/null @@ -1,25 +0,0 @@ -from copy import deepcopy -from typing import Any - -from bson import ObjectId -from ujson import dumps, loads - - -def json_to_string(json_object: Any) -> str: - json_object_copy: Any = deepcopy(json_object) - - if isinstance(json_object_copy, dict) and "_id" in json_object_copy: - json_object_copy["_id"] = str(json_object_copy["_id"]) - - return dumps( - json_object_copy, ensure_ascii=False, indent=0, escape_forward_slashes=False - ) - - -def string_to_json(json_string: str) -> Any: - json_object: Any = loads(json_string) - - if "_id" in json_object: - json_object["_id"] = ObjectId(json_object["_id"]) - - return json_object diff --git a/requirements.txt b/requirements.txt index 253e85a..9ddcc01 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,10 +4,8 @@ async_pymongo==0.1.11 colorthief==0.2.1 deepl==1.21.0 fastapi[all]~=0.115.0 -libbot[speed,pycord]==4.1.0 +libbot[speed,pycord,cache]==4.1.0 mongodb-migrations==1.3.1 -pymemcache~=4.0.0 pynacl~=1.5.0 pyrmv==0.4.0 -pytz~=2025.1 -redis~=5.2.1 \ No newline at end of file +pytz~=2025.1 \ No newline at end of file