WIP: Database migrations

Dockerfile

@@ -24,4 +24,4 @@ USER appuser
 
 COPY . .
 
-ENTRYPOINT ["python", "main.py"]
+ENTRYPOINT ["python", "main.py", "--migrate"]

README.md

@@ -30,9 +30,27 @@
 3. Clone the repository:  
    `git clone https://git.end-play.xyz/HoloUA/Discord.git`
 4. `cd Discord`
-5. Install dependencies:  
+5. Create a virtual environment:  
+   `python -m venv .venv` or `virtualenv .venv`
+6. Activate the virtual environment:  
+   Windows: `.venv\Scripts\activate.bat`  
+   Linux/macOS: `.venv/bin/activate`
+7. Install the dependencies:  
    `python -m pip install -r requirements.txt`
-6. Run the bot with `python main.py` after completing the [configuration](#Configuration)
+8. Run the bot with `python main.py` after completing the [configuration](#Configuration)
+
+## Upgrading with Git
+
+1. Go to the bot's directory
+2. `git pull`
+3. Activate the virtual environment:  
+   Windows: `.venv\Scripts\activate.bat`  
+   Linux/macOS: `.venv/bin/activate`
+4. Update the dependencies:  
+   `python -m pip install -r requirements.txt`
+5. First start after the upgrade must initiate the migration:  
+   `python main.py --migrate`
+6. Now the bot is up to date and the next run will not require `--migrate` anymore
 
 ## Configuration
 

classes/holo_user.py

@@ -5,48 +5,75 @@ from typing import Any, Dict
 from bson import ObjectId
 from discord import User, Member
 from libbot.utils import config_get
+from pymongo.results import InsertOneResult
 
 from errors import UserNotFoundError
-from modules.database import col_warnings, sync_col_users, sync_col_warnings, col_users
+from modules.database import col_warnings, col_users
 
 logger: Logger = logging.getLogger(__name__)
 
 
 class HoloUser:
-    def __init__(self, user: User | Member | int) -> None:
+    def __init__(
+        self,
+        _id: ObjectId,
+        id: int,
+        custom_role: int | None,
+        custom_channel: int | None,
+    ) -> None:
+        self._id: ObjectId = _id
+
+        self.id: int = id
+        self.custom_role: int | None = custom_role
+        self.custom_channel: int | None = custom_channel
+
+    @classmethod
+    async def from_user(
+        cls, user: User | Member, allow_creation: bool = True
+    ) -> "HoloUser":
         """Get an object that has a proper binding between Discord ID and database
 
         ### Args:
-            * `user` (User | Member | int): Object from which ID can be extracted
+            * `user` (User | Member): Object from which an ID can be extracted
 
         ### Raises:
             * `UserNotFoundError`: User with such ID does not seem to exist in database
         """
+        db_entry: Dict[str, Any] | None = await col_users.find_one({"user": user.id})
 
-        self.id: int = user if not hasattr(user, "id") else user.id
+        if db_entry is None:
+            if not allow_creation:
+                raise UserNotFoundError(user=user, user_id=user.id)
 
-        jav_user: Dict[str, Any] | None = sync_col_users.find_one({"user": self.id})
+            db_entry = {
+                "user": user.id,
+                "custom_role": None,
+                "custom_channel": None,
+            }
 
-        if jav_user is None:
-            raise UserNotFoundError(user=user, user_id=self.id)
+            insert_result: InsertOneResult = await col_users.insert_one(db_entry)
 
-        self.db_id: ObjectId = jav_user["_id"]
+            db_entry["_id"] = insert_result.inserted_id()
 
-        self.customrole: int | None = jav_user["customrole"]
-        self.customchannel: int | None = jav_user["customchannel"]
-        self.warnings: int = self.warns()
+        db_entry["id"] = db_entry.pop("user")
 
-    def warns(self) -> int:
+        return cls(**db_entry)
+
+    @classmethod
+    async def from_id(cls, user_id: int) -> "HoloUser":
+        return NotImplemented
+
+    async def get_warnings(self) -> int:
         """Get number of warnings user has
 
         ### Returns:
             * `int`: Number of warnings
         """
-        warns: Dict[str, Any] | None = sync_col_warnings.find_one({"user": self.id})
+        warns: Dict[str, Any] | None = await col_warnings.find_one({"user": self.id})
 
         return 0 if warns is None else warns["warns"]
 
-    async def warn(self, count=1, reason: str = "Not provided") -> None:
+    async def warn(self, count: int = 1, reason: str = "Not provided") -> None:
         """Warn and add count to warns number
 
         ### Args:
@@ -56,7 +83,7 @@ class HoloUser:
 
         if warns is not None:
             await col_warnings.update_one(
-                {"_id": self.db_id},
+                {"_id": self._id},
                 {"$set": {"warns": warns["warns"] + count}},
             )
         else:
@@ -64,8 +91,8 @@ class HoloUser:
 
         logger.info("User %s was warned %s times due to: %s", self.id, count, reason)
 
-    async def set(self, key: str, value: Any) -> None:
-        """Set attribute data and save it into database
+    async def _set(self, key: str, value: Any) -> None:
+        """Set attribute data and save it into the database
 
         ### Args:
             * `key` (str): Attribute to be changed
@@ -77,11 +104,44 @@ class HoloUser:
         setattr(self, key, value)
 
         await col_users.update_one(
-            {"_id": self.db_id}, {"$set": {key: value}}, upsert=True
+            {"_id": self._id}, {"$set": {key: value}}, upsert=True
         )
 
         logger.info("Set attribute %s of user %s to %s", key, self.id, value)
 
+    async def _remove(self, key: str) -> None:
+        """Remove attribute data and save it into the database
+
+        ### Args:
+            * `key` (str): Attribute to be removed
+        """
+        if not hasattr(self, key):
+            raise AttributeError()
+
+        setattr(self, key, None)
+
+        await col_users.update_one(
+            {"_id": self._id}, {"$unset": {key: None}}, upsert=True
+        )
+
+        logger.info("Removed attribute %s of user %s", key, self.id)
+
+    async def set_custom_channel(self, channel_id: int) -> None:
+        await self._set("custom_channel", channel_id)
+
+    async def set_custom_role(self, role_id: int) -> None:
+        await self._set("custom_role", role_id)
+
+    async def remove_custom_channel(self) -> None:
+        await self._remove("custom_channel")
+
+    async def remove_custom_role(self) -> None:
+        await self._remove("custom_role")
+
+    async def purge(self) -> None:
+        """Completely remove user data from database. Will not remove transactions logs and warnings."""
+        await col_users.delete_one({"_id": self._id})
+
     @staticmethod
     async def is_moderator(member: User | Member) -> bool:
         """Check if user is moderator or council member

cogs/custom_channels.py

@@ -25,7 +25,7 @@ class CustomChannels(commands.Cog):
     @commands.Cog.listener()
     async def on_guild_channel_delete(self, channel: GuildChannel) -> None:
         await col_users.find_one_and_update(
-            {"customchannel": channel.id}, {"$set": {"customchannel": None}}
+            {"custom_channel": channel.id}, {"$set": {"custom_channel": None}}
         )
 
     custom_channel_group: SlashCommandGroup = SlashCommandGroup(
@@ -47,7 +47,7 @@ class CustomChannels(commands.Cog):
 
         Command to create a custom channel for a user.
         """
-        holo_user_ctx: HoloUser = HoloUser(ctx.user)
+        holo_user_ctx: HoloUser = await HoloUser.from_user(ctx.user)
 
         # Return if the user is using the command outside of a guild
         if not hasattr(ctx.author, "guild"):
@@ -62,7 +62,7 @@ class CustomChannels(commands.Cog):
             return
 
         # Return if the user already has a custom channel
-        if holo_user_ctx.customchannel is not None:
+        if holo_user_ctx.custom_channel is not None:
             await ctx.defer(ephemeral=True)
             await ctx.respond(
                 embed=Embed(
@@ -80,7 +80,7 @@ class CustomChannels(commands.Cog):
             reason=f"Користувач {guild_name(ctx.user)} отримав власний приватний канал",
             category=ds_utils.get(
                 ctx.author.guild.categories,
-                id=await config_get("customchannels", "categories"),
+                id=await config_get("custom_channels", "categories"),
             ),
         )
 
@@ -100,7 +100,7 @@ class CustomChannels(commands.Cog):
             manage_channels=True,
         )
 
-        await holo_user_ctx.set("customchannel", created_channel.id)
+        await holo_user_ctx.set_custom_channel(created_channel.id)
 
         await ctx.respond(
             embed=Embed(
@@ -136,10 +136,10 @@ class CustomChannels(commands.Cog):
 
         Command to change properties of a custom channel.
         """
-        holo_user_ctx: HoloUser = HoloUser(ctx.user)
+        holo_user_ctx: HoloUser = await HoloUser.from_user(ctx.user)
 
         custom_channel: TextChannel | None = ds_utils.get(
-            ctx.guild.channels, id=holo_user_ctx.customchannel
+            ctx.guild.channels, id=holo_user_ctx.custom_channel
         )
 
         # Return if the channel was not found
@@ -182,10 +182,10 @@ class CustomChannels(commands.Cog):
         """Command /customchannel remove [<confirm>]
 
         Command to remove a custom channel. Requires additional confirmation."""
-        holo_user_ctx: HoloUser = HoloUser(ctx.user)
+        holo_user_ctx: HoloUser = await HoloUser.from_user(ctx.user)
 
         # Return if the user does not have a custom channel
-        if holo_user_ctx.customchannel is None:
+        if holo_user_ctx.custom_channel is None:
             await ctx.defer(ephemeral=True)
             await ctx.respond(
                 embed=Embed(
@@ -199,7 +199,7 @@ class CustomChannels(commands.Cog):
         await ctx.defer()
 
         custom_channel: TextChannel | None = ds_utils.get(
-            ctx.guild.channels, id=holo_user_ctx.customchannel
+            ctx.guild.channels, id=holo_user_ctx.custom_channel
         )
 
         # Return if the channel was not found
@@ -211,7 +211,7 @@ class CustomChannels(commands.Cog):
                     color=Color.FAIL,
                 )
             )
-            await holo_user_ctx.set("customchannel", None)
+            await holo_user_ctx.remove_custom_channel()
             return
 
         # Return if the confirmation is missing
@@ -227,7 +227,7 @@ class CustomChannels(commands.Cog):
 
         await custom_channel.delete(reason="Власник запросив видалення")
 
-        await holo_user_ctx.set("customchannel", None)
+        await holo_user_ctx.remove_custom_channel()
 
         try:
             await ctx.respond(

cogs/logger.py

@@ -40,7 +40,7 @@ class Logger(commands.Cog):
             (message.type == MessageType.thread_created)
             and (message.channel is not None)
             and (
-                await col_users.count_documents({"customchannel": message.channel.id})
+                await col_users.count_documents({"custom_channel": message.channel.id})
                 > 0
             )
         ):

config_example.json

@@ -28,12 +28,12 @@
     },
     "defaults": {
         "user": {
-            "customrole": null,
-            "customchannel": null
+            "custom_role": null,
+            "custom_channel": null
         }
     },
     "categories": {
-        "customchannels": 0
+        "custom_channels": 0
     },
     "channels": {
         "text": {

main.py

@@ -1,5 +1,7 @@
+import contextlib
 import logging
 import sys
+from argparse import ArgumentParser
 from logging import Logger
 from pathlib import Path
 
@@ -7,6 +9,7 @@ from discord import LoginFailure, Intents
 from libbot.utils import config_get
 
 from classes.holo_bot import HoloBot
+from modules.migrator import migrate_database
 from modules.scheduler import scheduler
 
 logging.basicConfig(
@@ -17,12 +20,24 @@ logging.basicConfig(
 
 logger: Logger = logging.getLogger(__name__)
 
-try:
-    import uvloop  # type: ignore
+# Declare the parser that retrieves the command line arguments
+parser = ArgumentParser(
+    prog="HoloUA Discord",
+    description="Discord bot for the HoloUA community.",
+)
+
+# Add a switch argument --migrate to be parsed...
+parser.add_argument("--migrate", action="store_true")
+
+# ...and parse the arguments we added
+args = parser.parse_args()
+
+# Try to import the module that improves performance
+# and ignore errors when module is not installed
+with contextlib.suppress(ImportError):
+    import uvloop
 
     uvloop.install()
-except ImportError:
-    pass
 
 
 def main() -> None:
@@ -32,6 +47,10 @@ def main() -> None:
         )
         sys.exit()
 
+    # Perform migration if command line argument was provided
+    if args.migrate:
+        migrate_database()
+
     intents: Intents = Intents().all()
     client: HoloBot = HoloBot(intents=intents, scheduler=scheduler)
 

migrations/202412272043.py

@@ -0,0 +1,79 @@
+import logging
+from logging import Logger
+
+from libbot.utils import config_get, config_set, config_delete
+from mongodb_migrations.base import BaseMigration
+
+logger: Logger = logging.getLogger(__name__)
+
+
+class Migration(BaseMigration):
+    def upgrade(self):
+        try:
+            # Categories
+            config_set(
+                "custom_channels",
+                config_get("customchannels", "categories"),
+                "categories",
+            )
+            config_delete("customchannels", "categories")
+
+            # User defaults
+            config_delete(
+                "user",
+                "defaults",
+            )
+        except Exception as exc:
+            logger.error(
+                "Could not upgrade the config during migration '%s' due to: %s",
+                __name__,
+                exc,
+            )
+
+        self.db.groups.update_many(
+            {"customchannel": {"$exists": True}},
+            {"$rename": {"customchannel": "custom_channel"}},
+        )
+        self.db.groups.update_many(
+            {"customrole": {"$exists": True}},
+            {"$rename": {"customrole": "custom_role"}},
+        )
+
+    def downgrade(self):
+        try:
+            # Categories
+            config_set(
+                "customchannels",
+                config_get("custom_channels", "categories"),
+                "categories",
+            )
+            config_delete("custom_channels", "categories")
+
+            # User defaults
+            config_set(
+                "customrole",
+                None,
+                "defaults",
+                "user",
+            )
+            config_set(
+                "customchannel",
+                None,
+                "defaults",
+                "user",
+            )
+        except Exception as exc:
+            logger.error(
+                "Could not downgrade the config during migration '%s' due to: %s",
+                __name__,
+                exc,
+            )
+
+        self.db.test_collection.update_many(
+            {"custom_channel": {"$exists": True}},
+            {"$rename": {"custom_channel": "customchannel"}},
+        )
+        self.db.test_collection.update_many(
+            {"custom_role": {"$exists": True}},
+            {"$rename": {"custom_role": "customrole"}},
+        )

modules/migrator.py

@@ -0,0 +1,22 @@
+from typing import Any, Mapping
+
+from libbot.utils import config_get
+from mongodb_migrations.cli import MigrationManager
+from mongodb_migrations.config import Configuration
+
+
+def migrate_database() -> None:
+    """Apply migrations from folder `migrations/` to the database"""
+    database_config: Mapping[str, Any] = config_get("database")
+
+    manager_config = Configuration(
+        {
+            "mongo_host": database_config["host"],
+            "mongo_port": database_config["port"],
+            "mongo_database": database_config["name"],
+            "mongo_username": database_config["user"],
+            "mongo_password": database_config["password"],
+        }
+    )
+    manager = MigrationManager(manager_config)
+    manager.run()

requirements.txt

@@ -6,6 +6,7 @@ aiofiles~=24.1.0
 apscheduler>=3.10.0
 async_pymongo==0.1.11
 libbot[speed,pycord]==4.0.0
+mongodb-migrations==1.3.1
 typing-extensions~=4.12.2
 ujson~=5.10.0
 WaifuPicsPython==0.2.0

validation/users.json

@@ -2,21 +2,26 @@
     "$jsonSchema": {
         "required": [
             "user",
-            "customrole",
-            "customchannel"
-
+            "custom_role",
+            "custom_channel"
         ],
         "properties": {
             "user": {
                 "bsonType": "long",
                 "description": "Discord ID of user"
             },
-            "customrole": {
-                "bsonType": ["null", "long"],
+            "custom_role": {
+                "bsonType": [
+                    "null",
+                    "long"
+                ],
                 "description": "Discord ID of custom role or 'null' if not set"
             },
-            "customchannel": {
-                "bsonType": ["null", "long"],
+            "custom_channel": {
+                "bsonType": [
+                    "null",
+                    "long"
+                ],
                 "description": "Discord ID of custom channel or 'null' if not set"
             }
         }

validation/warnings.json

@@ -2,14 +2,14 @@
     "$jsonSchema": {
         "required": [
             "user",
-            "warns"
+            "warnings"
         ],
         "properties": {
             "user": {
                 "bsonType": "long",
                 "description": "Discord ID of user"
             },
-            "warns": {
+            "warnings": {
                 "bsonType": "int",
                 "description": "Number of warnings on count"
             }