Merge pull request 'Changed a few import strings' (#27) from dev into i18n

Reviewed-on: #27

classes/importer/abstract.py

@@ -0,0 +1,20 @@
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class Importer(ABC):
+    """
+    The Importer class represents the object with
+    functionality to import/export garbage collection
+    records and convert them to other object types.
+    """
+
+    @abstractmethod
+    async def import_data(self, data: Any) -> None:
+        pass
+
+    @abstractmethod
+    async def export_data(self, data: Any) -> None:
+        pass

classes/importer/csv.py

@@ -0,0 +1,64 @@
+from codecs import decode
+from csv import DictReader
+from datetime import datetime
+from typing import Any, Dict, List, Union
+
+from bson import ObjectId
+
+from classes.importer.abstract import Importer
+from modules.database import col_entries
+
+
+class ImporterCSV(Importer):
+    """
+    The ImporterCSV class represents the object with
+    functionality to import/export garbage collection
+    records and convert them to other object types
+    from CSV files.
+    """
+
+    def __init__(self):
+        super(Importer, self).__init__()
+
+    async def import_data(self, data: bytes) -> List[ObjectId]:
+        entries: List[Dict[str, Any]] = list(
+            DictReader(decode(data).split("\n"), delimiter=";")
+        )
+
+        for entry in entries:
+            entry["locations"] = (
+                [int(entry["locations"])]
+                if "," not in entry["locations"]
+                else [int(id) for id in entry["locations"].split(",")]
+            )
+            entry["garbage_type"] = int(entry["garbage_type"])
+
+            for key in ("locations", "garbage_type", "date"):
+                if (
+                    key not in entry
+                    or (key == "garbage_type" and not isinstance(entry[key], int))
+                    or (key == "locations" and not isinstance(entry[key], list))
+                ):
+                    raise ValueError
+
+                if key == "date":
+                    try:
+                        datetime.fromisoformat(str(entry[key]))
+                    except (ValueError, TypeError) as exc:
+                        raise ValueError from exc
+
+        entries_clean: List[Dict[str, Union[str, int, datetime]]] = [
+            {
+                "locations": entry["locations"],
+                "garbage_type": entry["garbage_type"],
+                "date": datetime.fromisoformat(str(entry["date"])),
+            }
+            for entry in entries
+        ]
+
+        inserted = await col_entries.insert_many(entries_clean)
+
+        return [] if inserted is None else inserted.inserted_ids
+
+    async def export_data(self, data: Any) -> Any:
+        return None

classes/importer/json.py

@@ -0,0 +1,56 @@
+from datetime import datetime
+from typing import Any, Dict, List, Union
+
+from bson import ObjectId
+from ujson import loads
+
+from classes.importer.abstract import Importer
+from modules.database import col_entries
+
+
+class ImporterJSON(Importer):
+    """
+    The ImporterJSON class represents the object with
+    functionality to import/export garbage collection
+    records and convert them to other object types
+    from JSON files.
+    """
+
+    def __init__(self):
+        super(Importer, self).__init__()
+
+    async def import_data(self, data: bytes) -> List[ObjectId]:
+        entries: List[Dict[str, Any]] = loads(data)
+
+        for entry in entries:
+            for key in ("locations", "garbage_type", "date"):
+                if (
+                    key not in entry
+                    or (key == "garbage_type" and not isinstance(entry[key], int))
+                    or (key == "locations" and not isinstance(entry[key], list))
+                ):
+                    print("keys", entry)
+                    raise ValueError
+
+                if key == "date":
+                    try:
+                        datetime.fromisoformat(str(entry[key]))
+                    except (ValueError, TypeError) as exc:
+                        print("date", entry)
+                        raise ValueError from exc
+
+        entries_clean: List[Dict[str, Union[str, int, datetime]]] = [
+            {
+                "locations": entry["locations"],
+                "garbage_type": entry["garbage_type"],
+                "date": datetime.fromisoformat(str(entry["date"])),
+            }
+            for entry in entries
+        ]
+
+        inserted = await col_entries.insert_many(entries_clean)
+
+        return [] if inserted is None else inserted.inserted_ids
+
+    async def export_data(self, data: Any) -> Any:
+        return None

config_example.json

@@ -1,5 +1,6 @@
 {
     "locale": "en",
+    "debug": false,
     "bot": {
         "owner": 0,
         "api_id": 0,

locale/en.json

@@ -42,9 +42,9 @@
         "help": "🔔 This bot sends you notifications about garbage collection according to your local schedule.\n\n**Available commands**\n/help - Show this message\n/setup - Select the location\n/toggle - Disable/enable the reminders\n/set_time - Set the reminders' time\n/set_offset - Set offset between reminders and collection\n/upcoming - Show the upcoming collection\n/language - Select the bot's language\n/checkout - Export or remove your data\n\n💭 You can also suggest adding your town/district to the bot by contacting the admins using [this link]({url_contact}) and providing your schedule.\n\n⚙️ Want to host this bot yourself or make some changes? It's open-source, so you can basically fork it. Take a look at [bot's repository]({url_repo}) for details.\n\nHappy using! 🤗",
         "import_finished": "You have successfully inserted {count} entries.",
         "import_invalid_date": "Entries contain invalid date formats. Use **ISO 8601** date format.",
-        "import_invalid_filetype": "Invalid input. Please, send me a JSON file with entries. {cancel_notice}",
+        "import_invalid_filetype": "Invalid input. Please, send me a JSON or CSV file with entries. {cancel_notice}",
-        "import_invalid": "This is not a valid garbage collection JSON.",
+        "import_invalid": "This is not a valid garbage collection file.",
-        "import": "Alright. Send me a valid JSON.",
+        "import": "Alright. Send me a valid file. It can be in JSON or CSV format. Read more about supported formats in the documentation",
         "locale_choice": "Alright. Please choose the language using keyboard below.",
         "location_empty": "You have no location set. Use /setup to select your location first.",
         "location_name_empty": "Could not find any locations by this name. Try rephrasing it or make sure you use the same location language and name itself as it in written by your local authorities in garbage collection schedule.\n\n{cancel_notice}",

main.py

@@ -12,7 +12,7 @@ from modules.migrator import migrate_database
 from modules.scheduler import scheduler
 
 logging.basicConfig(
-    level=logging.INFO,
+    level=logging.DEBUG if sync.config_get("debug") else logging.INFO,
     format="%(name)s.%(funcName)s | %(levelname)s | %(message)s",
     datefmt="[%X]",
 )

modules/reminder.py

@@ -1,6 +1,7 @@
 import logging
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timedelta
 
+from bson import json_util
 from libbot.pyrogram.classes import PyroClient
 
 from classes.enums import GarbageType
@@ -15,11 +16,21 @@ logger = logging.getLogger(__name__)
 async def remind(app: PyroClient) -> None:
     utcnow = datetime.utcnow()
 
+    logger.debug("Performing reminder lookup for %s (UTCNOW)", utcnow)
+
     users = await col_users.find(
         {"time_hour": utcnow.hour, "time_minute": utcnow.minute}
     ).to_list()
 
+    logger.debug(
+        "Found following reminders for %s (UTC NOW): %s",
+        utcnow,
+        json_util.dumps(users, indent=None),
+    )
+
     for user_db in users:
+        logger.debug("Processing user %s...", json_util.dumps(user_db, indent=None))
+
         user = await PyroUser.from_dict(**user_db)
 
         if not user.enabled or user.location is None:
@@ -31,20 +42,26 @@ async def remind(app: PyroClient) -> None:
             continue
 
         user_date = from_utc(
-            datetime(1970, 1, 1, user.time_hour, user.time_minute),
+            datetime.utcnow() + timedelta(days=user.offset),
             user.location.timezone.zone,
-        )
+        ).replace(hour=0, minute=0, second=0, microsecond=0)
 
         entries = await col_entries.find(
             {
                 "locations": location.id,
-                "date": user_date.replace(hour=0, minute=0),
+                "date": user_date,
             }
         ).to_list()
 
         logger.info("Entries of %s for %s: %s", user.id, user_date, entries)
 
         for entry in entries:
+            logger.debug(
+                "Sending %s notification about %s",
+                user.id,
+                json_util.dumps(entry, indent=None),
+            )
+
             try:
                 garbage_type = app._(
                     str(GarbageType(entry["garbage_type"]).value),

plugins/commands/import.py

@@ -1,33 +1,26 @@
-from datetime import datetime
-from typing import List, Mapping, Union
-
 from convopyro import listen_message
 from pyrogram import filters
 from pyrogram.types import ForceReply, Message, ReplyKeyboardRemove
-from ujson import loads
 
+from classes.importer.csv import ImporterCSV
+from classes.importer.json import ImporterJSON
 from classes.pyroclient import PyroClient
 from modules import custom_filters
-from modules.database import col_entries
 
 
 @PyroClient.on_message(
-    ~filters.scheduled & filters.private & custom_filters.owner & filters.command(["import"], prefixes=["/"]) & ~custom_filters.context  # type: ignore
+    ~filters.scheduled & filters.private & custom_filters.owner & filters.command(["import"], prefixes=["/"])  # type: ignore
 )
 async def command_import(app: PyroClient, message: Message):
     user = await app.find_user(message.from_user)
 
     await message.reply_text(
         app._("import", "messages", locale=user.locale),
-        reply_markup=ForceReply(
+        reply_markup=ForceReply(placeholder=""),
-            placeholder=app._("import", "force_replies", locale=user.locale)
-        ),
     )
 
     while True:
-        app.contexts.append(message.from_user.id)
         answer = await listen_message(app, message.chat.id, 300)
-        app.contexts.remove(message.from_user.id)
 
         if answer is None or answer.text == "/cancel":
             await message.reply_text(
@@ -36,7 +29,10 @@ async def command_import(app: PyroClient, message: Message):
             )
             return
 
-        if answer.document is None or answer.document.mime_type != "application/json":
+        if answer.document is None or answer.document.mime_type not in [
+            "application/json",
+            "text/csv",
+        ]:
             await answer.reply_text(
                 app._("import_invalid_filetype", "messages", locale=user.locale).format(
                     cancel_notice=app._("cancel", "messages", locale=user.locale)
@@ -48,51 +44,36 @@ async def command_import(app: PyroClient, message: Message):
 
     file = await app.download_media(answer, in_memory=True)
 
-    entries: List[Mapping[str, Union[str, int]]] = loads(bytes(file.getbuffer()))  # type: ignore
+    data: bytes = bytes(file.getbuffer())  # type: ignore
 
-    for entry in entries:
+    # I'd like to replace it with switch-case, but 3.9 compatibility
-        if not isinstance(entries, list):
+    # is still more important to be there. Although refactor may be
+    # done in the near future as Python 3.9 EOL gets nearer.
+    if answer.document.mime_type == "application/json":
+        importer = ImporterJSON()
+    elif answer.document.mime_type == "text/csv":
+        importer = ImporterCSV()
+    else:
         await answer.reply_text(
-                app._("import_invalid", "messages", locale=user.locale),
+            app._("import_invalid_filetype", "messages", locale=user.locale).format(
-                reply_markup=ReplyKeyboardRemove(),
+                cancel_notice=""
-            )
+            ),
-            return
+            reply_markup=ReplyKeyboardRemove(),
-
+        )
-        for key in ("locations", "garbage_type", "date"):
+        return
-            if (
+
-                key not in entry
+    try:
-                or (key == "garbage_type" and not isinstance(entry[key], int))
+        import_result = await importer.import_data(data)
-                or (key == "locations" and not isinstance(entry[key], list))
+    except ValueError:
-            ):
+        await answer.reply_text(
-                await answer.reply_text(
+            app._("import_invalid", "messages", locale=user.locale),
-                    app._("import_invalid", "messages", locale=user.locale),
+            reply_markup=ReplyKeyboardRemove(),
-                    reply_markup=ReplyKeyboardRemove(),
+        )
-                )
+        return
-                return
+
-            if key == "date":
+    await answer.reply_text(
-                try:
+        app._("import_finished", "messages", locale=user.locale).format(
-                    datetime.fromisoformat(str(entry[key]))
+            count=len(import_result)
-                except (ValueError, TypeError):
-                    await answer.reply_text(
-                        app._("import_invalid_date", "messages", locale=user.locale),
-                        reply_markup=ReplyKeyboardRemove(),
-                    )
-                    return
-
-    entries_clean: List[Mapping[str, Union[str, int, datetime]]] = [
-        {
-            "locations": entry["locations"],
-            "garbage_type": entry["garbage_type"],
-            "date": datetime.fromisoformat(str(entry["date"])),
-        }
-        for entry in entries
-    ]
-
-    await col_entries.insert_many(entries_clean)
-
-    await answer.reply_text(
-        app._("import_finished", "messages", locale=user.locale).format(
-            count=len(entries_clean)
         ),
         reply_markup=ReplyKeyboardRemove(),
     )

plugins/commands/set_time.py

@@ -55,9 +55,12 @@ async def command_set_time(app: PyroClient, message: Message):
 
         break
 
+    now = datetime.now()
+
     parsed_time = datetime.strptime(answer.text, "%H:%M").replace(
-        year=1970, month=1, day=1, second=0, microsecond=0
+        year=now.year, month=now.month, day=now.day, second=0, microsecond=0
     )
+
     user_time = to_utc(parsed_time, user.location.timezone.zone)
 
     await user.update_time(hour=user_time.hour, minute=user_time.minute)

requirements.txt

@@ -5,7 +5,7 @@ mongodb-migrations==1.3.0
 pykeyboard==0.1.5
 tgcrypto==1.2.5
 ujson>=5.0.0
-uvloop==0.17.0
+uvloop==0.19.0
 --extra-index-url https://git.end-play.xyz/api/packages/profitroll/pypi/simple
 async_pymongo==0.1.4
 libbot[speed,pyrogram]==2.0.1