From ac7e26b25fddd51d0d94644d53f590f31d5ee883 Mon Sep 17 00:00:00 2001 From: Renovate Date: Sat, 14 Oct 2023 00:28:56 +0300 Subject: [PATCH 1/5] Update dependency uvloop to v0.18.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 39040b9..f667897 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ mongodb-migrations==1.3.0 pykeyboard==0.1.5 tgcrypto==1.2.5 ujson>=5.0.0 -uvloop==0.17.0 +uvloop==0.18.0 --extra-index-url https://git.end-play.xyz/api/packages/profitroll/pypi/simple async_pymongo==0.1.4 libbot[speed,pyrogram]==2.0.1 \ No newline at end of file From 192602471b06e1bd2d65e673b088612f6bc634e8 Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 15 Oct 2023 22:20:12 +0200 Subject: [PATCH 2/5] Added debug key --- config_example.json | 1 + main.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/config_example.json b/config_example.json index 40c264a..cff73b1 100644 --- a/config_example.json +++ b/config_example.json @@ -1,5 +1,6 @@ { "locale": "en", + "debug": false, "bot": { "owner": 0, "api_id": 0, diff --git a/main.py b/main.py index af04103..6aa78c8 100644 --- a/main.py +++ b/main.py @@ -12,7 +12,7 @@ from modules.migrator import migrate_database from modules.scheduler import scheduler logging.basicConfig( - level=logging.INFO, + level=logging.DEBUG if sync.config_get("debug") else logging.INFO, format="%(name)s.%(funcName)s | %(levelname)s | %(message)s", datefmt="[%X]", ) From 842218859cd5118c0fadc41d8d5cd576b6186958 Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 15 Oct 2023 22:21:10 +0200 Subject: [PATCH 3/5] Yet another timezones fix --- modules/reminder.py | 25 +++++++++++++++++++++---- plugins/commands/set_time.py | 5 ++++- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/modules/reminder.py b/modules/reminder.py index a662e36..e4b5794 100644 --- a/modules/reminder.py +++ b/modules/reminder.py @@ -1,6 +1,7 @@ import logging -from datetime import datetime, timedelta, timezone +from datetime import datetime, timedelta +from bson import json_util from libbot.pyrogram.classes import PyroClient from classes.enums import GarbageType @@ -15,11 +16,21 @@ logger = logging.getLogger(__name__) async def remind(app: PyroClient) -> None: utcnow = datetime.utcnow() + logger.debug("Performing reminder lookup for %s (UTCNOW)", utcnow) + users = await col_users.find( {"time_hour": utcnow.hour, "time_minute": utcnow.minute} ).to_list() + logger.debug( + "Found following reminders for %s (UTC NOW): %s", + utcnow, + json_util.dumps(users, indent=None), + ) + for user_db in users: + logger.debug("Processing user %s...", json_util.dumps(user_db, indent=None)) + user = await PyroUser.from_dict(**user_db) if not user.enabled or user.location is None: @@ -31,20 +42,26 @@ async def remind(app: PyroClient) -> None: continue user_date = from_utc( - datetime(1970, 1, 1, user.time_hour, user.time_minute), + datetime.utcnow() + timedelta(days=user.offset), user.location.timezone.zone, - ) + ).replace(hour=0, minute=0, second=0, microsecond=0) entries = await col_entries.find( { "locations": location.id, - "date": user_date.replace(hour=0, minute=0), + "date": user_date, } ).to_list() logger.info("Entries of %s for %s: %s", user.id, user_date, entries) for entry in entries: + logger.debug( + "Sending %s notification about %s", + user.id, + json_util.dumps(entry, indent=None), + ) + try: garbage_type = app._( str(GarbageType(entry["garbage_type"]).value), diff --git a/plugins/commands/set_time.py b/plugins/commands/set_time.py index 5e4fb40..166b384 100644 --- a/plugins/commands/set_time.py +++ b/plugins/commands/set_time.py @@ -55,9 +55,12 @@ async def command_set_time(app: PyroClient, message: Message): break + now = datetime.now() + parsed_time = datetime.strptime(answer.text, "%H:%M").replace( - year=1970, month=1, day=1, second=0, microsecond=0 + year=now.year, month=now.month, day=now.day, second=0, microsecond=0 ) + user_time = to_utc(parsed_time, user.location.timezone.zone) await user.update_time(hour=user_time.hour, minute=user_time.minute) From b5143ed961bab60d4d96eb5947ea76f014e97c74 Mon Sep 17 00:00:00 2001 From: Renovate Date: Mon, 23 Oct 2023 02:09:08 +0300 Subject: [PATCH 4/5] Update dependency uvloop to v0.19.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f667897..9c10296 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,7 +5,7 @@ mongodb-migrations==1.3.0 pykeyboard==0.1.5 tgcrypto==1.2.5 ujson>=5.0.0 -uvloop==0.18.0 +uvloop==0.19.0 --extra-index-url https://git.end-play.xyz/api/packages/profitroll/pypi/simple async_pymongo==0.1.4 libbot[speed,pyrogram]==2.0.1 \ No newline at end of file From c9c2d298ef2cec11e0b76238a4ddd30389ec4d95 Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 29 Oct 2023 18:18:43 +0100 Subject: [PATCH 5/5] Work being done on #11 --- classes/importer/abstract.py | 20 +++++++++ classes/importer/csv.py | 64 +++++++++++++++++++++++++++ classes/importer/json.py | 56 ++++++++++++++++++++++++ locale/en.json | 6 +-- plugins/commands/import.py | 85 ++++++++++++++---------------------- 5 files changed, 176 insertions(+), 55 deletions(-) create mode 100644 classes/importer/abstract.py create mode 100644 classes/importer/csv.py create mode 100644 classes/importer/json.py diff --git a/classes/importer/abstract.py b/classes/importer/abstract.py new file mode 100644 index 0000000..096ec11 --- /dev/null +++ b/classes/importer/abstract.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Any + + +class Importer(ABC): + """ + The Importer class represents the object with + functionality to import/export garbage collection + records and convert them to other object types. + """ + + @abstractmethod + async def import_data(self, data: Any) -> None: + pass + + @abstractmethod + async def export_data(self, data: Any) -> None: + pass diff --git a/classes/importer/csv.py b/classes/importer/csv.py new file mode 100644 index 0000000..c876f92 --- /dev/null +++ b/classes/importer/csv.py @@ -0,0 +1,64 @@ +from codecs import decode +from csv import DictReader +from datetime import datetime +from typing import Any, Dict, List, Union + +from bson import ObjectId + +from classes.importer.abstract import Importer +from modules.database import col_entries + + +class ImporterCSV(Importer): + """ + The ImporterCSV class represents the object with + functionality to import/export garbage collection + records and convert them to other object types + from CSV files. + """ + + def __init__(self): + super(Importer, self).__init__() + + async def import_data(self, data: bytes) -> List[ObjectId]: + entries: List[Dict[str, Any]] = list( + DictReader(decode(data).split("\n"), delimiter=";") + ) + + for entry in entries: + entry["locations"] = ( + [int(entry["locations"])] + if "," not in entry["locations"] + else [int(id) for id in entry["locations"].split(",")] + ) + entry["garbage_type"] = int(entry["garbage_type"]) + + for key in ("locations", "garbage_type", "date"): + if ( + key not in entry + or (key == "garbage_type" and not isinstance(entry[key], int)) + or (key == "locations" and not isinstance(entry[key], list)) + ): + raise ValueError + + if key == "date": + try: + datetime.fromisoformat(str(entry[key])) + except (ValueError, TypeError) as exc: + raise ValueError from exc + + entries_clean: List[Dict[str, Union[str, int, datetime]]] = [ + { + "locations": entry["locations"], + "garbage_type": entry["garbage_type"], + "date": datetime.fromisoformat(str(entry["date"])), + } + for entry in entries + ] + + inserted = await col_entries.insert_many(entries_clean) + + return [] if inserted is None else inserted.inserted_ids + + async def export_data(self, data: Any) -> Any: + return None diff --git a/classes/importer/json.py b/classes/importer/json.py new file mode 100644 index 0000000..86afaad --- /dev/null +++ b/classes/importer/json.py @@ -0,0 +1,56 @@ +from datetime import datetime +from typing import Any, Dict, List, Union + +from bson import ObjectId +from ujson import loads + +from classes.importer.abstract import Importer +from modules.database import col_entries + + +class ImporterJSON(Importer): + """ + The ImporterJSON class represents the object with + functionality to import/export garbage collection + records and convert them to other object types + from JSON files. + """ + + def __init__(self): + super(Importer, self).__init__() + + async def import_data(self, data: bytes) -> List[ObjectId]: + entries: List[Dict[str, Any]] = loads(data) + + for entry in entries: + for key in ("locations", "garbage_type", "date"): + if ( + key not in entry + or (key == "garbage_type" and not isinstance(entry[key], int)) + or (key == "locations" and not isinstance(entry[key], list)) + ): + print("keys", entry) + raise ValueError + + if key == "date": + try: + datetime.fromisoformat(str(entry[key])) + except (ValueError, TypeError) as exc: + print("date", entry) + raise ValueError from exc + + entries_clean: List[Dict[str, Union[str, int, datetime]]] = [ + { + "locations": entry["locations"], + "garbage_type": entry["garbage_type"], + "date": datetime.fromisoformat(str(entry["date"])), + } + for entry in entries + ] + + inserted = await col_entries.insert_many(entries_clean) + + return [] if inserted is None else inserted.inserted_ids + + async def export_data(self, data: Any) -> Any: + return None diff --git a/locale/en.json b/locale/en.json index 1509a69..b555e9e 100644 --- a/locale/en.json +++ b/locale/en.json @@ -42,9 +42,9 @@ "help": "šŸ”” This bot sends you notifications about garbage collection according to your local schedule.\n\n**Available commands**\n/help - Show this message\n/setup - Select the location\n/toggle - Disable/enable the reminders\n/set_time - Set the reminders' time\n/set_offset - Set offset between reminders and collection\n/upcoming - Show the upcoming collection\n/language - Select the bot's language\n/checkout - Export or remove your data\n\nšŸ’­ You can also suggest adding your town/district to the bot by contacting the admins using [this link]({url_contact}) and providing your schedule.\n\nāš™ļø Want to host this bot yourself or make some changes? It's open-source, so you can basically fork it. Take a look at [bot's repository]({url_repo}) for details.\n\nHappy using! šŸ¤—", "import_finished": "You have successfully inserted {count} entries.", "import_invalid_date": "Entries contain invalid date formats. Use **ISO 8601** date format.", - "import_invalid_filetype": "Invalid input. Please, send me a JSON file with entries. {cancel_notice}", - "import_invalid": "This is not a valid garbage collection JSON.", - "import": "Alright. Send me a valid JSON.", + "import_invalid_filetype": "Invalid input. Please, send me a JSON or CSV file with entries. {cancel_notice}", + "import_invalid": "This is not a valid garbage collection file.", + "import": "Alright. Send me a valid file. It can be in JSON or CSV format. Read more about supported formats in the documentation", "locale_choice": "Alright. Please choose the language using keyboard below.", "location_empty": "You have no location set. Use /setup to select your location first.", "location_name_empty": "Could not find any locations by this name. Try rephrasing it or make sure you use the same location language and name itself as it in written by your local authorities in garbage collection schedule.\n\n{cancel_notice}", diff --git a/plugins/commands/import.py b/plugins/commands/import.py index 64c38e1..9a68c56 100644 --- a/plugins/commands/import.py +++ b/plugins/commands/import.py @@ -1,33 +1,26 @@ -from datetime import datetime -from typing import List, Mapping, Union - from convopyro import listen_message from pyrogram import filters from pyrogram.types import ForceReply, Message, ReplyKeyboardRemove -from ujson import loads +from classes.importer.csv import ImporterCSV +from classes.importer.json import ImporterJSON from classes.pyroclient import PyroClient from modules import custom_filters -from modules.database import col_entries @PyroClient.on_message( - ~filters.scheduled & filters.private & custom_filters.owner & filters.command(["import"], prefixes=["/"]) & ~custom_filters.context # type: ignore + ~filters.scheduled & filters.private & custom_filters.owner & filters.command(["import"], prefixes=["/"]) # type: ignore ) async def command_import(app: PyroClient, message: Message): user = await app.find_user(message.from_user) await message.reply_text( app._("import", "messages", locale=user.locale), - reply_markup=ForceReply( - placeholder=app._("import", "force_replies", locale=user.locale) - ), + reply_markup=ForceReply(placeholder=""), ) while True: - app.contexts.append(message.from_user.id) answer = await listen_message(app, message.chat.id, 300) - app.contexts.remove(message.from_user.id) if answer is None or answer.text == "/cancel": await message.reply_text( @@ -36,7 +29,10 @@ async def command_import(app: PyroClient, message: Message): ) return - if answer.document is None or answer.document.mime_type != "application/json": + if answer.document is None or answer.document.mime_type not in [ + "application/json", + "text/csv", + ]: await answer.reply_text( app._("import_invalid_filetype", "messages", locale=user.locale).format( cancel_notice=app._("cancel", "messages", locale=user.locale) @@ -48,51 +44,36 @@ async def command_import(app: PyroClient, message: Message): file = await app.download_media(answer, in_memory=True) - entries: List[Mapping[str, Union[str, int]]] = loads(bytes(file.getbuffer())) # type: ignore + data: bytes = bytes(file.getbuffer()) # type: ignore - for entry in entries: - if not isinstance(entries, list): - await answer.reply_text( - app._("import_invalid", "messages", locale=user.locale), - reply_markup=ReplyKeyboardRemove(), - ) - return + # I'd like to replace it with switch-case, but 3.9 compatibility + # is still more important to be there. Although refactor may be + # done in the near future as Python 3.9 EOL gets nearer. + if answer.document.mime_type == "application/json": + importer = ImporterJSON() + elif answer.document.mime_type == "text/csv": + importer = ImporterCSV() + else: + await answer.reply_text( + app._("import_invalid_filetype", "messages", locale=user.locale).format( + cancel_notice="" + ), + reply_markup=ReplyKeyboardRemove(), + ) + return - for key in ("locations", "garbage_type", "date"): - if ( - key not in entry - or (key == "garbage_type" and not isinstance(entry[key], int)) - or (key == "locations" and not isinstance(entry[key], list)) - ): - await answer.reply_text( - app._("import_invalid", "messages", locale=user.locale), - reply_markup=ReplyKeyboardRemove(), - ) - return - if key == "date": - try: - datetime.fromisoformat(str(entry[key])) - except (ValueError, TypeError): - await answer.reply_text( - app._("import_invalid_date", "messages", locale=user.locale), - reply_markup=ReplyKeyboardRemove(), - ) - return - - entries_clean: List[Mapping[str, Union[str, int, datetime]]] = [ - { - "locations": entry["locations"], - "garbage_type": entry["garbage_type"], - "date": datetime.fromisoformat(str(entry["date"])), - } - for entry in entries - ] - - await col_entries.insert_many(entries_clean) + try: + import_result = await importer.import_data(data) + except ValueError: + await answer.reply_text( + app._("import_invalid", "messages", locale=user.locale), + reply_markup=ReplyKeyboardRemove(), + ) + return await answer.reply_text( app._("import_finished", "messages", locale=user.locale).format( - count=len(entries_clean) + count=len(import_result) ), reply_markup=ReplyKeyboardRemove(), )