Merge pull request 'Data export, warnings' improvements, bug fixes' (#35) from dev into master

Reviewed-on: profitroll/HoloCheckerBot#35

holochecker.py

@@ -39,10 +39,9 @@ from modules.callbacks.sub import *
 from modules.callbacks.sus import *
 from modules.callbacks.warnings import *
 
-from modules.handlers.analytics_group import *
 from modules.handlers.confirmation import *
 from modules.handlers.contact import *
-from modules.handlers.group_member_update import *
+from modules.handlers.group_join import *
 from modules.handlers.voice import *
 from modules.handlers.welcome import *
 from modules.handlers.everything import *

locale/uk.json

@@ -76,14 +76,12 @@
         "application_invalid_syntax": "Неправильний синтаксис!\nТреба: `/application ID/NAME/USERNAME`",
         "warned": "Попереджено **{0}** (`{1}`) про порушення правил",
         "warned_reason": "Попереджено **{0}** (`{1}`)\n\n**Причина:**\n{2}",
-        "warnings_1": "Користувач **{0}** (`{1}`) має **{2}** попередження\n\n{3}\n\nОбрати та зняти попередження:\n`/warnings {4} revoke`",
-        "warnings_2": "Користувач **{0}** (`{1}`) має **{2}** попереджень\n\n{3}\n\nОбрати та зняти попередження:\n`/warnings {4} revoke`",
+        "warnings_1": "Користувач **{0}** (`{1}`) має **{2}** попередження\n\nОбрати та зняти попередження:\n`/warnings {3} revoke`",
+        "warnings_2": "Користувач **{0}** (`{1}`) має **{2}** попереджень\n\nОбрати та зняти попередження:\n`/warnings {3} revoke`",
         "warnings_all": "**Список попереджень**\n\n{0}\n\nДля перегляду попереджень окремо взятого користувача слід використовувати `/warnings ID/NAME/USERNAME`",
         "warnings_entry": "• {0} (`{1}`)\n  Попереджень: {2}",
         "warnings_empty": "Щось тут порожньо...\nЗ іншого боку, це добре!",
         "warnings_revoke": "**Попередження {0}:**\n\n{1}\n\nБудь ласка, користуйтесь клавіатурою щоб зняти попередження з відповідним номером.",
-        "warning_revoked": "Попередження від {0} користувачеві `{1}` було скасовано адміном `{2}`",
-        "warning_revoked_auto": "Попередження від {0} користувачеві `{1}` було автоматично скасовано.",
         "no_warnings": "Користувач **{0}** (`{1}`) не має попереджень",
         "no_user_warnings": "Не знайдено користувачів за запитом **{0}**",
         "syntax_warnings": "Неправильний синтаксис!\nТреба: `/warnings ID/NAME/USERNAME`",
@@ -132,7 +130,6 @@
         "not_member": "❌ **Дія неможлива**\nУ тебе немає заповненої та схваленої анкети. Заповни таку за допомогою /reapply та спробуй ще раз після її підтвердження.",
         "issue": "**Допоможіть боту**\nЗнайшли баг або помилку? Маєте файну ідею для нової функції? Повідомте нас, створивши нову задачу на гіті.\n\nЗа можливості, опишіть свій запит максимально детально. Якщо є змога, також додайте скріншоти або додаткову відому інформацію.",
         "you_are_banned": "⚠️ **Вас було заблоковано**\nТепер не можна відправити анкету або користуватись командами бота.",
-        "user_left": "Користувач **{0}** залишив чат",
         "yes": "Так",
         "no": "Ні",
         "voice_message": [

modules/callbacks/warnings.py

@@ -3,8 +3,7 @@ from app import app
 from pyrogram import filters
 from pyrogram.types import CallbackQuery
 from pyrogram.client import Client
-from pykeyboard import InlineKeyboard, InlineButton
-from modules.utils import configGet, locale
+from modules.utils import locale
 from modules.database import col_warnings
 from bson import ObjectId
 
@@ -26,34 +25,3 @@ async def callback_query_warning_revoke(app: Client, clb: CallbackQuery):
         text=locale("warning_revoked", "callback", locale=clb.from_user).format(),
         show_alert=True,
     )
-    await app.send_message(
-        configGet("admin", "groups"),
-        locale("warning_revoked_auto", "message").format(
-            warning["user"], warning["date"].strftime("%d.%m.%Y")
-        ),
-    )
-    target_id = warning["user"]
-    if col_warnings.count_documents({"user": target_id, "active": True}) == 0:
-        await clb.edit_message_text(
-            locale("no_warnings", "message", locale=clb.from_user).format(
-                target_id, target_id
-            )
-        )
-        return
-    keyboard = InlineKeyboard()
-    buttons = []
-    warnings = []
-    for index, warning in enumerate(
-        list(col_warnings.find({"user": target_id, "active": True}))
-    ):
-        warnings.append(
-            f'{index+1}. {warning["date"].strftime("%d.%m.%Y, %H:%M")}\n  Адмін: {warning["admin"]}\n  Причина: {warning["reason"]}'
-        )
-        buttons.append(InlineButton(str(index + 1), f'w_rev_{str(warning["_id"])}'))
-    keyboard.add(*buttons)
-    await clb.edit_message_text(
-        locale("warnings_revoke", "message", locale=clb.from_user).format(
-            target_id, "\n".join(warnings)
-        ),
-    )
-    await clb.edit_message_reply_markup(reply_markup=keyboard)

modules/commands/message.py

@@ -63,12 +63,8 @@ async def cmd_message(app: Client, msg: Message):
             await msg.reply_text(
                 locale("message_enter", "message", locale=msg.from_user)
             )
-            message = await listen_message(app, msg.chat.id)
-            if (
-                message is None
-                or message.text is not None
-                and message.text == "/cancel"
-            ):
+            message = await listen_message(app, msg.chat.id, timeout=None)
+            if message.text is not None and message.text == "/cancel":
                 return
             sent = await app.forward_messages(
                 configGet("admin", "groups"), msg.chat.id, message.id

modules/commands/sponsorship.py

@@ -1,31 +1,11 @@
-from datetime import datetime, timedelta
-from typing import Union
 from app import app
 from pyrogram import filters
-from pyrogram.types import (
-    InlineKeyboardMarkup,
-    InlineKeyboardButton,
-    ReplyKeyboardMarkup,
-    ReplyKeyboardRemove,
-    ForceReply,
-    Message,
-)
+from pyrogram.types import InlineKeyboardMarkup, InlineKeyboardButton, Message
 from pyrogram.client import Client
 from classes.holo_user import HoloUser
 from modules import custom_filters
 from modules.utils import locale, should_quote
-from modules.database import col_sponsorships
-from convopyro import listen_message
-
-
-def is_none_or_cancel(message: Union[Message, None]) -> bool:
-    if (
-        message is None
-        or message.text is not None
-        and message.text.lower() == "/cancel"
-    ):
-        return True
-    return False
+from modules.database import col_applications
 
 
 @app.on_message(
@@ -46,168 +26,21 @@ async def cmd_sponsorship(app: Client, msg: Message):
     if holo_user.spoiler_state() is True:
         await msg.reply_text(locale("spoiler_in_progress", "message", locale=holo_user))
         return
-
-    existent = col_sponsorships.find_one(
-        {
-            "user": msg.from_user.id,
-            "sponsorship.expires": {"$gt": datetime.now() - timedelta(days=1)},
-        }
-    )
-
-    if existent is None:
-        await msg.reply_text(
-            locale("sponsorship_apply", "message", locale=msg.from_user),
-            reply_markup=InlineKeyboardMarkup(
-                [
-                    [
-                        InlineKeyboardButton(
-                            text=str(
-                                locale("sponsor_apply", "button", locale=msg.from_user)
-                            ),
-                            callback_data=f"sponsor_apply_{msg.from_user.id}",
-                        )
-                    ]
-                ]
-            ),
-            quote=should_quote(msg),
-        )
-        return
-
     await msg.reply_text(
-        f'You have an active membership for **{existent["sponsorship"]["streamer"]}**. Wanna resubmit it once more?',
-        reply_markup=ReplyKeyboardMarkup(
-            [["Yep, use old data"], ["Nope, refill it once more"]],
-            resize_keyboard=True,
-            one_time_keyboard=True,
-        ),
-    )
-
-    answer_decision = await listen_message(app, msg.chat.id)
-
-    if is_none_or_cancel(answer_decision):
-        return
-
-    input_streamer = existent["sponsorship"]["streamer"]
-
-    if answer_decision.text.lower() == "yep, use old data":
-        await answer_decision.reply_text(
-            "Okay, reusing the old data.\n\nUntil when is your sub?\n\nEnter the date as DD.MM.YYYY",
-            reply_markup=ForceReply(placeholder="Expiry date as DD.MM.YYYY"),
-        )
-        while True:
-            answer_date = await listen_message(app, msg.chat.id)
-
-            if is_none_or_cancel(answer_date):
-                return
-
-            try:
-                input_dt = datetime.strptime(answer_date.text, "%d.%m.%Y")
-                break
-            except ValueError:
-                await answer_date.reply_text(
-                    "Invalid date! Provide as DD.MM.YYYY",
-                    reply_markup=ForceReply(placeholder="Expiry date as DD.MM.YYYY"),
-                )
-                continue
-        while True:
-            await answer_date.reply_text(
-                "Alright. Now provide your proof **as a single screenshot**"
-            )
-            answer_proof = await listen_message(app, msg.chat.id)
-
-            if is_none_or_cancel(answer_proof):
-                return
-
-            if answer_proof.photo is None:
-                await answer_proof.reply_text(
-                    "Please, provide proof **as a single screenshot**"
-                )
-                continue
-            input_proof = answer_proof.photo.file_id
-            break
-        await msg.reply_text(
-            f'Almost done. Do you want to keep the label **{existent["sponsorship"]["label"]}** or you want to change it?',
-            reply_markup=ReplyKeyboardMarkup(
-                [["Keep the old one"], ["Set a new one instead"]],
-                resize_keyboard=True,
-                one_time_keyboard=True,
-            ),
-        )
-
-        while True:
-            answer_label_decision = await listen_message(app, msg.chat.id)
-
-            if is_none_or_cancel(answer_label_decision):
-                return
-
-            if answer_label_decision.text is None:
-                await answer_label_decision.reply_text(
-                    "Please, choose a valid option.",
-                    reply_markup=ReplyKeyboardMarkup(
-                        [["Keep the old one"], ["Set a new one instead"]],
-                        resize_keyboard=True,
-                        one_time_keyboard=True,
-                    ),
-                )
-                continue
-
-            if answer_label_decision.text.lower() == "keep the old one":
-                input_label = existent["sponsorship"]["label"]
-            elif answer_label_decision.text.lower() == "set a new one instead":
-                await answer_label_decision.reply_text(
-                    "Okay. Please provide a new label up to 16 characters long",
-                    reply_markup=ForceReply(placeholder="New label"),
-                )
-                while True:
-                    answer_label = await listen_message(app, msg.chat.id)
-
-                    if is_none_or_cancel(answer_label_decision):
-                        return
-
-                    if answer_label.text is None:
-                        await answer_label.reply_text(
-                            "Please provide valid label",
-                            reply_markup=ForceReply(placeholder="New label"),
-                        )
-                        continue
-                    elif len(answer_label.text) > 16:
-                        await answer_label.reply_text(
-                            "Please provide a label not longer than 16 characters long",
-                            reply_markup=ForceReply(placeholder="New label"),
-                        )
-                        continue
-
-                    input_label = answer_label.text
-                    break
-
-            await msg.reply_text(
-                f"So we did it for streamer **{input_streamer}**, til {input_dt.strftime('%d.%m.%Y')}, proofed by `{input_proof}` and labeled as **{input_label}**.",
-                reply_markup=ReplyKeyboardRemove(),
-            )
-            return
-
-    elif answer_decision.text.lower() == "nope, refill it once more":
-        await msg.reply_text(
-            locale("sponsorship_apply", "message", locale=msg.from_user),
-            reply_markup=InlineKeyboardMarkup(
+        locale("sponsorship_apply", "message", locale=msg.from_user),
+        reply_markup=InlineKeyboardMarkup(
+            [
                 [
-                    [
-                        InlineKeyboardButton(
-                            text=str(
-                                locale("sponsor_apply", "button", locale=msg.from_user)
-                            ),
-                            callback_data=f"sponsor_apply_{msg.from_user.id}",
-                        )
-                    ]
+                    InlineKeyboardButton(
+                        text=str(
+                            locale("sponsor_apply", "button", locale=msg.from_user)
+                        ),
+                        callback_data=f"sponsor_apply_{msg.from_user.id}",
+                    )
                 ]
-            ),
-            quote=should_quote(msg),
-        )
-        return
-    else:
-        await answer_decision.reply_text(
-            "Invalid option!", reply_markup=ReplyKeyboardRemove()
-        )
-        return
+            ]
+        ),
+        quote=should_quote(msg),
+    )
     # else:
     #     await msg.reply_text(locale("sponsorship_application_empty", "message"))

modules/commands/warnings.py

@@ -123,24 +123,17 @@ async def cmd_warnings(app: Client, msg: Message):
             quote=should_quote(msg),
         )
     else:
-        warnings = []
-        for index, warning in enumerate(
-            list(col_warnings.find({"user": target_id, "active": True}))
-        ):
-            warnings.append(
-                f'{index+1}. {warning["date"].strftime("%d.%m.%Y, %H:%M")}\n  Адмін: {warning["admin"]}\n  Причина: {warning["reason"]}'
-            )
         if warns <= 5:
             await msg.reply_text(
                 locale("warnings_1", "message", locale=msg.from_user).format(
-                    target_name, target_id, warns, "\n".join(warnings), target_id
+                    target_name, target_id, warns, target_id
                 ),
                 quote=should_quote(msg),
             )
         else:
             await msg.reply_text(
                 locale("warnings_2", "message", locale=msg.from_user).format(
-                    target_name, target_id, warns, "\n".join(warnings), target_id
+                    target_name, target_id, warns, target_id
                 ),
                 quote=should_quote(msg),
             )

modules/database.py

@@ -37,8 +37,6 @@ for collection in [
     "warnings",
     "applications",
     "sponsorships",
-    "analytics_group",
-    "analytics_users"
 ]:
     if not collection in collections:
         db.create_collection(collection)
@@ -53,7 +51,5 @@ col_messages = db.get_collection("messages")
 col_warnings = db.get_collection("warnings")
 col_applications = db.get_collection("applications")
 col_sponsorships = db.get_collection("sponsorships")
-col_analytics_group = db.get_collection("analytics_group")
-col_analytics_users = db.get_collection("analytics_users")
 
 col_applications.create_index([("application.3.location", GEOSPHERE)])

modules/handlers/analytics_group.py

@@ -1,255 +0,0 @@
-from datetime import datetime
-
-from polyglot.detect import Detector
-from pyrogram import filters
-from pyrogram.client import Client
-from pyrogram.enums import MessageEntityType, PollType
-from pyrogram.types import Message
-
-from app import app
-from modules import custom_filters
-from modules.database import col_analytics_group
-from modules.logging import logWrite
-from modules.utils import configGet
-
-
-@app.on_message(
-    custom_filters.enabled_general
-    & ~filters.scheduled
-    & filters.chat(configGet("users", "groups"))
-)
-async def msg_destination_group(app: Client, msg: Message):
-    analytics_entry = {
-        "id": msg.id,
-        "user": msg.from_user.id,
-        "date": datetime.now(),
-        "reply": {
-            "id": msg.reply_to_message_id,
-            "top_id": msg.reply_to_top_message_id,
-            "user": None
-            if msg.reply_to_message is None
-            else msg.reply_to_message.from_user.id,
-        },
-        "forward": {
-            "id": msg.forward_from_message_id,
-            "chat": None if msg.forward_from_chat is None else msg.forward_from_chat.id,
-            "user": None if msg.forward_from is None else msg.forward_from.id,
-            "date": msg.forward_date,
-        },
-        "media_spoilered": msg.has_media_spoiler,
-        "entities": {"links": [], "mentions": []},
-        "text": None,
-        "language": None,
-        "language_confidence": None,
-        "animation": None,
-        "audio": None,
-        "contact": None,
-        "document": None,
-        "location": None,
-        "photo": None,
-        "poll": None,
-        "sticker": None,
-        "venue": None,
-        "video": None,
-        "videonote": None,
-        "voice": None,
-    }
-
-    if msg.text is not None or msg.caption is not None:
-        text = msg.text if msg.text is not None else msg.caption
-        analytics_entry["text"] = text
-
-        if msg.entities is not None or msg.caption_entities is not None:
-            entities = (
-                msg.entities if msg.entities is not None else msg.caption_entities
-            )
-            for entity in entities:
-                if entity.type == MessageEntityType.TEXT_LINK:
-                    analytics_entry["entities"]["links"].append(entity.url)
-                elif entity.type == MessageEntityType.URL:
-                    analytics_entry["entities"]["links"].append(
-                        text[entity.offset : entity.offset + entity.length]
-                    )
-                elif entity.type == MessageEntityType.TEXT_MENTION:
-                    analytics_entry["entities"]["mentions"].append(entity.user.id)
-                elif entity.type == MessageEntityType.MENTION:
-                    analytics_entry["entities"]["mentions"].append(
-                        text[entity.offset : entity.offset + entity.length]
-                    )
-
-        lang = Detector(text, quiet=True).language
-
-        analytics_entry["language"] = lang.code
-        analytics_entry["language_confidence"] = lang.confidence
-
-        if lang.code == "ru":
-            logWrite(
-                f"Message '{text}' from {msg.from_user.first_name} ({msg.from_user.id}) is fucking russian [confidence {lang.confidence}]"
-            )
-
-    if msg.animation is not None:
-        analytics_entry["animation"] = {
-            "id": msg.animation.file_id,
-            "duration": msg.animation.duration,
-            "height": msg.animation.height,
-            "width": msg.animation.width,
-            "file_name": msg.animation.file_name,
-            "mime_type": msg.animation.mime_type,
-        }
-
-    if msg.audio is not None:
-        analytics_entry["audio"] = {
-            "id": msg.audio.file_id,
-            "title": msg.audio.title,
-            "performer": msg.audio.performer,
-            "duration": msg.audio.duration,
-            "file_name": msg.audio.file_name,
-            "file_size": msg.audio.file_size,
-            "mime_type": msg.audio.mime_type,
-        }
-
-    if msg.contact is not None:
-        analytics_entry["contact"] = {
-            "id": msg.contact.user_id,
-            "first_name": msg.contact.first_name,
-            "last_name": msg.contact.last_name,
-            "phone_number": msg.contact.phone_number,
-            "vcard": msg.contact.vcard,
-        }
-
-    if msg.document is not None:
-        analytics_entry["document"] = {
-            "id": msg.document.file_id,
-            "file_name": msg.document.file_name,
-            "file_size": msg.document.file_size,
-            "mime_type": msg.document.mime_type,
-        }
-
-    if msg.location is not None:
-        analytics_entry["location"] = {
-            "longitude": msg.location.longitude,
-            "latitude": msg.location.latitude,
-        }
-
-    if msg.photo is not None:
-        thumbnails = []
-        for thumbail in msg.photo.thumbs:
-            thumbnails.append(
-                {
-                    "id": thumbail.file_id,
-                    "height": thumbail.height,
-                    "width": thumbail.width,
-                    "file_size": thumbail.file_size,
-                }
-            )
-        analytics_entry["photo"] = {
-            "id": msg.photo.file_id,
-            "height": msg.photo.height,
-            "width": msg.photo.width,
-            "file_size": msg.photo.file_size,
-            "thumbnails": thumbnails,
-        }
-
-    if msg.poll is not None:
-        options = []
-        for option in msg.poll.options:
-            options.append(option.text)
-        analytics_entry["poll"] = {
-            "id": msg.poll.id,
-            "question": msg.poll.question,
-            "open_period": msg.poll.open_period,
-            "close_date": msg.poll.close_date,
-            "options": options,
-            "correct_option": msg.poll.correct_option_id,
-            "explanation": msg.poll.explanation,
-            "anonymous": msg.poll.is_anonymous,
-            "multiple_answers": msg.poll.allows_multiple_answers,
-            "quiz": True if msg.poll.type == PollType.QUIZ else False,
-        }
-
-    if msg.sticker is not None:
-        thumbnails = []
-        for thumbail in msg.sticker.thumbs:
-            thumbnails.append(
-                {
-                    "id": thumbail.file_id,
-                    "height": thumbail.height,
-                    "width": thumbail.width,
-                    "file_size": thumbail.file_size,
-                }
-            )
-        analytics_entry["sticker"] = {
-            "id": msg.sticker.file_id,
-            "emoji": msg.sticker.emoji,
-            "set_name": msg.sticker.set_name,
-            "animated": msg.sticker.is_animated,
-            "video": msg.sticker.is_video,
-            "height": msg.sticker.height,
-            "width": msg.sticker.width,
-            "file_name": msg.sticker.file_name,
-            "file_size": msg.sticker.file_size,
-            "mime_type": msg.sticker.mime_type,
-            "thumbnails": thumbnails,
-        }
-
-    if msg.venue is not None:
-        analytics_entry["venue"] = {
-            "title": msg.venue.title,
-            "address": msg.venue.address,
-            "longitude": msg.venue.location.longitude,
-            "latitude": msg.venue.location.latitude,
-            "foursquare_id": msg.venue.foursquare_id,
-            "foursquare_type": msg.venue.foursquare_type,
-        }
-
-    if msg.video is not None:
-        thumbnails = []
-        for thumbail in msg.video.thumbs:
-            thumbnails.append(
-                {
-                    "id": thumbail.file_id,
-                    "height": thumbail.height,
-                    "width": thumbail.width,
-                    "file_size": thumbail.file_size,
-                }
-            )
-        analytics_entry["video"] = {
-            "id": msg.video.file_id,
-            "duration": msg.video.duration,
-            "height": msg.video.height,
-            "width": msg.video.width,
-            "file_name": msg.video.file_name,
-            "file_size": msg.video.file_size,
-            "mime_type": msg.video.mime_type,
-            "thumbnails": thumbnails,
-        }
-
-    if msg.video_note is not None:
-        thumbnails = []
-        for thumbail in msg.video_note.thumbs:
-            thumbnails.append(
-                {
-                    "id": thumbail.file_id,
-                    "height": thumbail.height,
-                    "width": thumbail.width,
-                    "file_size": thumbail.file_size,
-                }
-            )
-        analytics_entry["video_note"] = {
-            "id": msg.video_note.file_id,
-            "duration": msg.video_note.duration,
-            "length": msg.video_note.length,
-            "file_size": msg.video_note.file_size,
-            "mime_type": msg.video_note.mime_type,
-            "thumbnails": thumbnails,
-        }
-
-    if msg.voice is not None:
-        analytics_entry["voice"] = {
-            "id": msg.voice.file_id,
-            "duration": msg.voice.duration,
-            "file_size": msg.voice.file_size,
-            "mime_type": msg.voice.mime_type,
-        }
-
-    col_analytics_group.insert_one(analytics_entry)

modules/handlers/group_join.py

@@ -149,16 +149,3 @@ async def filter_join(app: Client, member: ChatMemberUpdated):
                 can_send_polls=False,
             ),
         )
-        return
-
-    if member.new_chat_member is None:
-        await app.send_message(
-            configGet("users", "groups"),
-            locale("user_left", "message").format(
-                member.old_chat_member.user.first_name
-            ),
-        )
-        logWrite(
-            f"User {member.old_chat_member.user.first_name} ({member.old_chat_member.user.id}) left the destination group"
-        )
-        return

modules/scheduled.py

@@ -173,7 +173,7 @@ if configGet("enabled", "features", "sponsorships") is True:
                     try:
                         tg_user = await app.get_users(entry["user"])
                         logWrite(
-                            f"Notified user {entry['user']} that sponsorship expired"
+                            f"Notified user {entry['user']}  that sponsorship expired"
                         )
                     except Exception as exp:
                         logWrite(
@@ -224,12 +224,6 @@ if configGet("enabled", "features", "warnings") is True:
                     logWrite(
                         f'Revoked warning {str(warning["_id"])} of user {warning["user"]} because no active warnings for the last 90 days found.'
                     )
-                    await app.send_message(
-                        configGet("admin", "groups"),
-                        locale("warning_revoked_auto", "message").format(
-                            warning["user"], warning["date"].strftime("%d.%m.%Y")
-                        ),
-                    )
 
 
 # Register all bot commands

requirements.txt

@@ -5,11 +5,8 @@ convopyro==0.5
 fastapi~=0.95.0
 ftfy~=6.1.1
 psutil==5.9.4
-polyglot~=16.7.4
-PyICU==2.10.2
-pycld2==0.41
 pymongo==4.3.3
-Pyrogram~=2.0.103
+Pyrogram~=2.0.102
 python_dateutil==2.8.2
 pykeyboard==0.1.5
 requests==2.28.2