Improved logging

.gitignore

@@ -152,3 +152,4 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
 
+.vscode

PageSaver/package.json

@@ -1,5 +1,6 @@
 {
   "dependencies": {
-    "puppeteer": "^14.4.0"
+    "puppeteer": "~19.5.2",
+    "user-agents": "~1.0.1260"
   }
 }

PageSaver/pageSaver.js

@@ -1,12 +1,14 @@
 // npm install https://github.com/GoogleChrome/puppeteer/
 
 const puppeteer = require('puppeteer');
+const userAgent = require('user-agents');
 
 (async () => {
 
   const url = process.argv[2];
   const browser = await puppeteer.launch();
   const page = await browser.newPage();
+  await page.setUserAgent(userAgent.random().toString());
 
   await page.goto(url, {waitUntil: 'load'});
 

README.md

@@ -1,4 +1,6 @@
 # BWTAqua
+[![License: GPL v3](https://img.shields.io/badge/License-GPL_v3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0.html)  
+
 Simple yet helpful bot to check BWT Aqua's card balance
 
 ## Requirements
@@ -15,6 +17,7 @@ Simple yet helpful bot to check BWT Aqua's card balance
 3. Install PageSaver:
     1. `cd PageSaver`
     2. `npm install`
+    3. `chmod +x pageSaver` (If you want to use compiled page saver)
 4. Configure the bot:
     1. `cd ..`
     2. `nano config.json` (You can use any other text editor actually, for example `vim`)
@@ -22,5 +25,5 @@ Simple yet helpful bot to check BWT Aqua's card balance
     * `python3 bwtbot.py`
 
 ## Configuration
-You can edit with vim, nano, on Windows it's Notepad or Notepad++. Whatever.  
+You can edit with vim, nano, whatever.  
 If you don't know where to find bot_token and your id - here you can find some hints: [get bot token](https://www.siteguarding.com/en/how-to-get-telegram-bot-api-token), [get your id](https://www.alphr.com/telegram-find-user-id/), [get api_hash and api_id](https://core.telegram.org/api/obtaining_api_id).  

bwtbot.py

@@ -1,14 +1,15 @@
 #-*- coding: utf-8 -*-
 
-import traceback
+from os import getpid, system
-from pyrogram import Client, filters, idle
+from subprocess import call
-from pyrogram.types import ForceReply, BotCommand, BotCommandScopeChat
+from pyrogram import filters
+from pyrogram.client import Client
+from pyrogram.sync import idle
+from pyrogram.types import ForceReply, BotCommand, BotCommandScopeChat, Message
 from pyrogram.enums.chat_action import ChatAction
 from functions import *
 from modules.colors import *
 from modules.bwt import *
-import subprocess
-import os
 
 config = jsonLoad("config.json")
 
@@ -18,7 +19,7 @@ app = Client(config["bot_name"], api_id=config["api_id"], api_hash=config["api_h
 
 
 @app.on_message(~ filters.scheduled & filters.command(["setcard", "задать карту"], prefixes=["/", ""]))
-async def setcard(_, msg):
+async def setcard(_: Client, msg: Message):
     if userGet(msg.from_user.id, "context") is None:
         userSet(msg.from_user.id, "context", "set")
         await msg.reply_text(string("send_number"), reply_markup=ForceReply(placeholder=string("enter_number")))
@@ -27,7 +28,7 @@ async def setcard(_, msg):
 
 
 @app.on_message(~ filters.scheduled & filters.command(["cancel", "відміна"], prefixes=["/", ""]))
-async def cancel(_, msg):
+async def cancel(_: Client, msg: Message):
     if userGet(msg.from_user.id, "context") is not None:
         userReset(msg.from_user.id, "context")
         await msg.reply_text(string("cancel"))
@@ -36,7 +37,7 @@ async def cancel(_, msg):
 
 
 @app.on_message(~ filters.scheduled & filters.command(["resetcard", "забути картку"], prefixes=["/", ""]))
-async def resetcard(_, msg):
+async def resetcard(_: Client, msg: Message):
     if userGet(msg.from_user.id, "context") is None:
         if "card" in jsonLoad("data/database.json")[str(msg.from_user.id)]:
             userReset(msg.from_user.id, "card")
@@ -50,14 +51,15 @@ async def resetcard(_, msg):
 
 
 @app.on_message(~ filters.scheduled & filters.command(["balance", "баланс"], prefixes=["/", ""]))
-async def balance(_, msg):
+async def balance(_: Client, msg: Message):
     if userGet(msg.from_user.id, "context") is None:
         try:
             if "card" in jsonLoad("data/database.json")[str(msg.from_user.id)]:
                 await app.send_chat_action(chat_id=msg.chat.id, action=ChatAction.TYPING)
                 water_left = await getWaterLeft(userGet(msg.from_user.id, "card"), msg.from_user.id, app)
                 if water_left == "":
-                    raise EmptyCardException("Card information is empty")
+                    await msg.reply_text(string("error_new").format(f'https://bwtaqua.com.ua/card-topup/?id={userGet(msg.from_user.id, "card")}'))
+                    # raise EmptyCardException("Card information is empty")
                 elif water_left == "Failure":
                     await msg.reply_text(string("error_occured").format(string("get_number")))
                     appendLog(f"User {str(msg.from_user.id)} could not get left water amount")
@@ -68,15 +70,16 @@ async def balance(_, msg):
                 await msg.reply_text(string("card_not_linked").format(string("get_number")))
                 appendLog(f"User {str(msg.from_user.id)} tried to get balance without card set")
         except Exception as exp:
-            await msg.reply_text(string("error_occured").format(string("get_number")))
+            if msg.from_user.id != config["owner_id"]:
-            await app.send_message(owner_id, f"Error occured by {str(msg.from_user.id)}:\nException: `{exp}`\nTraceback: `{traceback.format_exc()}`")
+                await msg.reply_text(string("error_occured").format(string("get_number")))
+            await app.send_message(owner_id, f"Error occured by {str(msg.from_user.id)}:\nException: `{exp}`\nTraceback: `{format_exc()}`")
             appendLog(f"User {str(msg.from_user.id)} could not get left water amount")
     else:
         await msg.reply_text(string("cancel_first"))
 
 
 @app.on_message(~ filters.scheduled & filters.command(["topup", "refill", "поповнити"], prefixes=["/", ""]))
-async def topup_cmd(_, msg):
+async def topup_cmd(_: Client, msg: Message):
     if userGet(msg.from_user.id, "context") is None:
         try:
             if "card" in jsonLoad("data/database.json")[str(msg.from_user.id)]:
@@ -87,13 +90,13 @@ async def topup_cmd(_, msg):
                 await msg.reply_text(string("card_not_linked").format(string("get_number")))
                 appendLog(f"User {str(msg.from_user.id)} tried to request top up without card set")
         except Exception as exp:
-            await msg.reply_text(exp)
+            await msg.reply_text(str(exp))
     else:
         await msg.reply_text(string("cancel_first"))
 
 
 @app.on_message(~ filters.scheduled & filters.command(["start", "help", "допомога"], prefixes=["/", ""]))
-async def help(_, msg):
+async def help(_: Client, msg: Message):
     if userGet(msg.from_user.id, "context") is None:
         await msg.reply_text(string("welcome").format(string("get_number")))
         if msg.from_user.language_code in jsonLoad("strings.json"):
@@ -103,13 +106,13 @@ async def help(_, msg):
     else:
         await msg.reply_text(string("cancel_first"))
 
-pid = os.getpid()
+pid = getpid()
 
 @app.on_message(~ filters.scheduled & filters.command(["kill", "die", "shutdown"], prefixes="/"))
-async def kill(_, msg):
+async def kill(_: Client, msg: Message):
     if msg.from_user.id == owner_id:
         await msg.reply_text(f"Shutting down bot with pid **{pid}**")
-        os.system(f"kill -9 {pid}")
+        system(f"kill -9 {pid}")
 
 
 @app.on_message(~ filters.scheduled)
@@ -122,8 +125,8 @@ async def any_message_handler(app, msg):
 
 print(f'{nowtime()} {WHITE}Starting with PID {YELLOW}{pid}{RESET}')
 
-app.start()
+app.start() # type: ignore
-app.send_message(owner_id, f"Starting bot with pid **{pid}**")
+app.send_message(owner_id, f"Starting bot with pid **{pid}**") # type: ignore
 
 app.set_bot_commands([
     BotCommand("help", "Меню допомоги"),
@@ -133,7 +136,7 @@ app.set_bot_commands([
     BotCommand("resetcard", "Відв'язати картку"),
     BotCommand("cancel", "Відмінити операцію"),
     ],
-    language_code="uk")
+    language_code="uk") # type: ignore
 
 app.set_bot_commands([
     BotCommand("help", "Меню допомоги"),
@@ -143,7 +146,7 @@ app.set_bot_commands([
     BotCommand("resetcard", "Відв'язати картку"),
     BotCommand("cancel", "Відмінити операцію"),
     ],
-    language_code="ru")
+    language_code="ru") # type: ignore
 
 app.set_bot_commands([
     BotCommand("help", "Help menu"),
@@ -152,7 +155,7 @@ app.set_bot_commands([
     BotCommand("setcard", "Link card"),
     BotCommand("resetcard", "Unlink card"),
     BotCommand("cancel", "Cancel operation"),
-    ])
+    ]) # type: ignore
 
 app.set_bot_commands([
     BotCommand("help", "Help menu"),
@@ -163,11 +166,11 @@ app.set_bot_commands([
     BotCommand("shutdown", "Turn off the bot"),
     BotCommand("cancel", "Cancel operation"),
     ],
-    scope=BotCommandScopeChat(chat_id=owner_id))
+    scope=BotCommandScopeChat(chat_id=owner_id)) # type: ignore
 
 idle()
 
-app.send_message(owner_id, f"Shutting down bot with pid **{pid}**")
+app.send_message(owner_id, f"Shutting down bot with pid **{pid}**") # type: ignore
 print(f'\n{nowtime()} {WHITE}Shutting down with PID {YELLOW}{pid}{RESET}')
 
-subprocess.call(f'kill -9 {pid}', shell=True)
+call(f'kill -9 {pid}', shell=True)

config.json

@@ -4,5 +4,6 @@
     "api_id": 0,
     "api_hash": "",
     "bot_token": "",
-    "bot_name": ""
+    "bot_name": "",
+    "use_compiled_page_saver": false
 }

functions.py

@@ -1,32 +1,28 @@
-import json
+#-*- coding: utf-8 -*-
-import os
+
-import shutil
+from os import makedirs, stat
-import gzip
+from gzip import open as gzipopen
-import time
+from shutil import copyfileobj
+from time import sleep
+from ujson import loads, dumps
 from modules.colors import *
 from datetime import datetime
-from pathlib import Path
 
-path = Path(__file__).resolve().parent
+users_path = "users/"
-
+logs_folder = "logs/"
-days_path = str(path)+"/assets/days/"
-users_path = str(path)+"/users/"
-logs_folder = str(path)+"/logs/"
 
 
 def jsonSave(filename, value):
     with open(filename, 'w', encoding="utf-8") as f:
-        json.dump(value, f, indent=4, ensure_ascii=False)
+        f.write(dumps(value, indent=4, ensure_ascii=False))
-        f.close()
 
 def jsonLoad(filename):
     with open(filename, 'r', encoding="utf-8") as f:
-        value = json.load(f)
+        value = loads(f.read())
-        f.close()
     return value
 
 
-config = jsonLoad(f"{path}/config.json")
+config = jsonLoad(f"config.json")
 
 log_size = config["log_size"]
 owner_id = config["owner_id"]
@@ -42,12 +38,12 @@ def checkSize():
 
     while i < 2:
         try:
-            log = os.stat(logs_folder + 'latest.log')
+            log = stat(logs_folder + 'latest.log')
 
             if (log.st_size / 1024) > log_size:
                 with open(logs_folder + 'latest.log', 'rb') as f_in:
-                    with gzip.open(f'{logs_folder}{datetime.now().strftime("%d.%m.%Y_%H:%M:%S")}.zip', 'wb') as f_out:
+                    with gzipopen(f'{logs_folder}{datetime.now().strftime("%d.%m.%Y_%H:%M:%S")}.zip', 'wb') as f_out:
-                        shutil.copyfileobj(f_in, f_out)
+                        copyfileobj(f_in, f_out)
                             
                 open(logs_folder + 'latest.log', 'w').close()
                 
@@ -60,7 +56,7 @@ def checkSize():
                 open(logs_folder + 'latest.log', 'a').close()
             except:
                 try:
-                    os.mkdir(logs_folder)
+                    makedirs(logs_folder, exist_ok=True)
                     log = open(logs_folder + 'latest.log', 'a')
                     open(logs_folder + 'latest.log', 'a').close()
                 except:
@@ -78,14 +74,15 @@ def appendLog(message):
         open(logs_folder + 'latest.log', 'a').close()
     except:
         try:
-            os.mkdir(logs_folder)
+            makedirs(logs_folder, exist_ok=True)
             log = open(logs_folder + 'latest.log', 'a')
             open(logs_folder + 'latest.log', 'a').close()
         except:
-            time.sleep(2)
+            sleep(2)
             print('Log file could not be created')
             return
             
+    print(message, flush=True)
     log.write(f'[{datetime.now().strftime("%H:%M:%S | %d.%m.%Y")}] {message}\n')
         
     log.close()

modules/bwt.py

@@ -1,7 +1,9 @@
 #-*- coding: utf-8 -*-
 
-import os
+from os import makedirs, path
-import traceback
+from subprocess import check_output
+from traceback import format_exc
+from uuid import uuid4
 from functions import *
 from bs4 import BeautifulSoup
 
@@ -16,12 +18,21 @@ async def getWaterLeft(cardid, filename, app=None):
     
     try:
 
-        os.system(f'touch data/pages/{str(filename)}.html')
+        # if path.exists(f"data/pages/{str(filename)}.html") is False:
-        os.system(f'PageSaver/pageSaver "https://bwtaqua.com.ua/card-topup/?id={cardid}" > data/pages/{str(filename)}.html')
+        #     run(["touch", f"data/pages/{str(filename)}.html"])
 
-        with open(f'data/pages/{str(filename)}.html') as f:
+        appendLog(f"Trying to get liters for url '{url}'")
-            html_file = f.read()
+        
-            f.close()
+        if config["use_compiled_page_saver"] is True:
+            proc = check_output(["PageSaver/pageSaver", f"https://bwtaqua.com.ua/card-topup/?id={cardid}"]) #, ">", f"data/pages/{str(filename)}.html"])
+            html_file = proc.decode("utf-8")
+        else:
+            proc = check_output(["node", "./PageSaver/pageSaver.js", f"https://bwtaqua.com.ua/card-topup/?id={cardid}"]) #, ">", f"data/pages/{str(filename)}.html"])
+            html_file = proc.decode("utf-8")
+
+        # with open(f'data/pages/{str(filename)}.html') as f:
+        #     html_file = f.read()
+        #     f.close()
     
         soup = BeautifulSoup(html_file, 'html.parser')
 
@@ -33,10 +44,19 @@ async def getWaterLeft(cardid, filename, app=None):
         
         appendLog(f"Exception occured: {exp} (user: {str(filename)}, cardid: {cardid})")
 
+        try:
+            tmp_name = str(uuid4())
+            makedirs("tmp", exist_ok=True)
+            with open(path.join("tmp", tmp_name), "w", encoding="utf-8") as f:
+                f.write(html_file)
+        except NameError:
+            tmp_name = "N/A"
+            appendLog(f"'html_file' is not defined so I won't gather any tmp data")
+        
         if app != None:
-            await app.send_message(config["owner_id"], f"**Exception occured:**\n • User: `{str(filename)}`\n • Card: [{cardid}]({url})\n • Exception: `{exp}`\n • Traceback: `{traceback.format_exc()}`", disable_web_page_preview=True)
+            await app.send_message(config["owner_id"], f"**Exception occured:**\n • User: `{str(filename)}`\n • Card: [{cardid}]({url})\n • Exception: `{exp}`\n • TMP UUID: `{tmp_name}`\n • Traceback: `{format_exc()}`", disable_web_page_preview=True)
         else:
-            print(f'Exception occured and could not send to user: {exp}')
+            appendLog(f'Exception occured and could not send to user: {exp}')
         
         output = "Failure"
     

requirements.txt

@@ -1 +1,5 @@
-beautifulsoup4
+beautifulsoup4~=4.11.1
+pyrogram~=2.0.97
+pathlib~=1.0.1
+tgcrypto~=1.2.5
+ujson~=5.7.0

strings.json

@@ -6,6 +6,7 @@
         "card_unlinked": "Card was unlinked from your Telegram",
         "card_not_linked": "You don't have any linked card.\n\nВы можете задать её с помощью команды /setcard\n\n{0}",
         "error_occured": "An error occurred while getting the amount of remaining water on the card.\n\nPlease make sure the linked card number is correct. If you are sure that the bot is broken, please contact @profitroll.\n\nLink your card: /setcard\n\n{0}",
+        "error_new": "An error occurred while getting the amount of remaining water on the card.\n\nLast a few weeks BWT seems to return empty string to balance request from our server. We assume that our server has been blacklisted.\n\nTo check your balance you can use official [BWT App](https://bwtaqua.com.ua/en/#app) or simply bookmark this page: {0}.",
         "card_balance": "Card's balance is {0} l. of water",
         "top_up": "[Click here to top up](https://bwtaqua.com.ua/card-topup/?id={0})",
         "cancel": "Operation cancelled",
@@ -21,6 +22,7 @@
         "card_unlinked": "Картку відв'язано від вашого Telegram",
         "card_not_linked": "У вас немає прив'язаної картки.\n\nВи можете зробити це за допомогою команди /setcard\n\n{0}",
         "error_occured": "При отриманні води на карточці виникла помилка.\n\nБудь ласка, упевніться що номер карти правильний. Якщо ви впевнені, що номер картки правильний та бот зламався – зв'яжіться з @profitroll.\n\nПрив'язати карту: /setcard\n\n{0}",
+        "error_new": "При отриманні води на карточці виникла помилка.\n\nОстанні тижні BWT повертає нашому серверу порожні строки замість балансу. Є підозри, що сервер потрапив у блеклист.\n\nДля перевірки балансу рекомендуємо користуватись офіційним [додатком BWT](https://bwtaqua.com.ua/#app) або просто додати цю сторінку у закладки: {0}.",
         "card_balance": "На карточці {0} л. води",
         "top_up": "[Натисніть для поповнення](https://bwtaqua.com.ua/card-topup/?id={0})",
         "cancel": "Операція відмінена",