From aebe804b58cf37eff58c8de3860e770af241671a Mon Sep 17 00:00:00 2001 From: profitroll Date: Tue, 17 Jan 2023 14:11:23 +0100 Subject: [PATCH] Optimized for API --- config.json | 6 +-- locale/en.json | 3 ++ locale/uk.json | 3 ++ modules/api_client.py | 60 +++++++++++++++++++++++ modules/callbacks/submission.py | 25 ++++++++-- modules/sender.py | 86 ++++++++++++++++++++++----------- poster.py | 9 ++++ 7 files changed, 156 insertions(+), 36 deletions(-) create mode 100644 modules/api_client.py diff --git a/config.json b/config.json index 2ba707b..a580efa 100644 --- a/config.json +++ b/config.json @@ -25,6 +25,7 @@ "location": "logs" }, "locations": { + "tmp": "tmp", "data": "data", "cache": "cache", "sent": "data/sent", @@ -67,10 +68,7 @@ "address": "http://localhost:8054", "username": "", "password": "", - "albums": { - "queue": "poster_queue", - "sent": "poster_sent" - } + "album": "" } }, "caption": { diff --git a/locale/en.json b/locale/en.json index 7e447ba..3aeb26a 100644 --- a/locale/en.json +++ b/locale/en.json @@ -19,6 +19,9 @@ "sub_by": "\n\nSubmitted by:", "sub_sent": "Media has been submitted.\nWe'll notify you whether it will be accepted or not soon.", "sub_cooldown": "You can only submit 1 media per {0} seconds", + "sub_media_failed": "Could not upload submission to API. Please check logs for details.", + "sub_media_duplicates": "⚠️ Image duplicates found", + "sub_media_duplicates_list": "It seems like following image has duplicates in API's Db.\n\nNext files marked as similar:\n • {0}", "document_too_large": "File you've sent is too large. Please submit files not bigger than {0} MB", "mime_not_allowed": "File type not allowed. Please, consider using one of these: {0}", "post_exception": "Could not send content due to `{exp}`\n\nTraceback:\n```{0}```", diff --git a/locale/uk.json b/locale/uk.json index 378eb93..7d6717d 100644 --- a/locale/uk.json +++ b/locale/uk.json @@ -19,6 +19,9 @@ "sub_by": "\n\nПредставлено:", "sub_sent": "Медіа-файл надіслано.\nСкоро ми повідомимо вас, чи буде його прийнято.", "sub_cooldown": "Ви можете надсилати лише 1 медіафайл на {0} секунд", + "sub_media_failed": "Не вдалося завантажити подання на сервер. Перевірте логи для деталей.", + "sub_media_duplicates": "⚠️ Знайдено зображення-дублікати", + "sub_media_duplicates_list": "Здається, подане зображення має дублікати в базі даних.\n\nНаступні файли було відмічено як дуже схожі з поданням:\n • {0}", "document_too_large": "Надісланий файл завеликий. Будь ласка, надсилайте файли не більше {0} Мб", "mime_not_allowed": "Тип файлу не дозволений. Розгляньте можливість використання одного з цих: {0}", "post_exception": "Не вдалося надіслати контент через `{exp}`\n\nTraceback:\n```{0}```", diff --git a/modules/api_client.py b/modules/api_client.py new file mode 100644 index 0000000..86ee92b --- /dev/null +++ b/modules/api_client.py @@ -0,0 +1,60 @@ +from os import makedirs, path, sep +from base64 import b64encode, b64decode +from random import choice +from typing import Tuple +from requests import get, post, put, patch +from modules.utils import configGet + +def authorize() -> str: + makedirs(configGet("cache", "locations"), exist_ok=True) + if path.exists(configGet("cache", "locations")+sep+"api_access") is True: + with open(configGet("cache", "locations")+sep+"api_access", "rb") as file: + token = b64decode(file.read()).decode("utf-8") + if "user" in get(configGet("address", "posting", "api")+"/users/me/", headers={"Authorization": f"Bearer {token}"}).json(): + return token + payload = { + "grant_type": "password", + "scope": "me albums.list albums.read albums.write photos.list photos.read photos.write videos.list videos.read videos.write", + "username": configGet("username", "posting", "api"), + "password": configGet("password", "posting", "api") + } + response = post(configGet("address", "posting", "api")+"/token", data=payload).json() + with open(configGet("cache", "locations")+sep+"api_access", "wb") as file: + file.write(b64encode(response["access_token"].encode("utf-8"))) + return response["access_token"] + +def random_pic() -> Tuple[str, str]: + """Returns random image id and filename from the queue. + + ### Returns: + * `Tuple[str, str]`: First value is an ID and the filename in the filesystem to be indexed. + """ + token = authorize() + pic = choice(get(f'{configGet("address", "posting", "api")}/albums/{configGet("album", "posting", "api")}/photos?q=&page_size=100&caption=queue', headers={"Authorization": f"Bearer {token}"}).json()["results"]) + return pic["id"], pic["filename"] + +def upload_pic(filepath: str) -> bool: + token = authorize() + try: + pic_name = path.basename(filepath) + files = {'file': (pic_name, open(filepath, 'rb'), 'image/jpeg')} + response = post(f'{configGet("address", "posting", "api")}/albums/{configGet("album", "posting", "api")}/photos&caption=queue', headers={"Authorization": f"Bearer {token}"}, files=files).json() + print(response, flush=True) + duplicates = [] + if "duplicates" in response: + for duplicate in response["duplicates"]: + duplicates.append(f'{configGet("address", "posting", "api")}/photos/{duplicate["id"]}') + return True, duplicates + except: + return False, [] + +def move_pic(id: str) -> bool: + token = authorize() + try: + patch(f'{configGet("address", "posting", "api")}/photos/{id}?caption=sent', headers={"Authorization": f"Bearer {token}"}) + return True + except: + return False + +if __name__ == "__main__": + print(authorize()) \ No newline at end of file diff --git a/modules/callbacks/submission.py b/modules/callbacks/submission.py index 4f57ae6..6ac9464 100644 --- a/modules/callbacks/submission.py +++ b/modules/callbacks/submission.py @@ -3,6 +3,7 @@ from pathlib import Path from pyrogram import filters from pyrogram.client import Client from pyrogram.types import CallbackQuery +from modules.api_client import upload_pic from modules.utils import jsonLoad, jsonSave, configGet, locale from modules.submissions import subBlock, subUnblock from modules.app import app @@ -18,11 +19,25 @@ async def callback_query_yes(app: Client, clb: CallbackQuery): await clb.answer(text=locale("sub_msg_unavail", "message", locale=user_locale), show_alert=True) return try: - media = await app.download_media(submission, file_name=configGet("queue", "locations")+sep) - if clb.data.endswith("_caption"): - index = jsonLoad(configGet("index", "locations")) - index["captions"][Path(media).name] = submission.caption - jsonSave(index, configGet("index", "locations")) + if configGet("api_based", "mode") is True: + media = await app.download_media(submission, file_name=configGet("tmp", "locations")+sep) + upload = upload_pic(media) + if upload[0] is False: + await clb.answer(text=locale("sub_media_failed", "message", locale=user_locale), show_alert=True) + elif len(upload[1]) > 0: + await clb.answer(text=locale("sub_media_duplicates", "message", locale=user_locale)) + await clb.message.reply_text(locale("sub_media_duplicates_list", "message", locale=user_locale).format("\n • ".join(upload[1]))) + else: + if clb.data.endswith("_caption"): + index = jsonLoad(configGet("index", "locations")) + index["captions"][Path(media).name] = submission.caption + jsonSave(index, configGet("index", "locations")) + else: + media = await app.download_media(submission, file_name=configGet("queue", "locations")+sep) + if clb.data.endswith("_caption"): + index = jsonLoad(configGet("index", "locations")) + index["captions"][Path(media).name] = submission.caption + jsonSave(index, configGet("index", "locations")) except: await clb.answer(text=locale("sub_media_unavail", "message", locale=user_locale), show_alert=True) return diff --git a/modules/sender.py b/modules/sender.py index 000f55d..e5ba728 100644 --- a/modules/sender.py +++ b/modules/sender.py @@ -1,10 +1,12 @@ -from os import listdir, sep +from os import listdir, remove, sep from random import choice -from shutil import move +from shutil import copyfileobj, move +from requests import get from traceback import format_exc from pyrogram.client import Client from pyrogram.types import InlineKeyboardMarkup, InlineKeyboardButton from modules.logger import logWrite +from modules.api_client import authorize, move_pic, random_pic from modules.utils import jsonLoad, jsonSave, configGet, locale async def send_content(app: Client): @@ -13,40 +15,66 @@ async def send_content(app: Client): try: index = jsonLoad(configGet("index", "locations")) - list_queue = listdir(configGet("queue", "locations")) - for file in list_queue: + if configGet("api_based", "mode"): - if not file in index["sent"]: + try: + pic = random_pic() + except: + logWrite(locale("post_empty", "console", locale=configGet("locale"))) + if configGet("error", "reports"): + await app.send_message(configGet("admin"), locale("post_empty", "message", locale=configGet("locale"))) + return - ext_match = False + token = authorize() - for ext in configGet("photo", "posting", "extensions"): - if file.endswith(ext): - ext_match = True - ext_type = "photo" - break + response = get(f'{configGet("address", "posting", "api")}/photos/{pic[0]}', headers={"Authorization": f"Bearer {token}"}, stream=True) - for ext in configGet("video", "posting", "extensions"): - if file.endswith(ext): - ext_match = True - ext_type = "video" - break + with open(configGet("tmp", "locations")+sep+pic[0]+".jpg", 'wb') as out_file: + copyfileobj(response.raw, out_file) - if not ext_match: + del response + + candidate = configGet("tmp", "locations")+sep+pic[0]+".jpg" + candidate_file = pic[1] + ext_type = "photo" + + if not configGet("api_based", "mode"): + + list_queue = listdir(configGet("queue", "locations")) + + for file in list_queue: + + if not file in index["sent"]: + + ext_match = False + + for ext in configGet("photo", "posting", "extensions"): + if file.endswith(ext): + ext_match = True + ext_type = "photo" + break + + for ext in configGet("video", "posting", "extensions"): + if file.endswith(ext): + ext_match = True + ext_type = "video" + break + + if not ext_match: + list_queue.remove(file) + + else: list_queue.remove(file) + if len(list_queue) > 0: + candidate_file = choice(list_queue) + candidate = configGet("queue", "locations")+sep+candidate_file else: - list_queue.remove(file) - - if len(list_queue) > 0: - candidate_file = choice(list_queue) - candidate = configGet("queue", "locations")+sep+candidate_file - else: - logWrite(locale("post_empty", "console", locale=configGet("locale"))) - if configGet("error", "reports"): - await app.send_message(configGet("admin"), locale("post_empty", "message", locale=configGet("locale"))) - return + logWrite(locale("post_empty", "console", locale=configGet("locale"))) + if configGet("error", "reports"): + await app.send_message(configGet("admin"), locale("post_empty", "message", locale=configGet("locale"))) + return if candidate_file in index["captions"]: caption = index["captions"][candidate_file] @@ -84,6 +112,10 @@ async def send_content(app: Client): else: return + if configGet("api_based", "mode"): + remove(configGet("tmp", "locations")+sep+pic[0]+".jpg") + move_pic(pic[0]) + index["sent"].append(candidate_file) index["last_id"] = sent.id diff --git a/poster.py b/poster.py index 8d5295e..0192006 100644 --- a/poster.py +++ b/poster.py @@ -70,6 +70,7 @@ if args.norun: try: from modules.app import app from pyrogram import idle + from requests import get, post except ModuleNotFoundError: print(locale("deps_missing", "console", locale=configGet("locale")), flush=True) exit() @@ -122,6 +123,9 @@ if configGet("submit", "mode"): from modules.callbacks.submission import * from modules.commands.mode_submit import * from modules.handlers.submission import * + +if configGet("api_based", "mode"): + from modules.api_client import authorize #=========================================================================================================================================== # Work in progress @@ -147,6 +151,11 @@ if __name__ == "__main__": if configGet("post", "mode"): scheduler.start() + if configGet("api_based", "mode"): + token = authorize() + if len(get(f'{configGet("address", "posting", "api")}/albums?q={configGet("queue", "posting", "api", "albums")}', headers={"Authorization": f"Bearer {token}"}).json()["results"]) == 0: + post(f'{configGet("address", "posting", "api")}/albums?name={configGet("queue", "posting", "api", "albums")}&title={configGet("queue", "posting", "api", "albums")}', headers={"Authorization": f"Bearer {token}"}) + idle() app.send_message(configGet("admin"), locale("shutdown", "message", locale=configGet("locale")).format(str(pid)))