From e9f3237fbb91478b072bd27b6ec55532ce9b5803 Mon Sep 17 00:00:00 2001 From: Renovate Date: Fri, 30 Jun 2023 15:53:36 +0300 Subject: [PATCH 01/25] Update dependency opencv-python to ~=4.8.0.74 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index aa21c5b..026cb9a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ aiofiles==23.1.0 apscheduler~=3.10.1 exif==1.6.0 fastapi[all]==0.98.0 -opencv-python~=4.7.0.72 +opencv-python~=4.8.0.74 passlib~=1.7.4 pymongo==4.4.0 python-jose[cryptography]~=3.3.0 From 4545e26f32ae25409d460f13b4acebb0985bc2e5 Mon Sep 17 00:00:00 2001 From: Renovate Date: Sun, 2 Jul 2023 19:30:19 +0300 Subject: [PATCH 02/25] Update dependency fastapi to v0.99.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 026cb9a..00cfcc4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiofiles==23.1.0 apscheduler~=3.10.1 exif==1.6.0 -fastapi[all]==0.98.0 +fastapi[all]==0.99.1 opencv-python~=4.8.0.74 passlib~=1.7.4 pymongo==4.4.0 From 01b6222f6b574465b3da6bbe3d4c4fab5c08052c Mon Sep 17 00:00:00 2001 From: Renovate Date: Fri, 7 Jul 2023 21:00:24 +0300 Subject: [PATCH 03/25] Update dependency fastapi to v0.100.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 00cfcc4..f197933 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiofiles==23.1.0 apscheduler~=3.10.1 exif==1.6.0 -fastapi[all]==0.99.1 +fastapi[all]==0.100.0 opencv-python~=4.8.0.74 passlib~=1.7.4 pymongo==4.4.0 From e6fae5767929b5a9222ad917fa3360fcbd956880 Mon Sep 17 00:00:00 2001 From: Renovate Date: Fri, 14 Jul 2023 16:02:07 +0300 Subject: [PATCH 04/25] Update dependency pymongo to v4.4.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f197933..3cc8379 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ exif==1.6.0 fastapi[all]==0.100.0 opencv-python~=4.8.0.74 passlib~=1.7.4 -pymongo==4.4.0 +pymongo==4.4.1 python-jose[cryptography]~=3.3.0 python-magic~=0.4.27 scipy~=1.11.0 From 7c725bf04a26a51ade56a07e6ebdf0bd2bea6a67 Mon Sep 17 00:00:00 2001 From: Renovate Date: Thu, 27 Jul 2023 23:20:36 +0300 Subject: [PATCH 05/25] Update dependency fastapi to v0.100.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3cc8379..f22b617 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiofiles==23.1.0 apscheduler~=3.10.1 exif==1.6.0 -fastapi[all]==0.100.0 +fastapi[all]==0.100.1 opencv-python~=4.8.0.74 passlib~=1.7.4 pymongo==4.4.1 From 6cc0d3814e9a22d346f0f1d8a452e00ffbf3c285 Mon Sep 17 00:00:00 2001 From: Renovate Date: Sat, 5 Aug 2023 01:06:50 +0300 Subject: [PATCH 06/25] Update dependency fastapi to v0.101.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f22b617..ed03f06 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiofiles==23.1.0 apscheduler~=3.10.1 exif==1.6.0 -fastapi[all]==0.100.1 +fastapi[all]==0.101.0 opencv-python~=4.8.0.74 passlib~=1.7.4 pymongo==4.4.1 From 3b4d108d454b97d273318e7dfb3f87651bff4852 Mon Sep 17 00:00:00 2001 From: Renovate Date: Wed, 9 Aug 2023 18:50:11 +0300 Subject: [PATCH 07/25] Update dependency aiofiles to v23.2.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ed03f06..4927042 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -aiofiles==23.1.0 +aiofiles==23.2.1 apscheduler~=3.10.1 exif==1.6.0 fastapi[all]==0.101.0 From bcc7012744e02bd9ced366927580999a9a0c52d9 Mon Sep 17 00:00:00 2001 From: Renovate Date: Mon, 14 Aug 2023 13:12:49 +0300 Subject: [PATCH 08/25] Update dependency fastapi to v0.101.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4927042..3bc682e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiofiles==23.2.1 apscheduler~=3.10.1 exif==1.6.0 -fastapi[all]==0.101.0 +fastapi[all]==0.101.1 opencv-python~=4.8.0.74 passlib~=1.7.4 pymongo==4.4.1 From a1acaed6ddb4574150c847f9ec3a143c9accf235 Mon Sep 17 00:00:00 2001 From: profitroll Date: Mon, 14 Aug 2023 13:44:07 +0200 Subject: [PATCH 09/25] WIP: Migration to async_pymongo --- extensions/albums.py | 43 +++++++------- extensions/photos.py | 111 ++++++++++++++++++----------------- extensions/security.py | 2 +- extensions/users.py | 28 ++++----- extensions/videos.py | 94 +++++++++++++++-------------- modules/database.py | 11 +--- modules/exif_reader.py | 4 +- modules/extensions_loader.py | 17 ++++-- modules/hasher.py | 12 ++-- modules/mailer.py | 8 +-- modules/security.py | 32 ++++++---- modules/utils.py | 4 +- requirements.txt | 5 +- 13 files changed, 196 insertions(+), 175 deletions(-) diff --git a/extensions/albums.py b/extensions/albums.py index ab9273d..a0158fc 100644 --- a/extensions/albums.py +++ b/extensions/albums.py @@ -47,12 +47,12 @@ async def album_create( if 2 > len(title) > 40: raise AlbumIncorrectError("title", "must be >2 and <40 characters.") - if col_albums.find_one({"name": name}) is not None: + if (await col_albums.find_one({"name": name})) is not None: raise AlbumAlreadyExistsError(name) makedirs(Path(f"data/users/{current_user.user}/albums/{name}"), exist_ok=True) - uploaded = col_albums.insert_one( + uploaded = await col_albums.insert_one( {"user": current_user.user, "name": name, "title": title, "cover": None} ) @@ -67,9 +67,10 @@ async def album_find( current_user: User = Security(get_current_active_user, scopes=["albums.list"]), ): output = {"results": []} - albums = list(col_albums.find({"user": current_user.user, "name": re.compile(q)})) - for album in albums: + async for album in col_albums.find( + {"user": current_user.user, "name": re.compile(q)} + ): output["results"].append( { "id": album["_id"].__str__(), @@ -102,11 +103,11 @@ async def album_patch( current_user: User = Security(get_current_active_user, scopes=["albums.write"]), ): try: - album = col_albums.find_one({"_id": ObjectId(id)}) + album = await col_albums.find_one({"_id": ObjectId(id)}) if album is None: raise InvalidId(id) - except InvalidId: - raise AlbumNotFoundError(id) + except InvalidId as exc: + raise AlbumNotFoundError(id) from exc if title is None: title = album["title"] @@ -125,7 +126,7 @@ async def album_patch( Path(f"data/users/{current_user.user}/albums/{album['name']}"), Path(f"data/users/{current_user.user}/albums/{name}"), ) - col_photos.update_many( + await col_photos.update_many( {"user": current_user.user, "album": album["name"]}, {"$set": {"album": name}}, ) @@ -133,12 +134,14 @@ async def album_patch( name = album["name"] if cover is not None: - image = col_photos.find_one({"_id": ObjectId(cover), "album": album["name"]}) + image = await col_photos.find_one( + {"_id": ObjectId(cover), "album": album["name"]} + ) cover = image["_id"].__str__() if image is not None else album["cover"] else: cover = album["cover"] - col_albums.update_one( + await col_albums.update_one( {"_id": ObjectId(id)}, {"$set": {"name": name, "title": title, "cover": cover}} ) @@ -166,11 +169,11 @@ async def album_put( current_user: User = Security(get_current_active_user, scopes=["albums.write"]), ): try: - album = col_albums.find_one({"_id": ObjectId(id)}) + album = await col_albums.find_one({"_id": ObjectId(id)}) if album is None: raise InvalidId(id) - except InvalidId: - raise AlbumNotFoundError(id) + except InvalidId as exc: + raise AlbumNotFoundError(id) from exc if re.search(re.compile("^[a-z,0-9,_]*$"), name) is False: raise AlbumIncorrectError("name", "can only contain a-z, 0-9 and _ characters.") @@ -181,7 +184,7 @@ async def album_put( if 2 > len(title) > 40: raise AlbumIncorrectError("title", "must be >2 and <40 characters.") - image = col_photos.find_one({"_id": ObjectId(cover), "album": album["name"]}) + image = await col_photos.find_one({"_id": ObjectId(cover), "album": album["name"]}) cover = image["_id"].__str__() if image is not None else None # type: ignore rename( @@ -189,10 +192,10 @@ async def album_put( Path(f"data/users/{current_user.user}/albums/{name}"), ) - col_photos.update_many( + await col_photos.update_many( {"user": current_user.user, "album": album["name"]}, {"$set": {"album": name}} ) - col_albums.update_one( + await col_albums.update_one( {"_id": ObjectId(id)}, {"$set": {"name": name, "title": title, "cover": cover}} ) @@ -213,13 +216,13 @@ async def album_delete( current_user: User = Security(get_current_active_user, scopes=["albums.write"]), ): try: - album = col_albums.find_one_and_delete({"_id": ObjectId(id)}) + album = await col_albums.find_one_and_delete({"_id": ObjectId(id)}) if album is None: raise InvalidId(id) - except InvalidId: - raise AlbumNotFoundError(id) + except InvalidId as exc: + raise AlbumNotFoundError(id) from exc - col_photos.delete_many({"album": album["name"]}) + await col_photos.delete_many({"album": album["name"]}) rmtree(Path(f"data/users/{current_user.user}/albums/{album['name']}")) diff --git a/extensions/photos.py b/extensions/photos.py index aedf19e..3015bf4 100644 --- a/extensions/photos.py +++ b/extensions/photos.py @@ -122,7 +122,7 @@ async def photo_upload( caption: Union[str, None] = None, current_user: User = Security(get_current_active_user, scopes=["photos.write"]), ): - if col_albums.find_one({"user": current_user.user, "name": album}) is None: + if (await col_albums.find_one({"user": current_user.user, "name": album})) is None: raise AlbumNameNotFoundError(album) makedirs(Path(f"data/users/{current_user.user}/albums/{album}"), exist_ok=True) @@ -158,7 +158,7 @@ async def photo_upload( expires_delta=timedelta(hours=configGet("media_token_valid_hours")), ) access_token_short = uuid4().hex[:12].lower() - col_tokens.insert_one( + await col_tokens.insert_one( { "short": access_token_short, "access_token": access_token, @@ -183,7 +183,7 @@ async def photo_upload( except (UnpackError, ValueError): coords = {"lng": 0.0, "lat": 0.0, "alt": 0.0} - uploaded = col_photos.insert_one( + uploaded = await col_photos.insert_one( { "user": current_user.user, "album": album, @@ -231,7 +231,7 @@ if configGet("media_token_access") is True: responses=photo_get_token_responses, ) async def photo_get_token(token: str, id: int): - db_entry = col_tokens.find_one({"short": token}) + db_entry = await col_tokens.find_one({"short": token}) if db_entry is None: raise AccessTokenInvalidError() @@ -246,24 +246,23 @@ if configGet("media_token_access") is True: raise AccessTokenInvalidError() token_scopes = payload.get("scopes", []) token_data = TokenData(scopes=token_scopes, user=user) - except (JWTError, ValidationError) as exp: - print(exp, flush=True) - raise AccessTokenInvalidError() + except (JWTError, ValidationError) as exc: + raise AccessTokenInvalidError() from exc - user = get_user(user=token_data.user) + user_record = await get_user(user=token_data.user) if id not in payload.get("allowed", []): raise AccessTokenInvalidError() try: - image = col_photos.find_one({"_id": ObjectId(id)}) + image = await col_photos.find_one({"_id": ObjectId(id)}) if image is None: raise InvalidId(id) - except InvalidId: - raise PhotoNotFoundError(id) + except InvalidId as exc: + raise PhotoNotFoundError(id) from exc image_path = Path( - f"data/users/{user.user}/albums/{image['album']}/{image['filename']}" + f"data/users/{user_record.user}/albums/{image['album']}/{image['filename']}" ) mime = Magic(mime=True).from_file(image_path) @@ -301,11 +300,11 @@ async def photo_get( current_user: User = Security(get_current_active_user, scopes=["photos.read"]), ): try: - image = col_photos.find_one({"_id": ObjectId(id)}) + image = await col_photos.find_one({"_id": ObjectId(id)}) if image is None: raise InvalidId(id) - except InvalidId: - raise PhotoNotFoundError(id) + except InvalidId as exc: + raise PhotoNotFoundError(id) from exc image_path = Path( f"data/users/{current_user.user}/albums/{image['album']}/{image['filename']}" @@ -334,13 +333,13 @@ async def photo_move( current_user: User = Security(get_current_active_user, scopes=["photos.write"]), ): try: - image = col_photos.find_one({"_id": ObjectId(id)}) + image = await col_photos.find_one({"_id": ObjectId(id)}) if image is None: raise InvalidId(id) - except InvalidId: - raise PhotoNotFoundError(id) + except InvalidId as exc: + raise PhotoNotFoundError(id) from exc - if col_albums.find_one({"user": current_user.user, "name": album}) is None: + if (await col_albums.find_one({"user": current_user.user, "name": album})) is None: raise AlbumNameNotFoundError(album) if Path( @@ -354,7 +353,7 @@ async def photo_move( else: filename = image["filename"] - col_photos.find_one_and_update( + await col_photos.find_one_and_update( {"_id": ObjectId(id)}, { "$set": { @@ -396,13 +395,13 @@ async def photo_patch( current_user: User = Security(get_current_active_user, scopes=["photos.write"]), ): try: - image = col_photos.find_one({"_id": ObjectId(id)}) + image = await col_photos.find_one({"_id": ObjectId(id)}) if image is None: raise InvalidId(id) - except InvalidId: - raise PhotoNotFoundError(id) + except InvalidId as exc: + raise PhotoNotFoundError(id) from exc - col_photos.find_one_and_update( + await col_photos.find_one_and_update( {"_id": ObjectId(id)}, {"$set": {"caption": caption, "dates.modified": datetime.now(tz=timezone.utc)}}, ) @@ -430,16 +429,16 @@ async def photo_delete( current_user: User = Security(get_current_active_user, scopes=["photos.write"]), ): try: - image = col_photos.find_one_and_delete({"_id": ObjectId(id)}) + image = await col_photos.find_one_and_delete({"_id": ObjectId(id)}) if image is None: raise InvalidId(id) - except InvalidId: - raise PhotoNotFoundError(id) + except InvalidId as exc: + raise PhotoNotFoundError(id) from exc - album = col_albums.find_one({"name": image["album"]}) + album = await col_albums.find_one({"name": image["album"]}) if album is not None and album["cover"] == image["_id"].__str__(): - col_albums.update_one({"name": image["album"]}, {"$set": {"cover": None}}) + await col_albums.update_one({"name": image["album"]}, {"$set": {"cover": None}}) remove( Path( @@ -469,7 +468,7 @@ async def photo_random( limit: int = 100, current_user: User = Security(get_current_active_user, scopes=["photos.list"]), ): - if col_albums.find_one({"user": current_user.user, "name": album}) is None: + if (await col_albums.find_one({"user": current_user.user, "name": album})) is None: raise AlbumNameNotFoundError(album) if limit <= 0: @@ -490,20 +489,16 @@ async def photo_random( } ) - documents_count = col_photos.count_documents(db_query) + documents_count = await col_photos.count_documents(db_query) skip = randint(0, documents_count - 1) if documents_count > 1 else 0 - images = list( - col_photos.aggregate( - [ - {"$match": db_query}, - {"$skip": skip}, - {"$limit": limit}, - ] - ) - ) - - for image in images: + async for image in col_photos.aggregate( + [ + {"$match": db_query}, + {"$skip": skip}, + {"$limit": limit}, + ] + ): output["results"].append( { "id": image["_id"].__str__(), @@ -543,7 +538,7 @@ async def photo_find( current_user: User = Security(get_current_active_user, scopes=["photos.list"]), ): if token is not None: - found_record = col_tokens.find_one({"token": token}) + found_record = await col_tokens.find_one({"token": token}) if found_record is None: raise SearchTokenInvalidError() @@ -560,7 +555,7 @@ async def photo_find( current_user=current_user, ) - if col_albums.find_one({"user": current_user.user, "name": album}) is None: + if (await col_albums.find_one({"user": current_user.user, "name": album})) is None: raise AlbumNameNotFoundError(album) if page <= 0 or page_size <= 0: @@ -612,16 +607,22 @@ async def photo_find( "filename": re.compile(q), } else: - db_query = {"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)} # type: ignore - db_query_count = {"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)} # type: ignore + db_query = { + "user": current_user.user, + "album": album, + "filename": re.compile(q), + "caption": re.compile(caption), + } + db_query_count = { + "user": current_user.user, + "album": album, + "filename": re.compile(q), + "caption": re.compile(caption), + } - images = list( - col_photos.find(db_query, limit=page_size, skip=skip).sort( - "dates.uploaded", DESCENDING - ) - ) - - for image in images: + async for image in col_photos.find(db_query, limit=page_size, skip=skip).sort( + "dates.uploaded", DESCENDING + ): output["results"].append( { "id": image["_id"].__str__(), @@ -630,9 +631,9 @@ async def photo_find( } ) - if col_photos.count_documents(db_query_count) > page * page_size: + if (await col_photos.count_documents(db_query_count)) > page * page_size: token = str(token_urlsafe(32)) - col_tokens.insert_one( + await col_tokens.insert_one( { "token": token, "query": q, diff --git a/extensions/security.py b/extensions/security.py index 7b7db55..c2fb697 100644 --- a/extensions/security.py +++ b/extensions/security.py @@ -17,7 +17,7 @@ token_post_responses = {401: UserCredentialsInvalid().openapi} @app.post("/token", response_model=Token, responses=token_post_responses) async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends()): - user = authenticate_user(form_data.username, form_data.password) + user = await authenticate_user(form_data.username, form_data.password) if not user: raise UserCredentialsInvalid() access_token_expires = timedelta(days=ACCESS_TOKEN_EXPIRE_DAYS) diff --git a/extensions/users.py b/extensions/users.py index 4ae71a3..d53052e 100644 --- a/extensions/users.py +++ b/extensions/users.py @@ -41,14 +41,14 @@ async def send_confirmation(user: str, email: str): + f"/users/{user}/confirm?code={confirmation_code}" ), ) - col_emails.insert_one( + await col_emails.insert_one( {"user": user, "email": email, "used": False, "code": confirmation_code} ) logger.info( "Sent confirmation email to '%s' with code %s", email, confirmation_code ) - except Exception as exp: - logger.error("Could not send confirmation email to '%s' due to: %s", email, exp) + except Exception as exc: + logger.error("Could not send confirmation email to '%s' due to: %s", email, exc) @app.get("/users/me/", response_model=User) @@ -80,15 +80,15 @@ if configGet("registration_requires_confirmation") is True: responses=user_confirm_responses, ) async def user_confirm(user: str, code: str): - confirm_record = col_emails.find_one( + confirm_record = await col_emails.find_one( {"user": user, "code": code, "used": False} ) if confirm_record is None: raise UserEmailCodeInvalid() - col_emails.find_one_and_update( + await col_emails.find_one_and_update( {"_id": confirm_record["_id"]}, {"$set": {"used": True}} ) - col_users.find_one_and_update( + await col_users.find_one_and_update( {"user": confirm_record["user"]}, {"$set": {"disabled": False}} ) return UJSONResponse({"detail": configGet("email_confirmed", "messages")}) @@ -103,9 +103,9 @@ if configGet("registration_enabled") is True: async def user_create( user: str = Form(), email: str = Form(), password: str = Form() ): - if col_users.find_one({"user": user}) is not None: + if (await col_users.find_one({"user": user})) is not None: raise UserAlreadyExists() - col_users.insert_one( + await col_users.insert_one( { "user": user, "email": email, @@ -132,14 +132,14 @@ user_delete_responses = {401: UserCredentialsInvalid().openapi} async def user_delete( password: str = Form(), current_user: User = Depends(get_current_active_user) ): - user = get_user(current_user.user) + user = await get_user(current_user.user) if not user: return False if not verify_password(password, user.hash): raise UserCredentialsInvalid() - col_users.delete_many({"user": current_user.user}) - col_emails.delete_many({"user": current_user.user}) - col_photos.delete_many({"user": current_user.user}) - col_videos.delete_many({"user": current_user.user}) - col_albums.delete_many({"user": current_user.user}) + await col_users.delete_many({"user": current_user.user}) + await col_emails.delete_many({"user": current_user.user}) + await col_photos.delete_many({"user": current_user.user}) + await col_videos.delete_many({"user": current_user.user}) + await col_albums.delete_many({"user": current_user.user}) return Response(status_code=HTTP_204_NO_CONTENT) diff --git a/extensions/videos.py b/extensions/videos.py index 7e19997..031dcb9 100644 --- a/extensions/videos.py +++ b/extensions/videos.py @@ -50,7 +50,7 @@ async def video_upload( caption: Union[str, None] = None, current_user: User = Security(get_current_active_user, scopes=["videos.write"]), ): - if col_albums.find_one({"user": current_user.user, "name": album}) is None: + if (await col_albums.find_one({"user": current_user.user, "name": album})) is None: raise AlbumNameNotFoundError(album) makedirs(Path(f"data/users/{current_user.user}/albums/{album}"), exist_ok=True) @@ -73,7 +73,7 @@ async def video_upload( # Coords extraction should be here - uploaded = col_videos.insert_one( + uploaded = await col_videos.insert_one( { "user": current_user.user, "album": album, @@ -123,11 +123,11 @@ async def video_get( current_user: User = Security(get_current_active_user, scopes=["videos.read"]), ): try: - video = col_videos.find_one({"_id": ObjectId(id)}) + video = await col_videos.find_one({"_id": ObjectId(id)}) if video is None: raise InvalidId(id) - except InvalidId: - raise VideoNotFoundError(id) + except InvalidId as exc: + raise VideoNotFoundError(id) from exc video_path = Path( f"data/users/{current_user.user}/albums/{video['album']}/{video['filename']}" @@ -156,13 +156,13 @@ async def video_move( current_user: User = Security(get_current_active_user, scopes=["videos.write"]), ): try: - video = col_videos.find_one({"_id": ObjectId(id)}) + video = await col_videos.find_one({"_id": ObjectId(id)}) if video is None: raise InvalidId(id) - except InvalidId: - raise VideoNotFoundError(id) + except InvalidId as exc: + raise VideoNotFoundError(id) from exc - if col_albums.find_one({"user": current_user.user, "name": album}) is None: + if (await col_albums.find_one({"user": current_user.user, "name": album})) is None: raise AlbumNameNotFoundError(album) if Path( @@ -176,7 +176,7 @@ async def video_move( else: filename = video["filename"] - col_videos.find_one_and_update( + await col_videos.find_one_and_update( {"_id": ObjectId(id)}, { "$set": { @@ -218,13 +218,13 @@ async def video_patch( current_user: User = Security(get_current_active_user, scopes=["videos.write"]), ): try: - video = col_videos.find_one({"_id": ObjectId(id)}) + video = await col_videos.find_one({"_id": ObjectId(id)}) if video is None: raise InvalidId(id) - except InvalidId: - raise VideoNotFoundError(id) + except InvalidId as exc: + raise VideoNotFoundError(id) from exc - col_videos.find_one_and_update( + await col_videos.find_one_and_update( {"_id": ObjectId(id)}, {"$set": {"caption": caption, "dates.modified": datetime.now(tz=timezone.utc)}}, ) @@ -252,13 +252,13 @@ async def video_delete( current_user: User = Security(get_current_active_user, scopes=["videos.write"]), ): try: - video = col_videos.find_one_and_delete({"_id": ObjectId(id)}) + video = await col_videos.find_one_and_delete({"_id": ObjectId(id)}) if video is None: raise InvalidId(id) - except InvalidId: - raise VideoNotFoundError(id) + except InvalidId as exc: + raise VideoNotFoundError(id) from exc - album = col_albums.find_one({"name": video["album"]}) + album = await col_albums.find_one({"name": video["album"]}) remove( Path( @@ -288,7 +288,7 @@ async def video_random( limit: int = 100, current_user: User = Security(get_current_active_user, scopes=["videos.list"]), ): - if col_albums.find_one({"user": current_user.user, "name": album}) is None: + if (await col_albums.find_one({"user": current_user.user, "name": album})) is None: raise AlbumNameNotFoundError(album) if limit <= 0: @@ -309,20 +309,16 @@ async def video_random( } ) - documents_count = col_videos.count_documents(db_query) + documents_count = await col_videos.count_documents(db_query) skip = randint(0, documents_count - 1) if documents_count > 1 else 0 - videos = list( - col_videos.aggregate( - [ - {"$match": db_query}, - {"$skip": skip}, - {"$limit": limit}, - ] - ) - ) - - for video in videos: + async for video in col_videos.aggregate( + [ + {"$match": db_query}, + {"$skip": skip}, + {"$limit": limit}, + ] + ): output["results"].append( { "id": video["_id"].__str__(), @@ -359,7 +355,7 @@ async def video_find( current_user: User = Security(get_current_active_user, scopes=["videos.list"]), ): if token is not None: - found_record = col_tokens.find_one({"token": token}) + found_record = await col_tokens.find_one({"token": token}) if found_record is None: raise SearchTokenInvalidError() @@ -373,7 +369,7 @@ async def video_find( current_user=current_user, ) - if col_albums.find_one({"user": current_user.user, "name": album}) is None: + if (await col_albums.find_one({"user": current_user.user, "name": album})) is None: raise AlbumNameNotFoundError(album) if page <= 0 or page_size <= 0: @@ -410,16 +406,28 @@ async def video_find( "caption": re.compile(q), } else: - db_query = list(col_videos.find({"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)}, limit=page_size, skip=skip).sort("dates.uploaded", DESCENDING)) # type: ignore - db_query_count = {"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)} # type: ignore - - videos = list( - col_videos.find(db_query, limit=page_size, skip=skip).sort( - "dates.uploaded", DESCENDING + db_query = list( + col_videos.find( + { + "user": current_user.user, + "album": album, + "filename": re.compile(q), + "caption": re.compile(caption), + }, + limit=page_size, + skip=skip, + ).sort("dates.uploaded", DESCENDING) ) - ) + db_query_count = { + "user": current_user.user, + "album": album, + "filename": re.compile(q), + "caption": re.compile(caption), + } - for video in videos: + async for video in col_videos.find(db_query, limit=page_size, skip=skip).sort( + "dates.uploaded", DESCENDING + ): output["results"].append( { "id": video["_id"].__str__(), @@ -428,9 +436,9 @@ async def video_find( } ) - if col_videos.count_documents(db_query_count) > page * page_size: + if (await col_videos.count_documents(db_query_count)) > page * page_size: token = str(token_urlsafe(32)) - col_tokens.insert_one( + await col_tokens.insert_one( { "token": token, "query": q, diff --git a/modules/database.py b/modules/database.py index 79346f1..65fe87d 100644 --- a/modules/database.py +++ b/modules/database.py @@ -1,4 +1,5 @@ -from pymongo import GEOSPHERE, MongoClient +from async_pymongo import AsyncClient +from pymongo import GEOSPHERE from modules.utils import configGet @@ -17,16 +18,10 @@ else: db_config["host"], db_config["port"], db_config["name"] ) -db_client = MongoClient(con_string) +db_client = AsyncClient(con_string) db = db_client.get_database(name=db_config["name"]) -collections = db.list_collection_names() - -for collection in ["users", "albums", "photos", "videos", "tokens", "emails"]: - if collection not in collections: - db.create_collection(collection) - col_users = db.get_collection("users") col_albums = db.get_collection("albums") col_photos = db.get_collection("photos") diff --git a/modules/exif_reader.py b/modules/exif_reader.py index 8eb810b..2f74429 100644 --- a/modules/exif_reader.py +++ b/modules/exif_reader.py @@ -1,4 +1,6 @@ import contextlib +from pathlib import Path +from typing import Mapping, Union from exif import Image @@ -21,7 +23,7 @@ def decimal_coords(coords: float, ref: str) -> float: return round(decimal_degrees, 5) -def extract_location(filepath: str) -> dict: +def extract_location(filepath: Union[str, Path]) -> Mapping[str, float]: """Get location data from image ### Args: diff --git a/modules/extensions_loader.py b/modules/extensions_loader.py index fd11f58..8c27745 100644 --- a/modules/extensions_loader.py +++ b/modules/extensions_loader.py @@ -1,6 +1,7 @@ from importlib.util import module_from_spec, spec_from_file_location from os import getcwd, path, walk from pathlib import Path +from typing import Union # ================================================================================= @@ -17,11 +18,15 @@ def get_py_files(src): return py_files -def dynamic_import(module_name, py_path): +def dynamic_import(module_name: str, py_path: str): try: module_spec = spec_from_file_location(module_name, py_path) - module = module_from_spec(module_spec) # type: ignore - module_spec.loader.exec_module(module) # type: ignore + if module_spec is None: + raise RuntimeError( + f"Module spec from module name {module_name} and path {py_path} is None" + ) + module = module_from_spec(module_spec) + module_spec.loader.exec_module(module) return module except SyntaxError: print( @@ -29,12 +34,12 @@ def dynamic_import(module_name, py_path): flush=True, ) return - except Exception as exp: - print(f"Could not load extension {module_name} due to {exp}", flush=True) + except Exception as exc: + print(f"Could not load extension {module_name} due to {exc}", flush=True) return -def dynamic_import_from_src(src, star_import=False): +def dynamic_import_from_src(src: Union[str, Path], star_import=False): my_py_files = get_py_files(src) for py_file in my_py_files: module_name = Path(py_file).stem diff --git a/modules/hasher.py b/modules/hasher.py index 96200a1..8f5d0f1 100644 --- a/modules/hasher.py +++ b/modules/hasher.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Union +from typing import Any, List, Mapping, Union import cv2 import numpy as np @@ -9,7 +9,7 @@ from scipy import spatial from modules.database import col_photos -def hash_array_to_hash_hex(hash_array): +def hash_array_to_hash_hex(hash_array) -> str: # convert hash array of 0 or 1 to hash string in hex hash_array = np.array(hash_array, dtype=np.uint8) hash_str = "".join(str(i) for i in 1 * hash_array.flatten()) @@ -23,10 +23,10 @@ def hash_hex_to_hash_array(hash_hex) -> NDArray: return np.array(list(array_str), dtype=np.float32) -def get_duplicates_cache(album: str) -> dict: +async def get_duplicates_cache(album: str) -> Mapping[str, Any]: return { photo["filename"]: [photo["_id"].__str__(), photo["hash"]] - for photo in col_photos.find({"album": album}) + async for photo in col_photos.find({"album": album}) } @@ -52,9 +52,9 @@ async def get_phash(filepath: Union[str, Path]) -> str: return hash_array_to_hash_hex(dct_block.flatten()) -async def get_duplicates(hash_string: str, album: str) -> list: +async def get_duplicates(hash_string: str, album: str) -> List[Mapping[str, Any]]: duplicates = [] - cache = get_duplicates_cache(album) + cache = await get_duplicates_cache(album) for image_name, image_object in cache.items(): try: distance = spatial.distance.hamming( diff --git a/modules/mailer.py b/modules/mailer.py index d8e0a92..6f04b21 100644 --- a/modules/mailer.py +++ b/modules/mailer.py @@ -28,8 +28,8 @@ try: ) mail_sender.ehlo() logger.info("Initialized SMTP connection") -except Exception as exp: - logger.error("Could not initialize SMTP connection to: %s", exp) +except Exception as exc: + logger.error("Could not initialize SMTP connection to: %s", exc) print_exc() try: @@ -37,5 +37,5 @@ try: configGet("login", "mailer", "smtp"), configGet("password", "mailer", "smtp") ) logger.info("Successfully initialized mailer") -except Exception as exp: - logger.error("Could not login into provided SMTP account due to: %s", exp) +except Exception as exc: + logger.error("Could not login into provided SMTP account due to: %s", exc) diff --git a/modules/security.py b/modules/security.py index 4ac6feb..f5e3ca8 100644 --- a/modules/security.py +++ b/modules/security.py @@ -54,16 +54,20 @@ oauth2_scheme = OAuth2PasswordBearer( ) -def verify_password(plain_password, hashed_password): +def verify_password(plain_password, hashed_password) -> bool: return pwd_context.verify(plain_password, hashed_password) -def get_password_hash(password): +def get_password_hash(password) -> str: return pwd_context.hash(password) -def get_user(user: str): - found_user = col_users.find_one({"user": user}) +async def get_user(user: str) -> UserInDB: + found_user = await col_users.find_one({"user": user}) + + if found_user is None: + raise RuntimeError(f"User {user} does not exist") + return UserInDB( user=found_user["user"], email=found_user["email"], @@ -72,14 +76,16 @@ def get_user(user: str): ) -def authenticate_user(user_name: str, password: str): - if user := get_user(user_name): +async def authenticate_user(user_name: str, password: str) -> Union[UserInDB, bool]: + if user := await get_user(user_name): return user if verify_password(password, user.hash) else False else: return False -def create_access_token(data: dict, expires_delta: Union[timedelta, None] = None): +def create_access_token( + data: dict, expires_delta: Union[timedelta, None] = None +) -> str: to_encode = data.copy() if expires_delta: expire = datetime.now(tz=timezone.utc) + expires_delta @@ -93,7 +99,7 @@ def create_access_token(data: dict, expires_delta: Union[timedelta, None] = None async def get_current_user( security_scopes: SecurityScopes, token: str = Depends(oauth2_scheme) -): +) -> UserInDB: if security_scopes.scopes: authenticate_value = f'Bearer scope="{security_scopes.scope_str}"' else: @@ -112,12 +118,12 @@ async def get_current_user( raise credentials_exception token_scopes = payload.get("scopes", []) token_data = TokenData(scopes=token_scopes, user=user) - except (JWTError, ValidationError): - raise credentials_exception + except (JWTError, ValidationError) as exc: + raise credentials_exception from exc - user = get_user(user=token_data.user) + user_record = await get_user(user=token_data.user) - if user is None: + if user_record is None: raise credentials_exception for scope in security_scopes.scopes: @@ -127,7 +133,7 @@ async def get_current_user( detail="Not enough permissions", headers={"WWW-Authenticate": authenticate_value}, ) - return user + return user_record async def get_current_active_user( diff --git a/modules/utils.py b/modules/utils.py index 9f1ed5a..df3e611 100644 --- a/modules/utils.py +++ b/modules/utils.py @@ -49,8 +49,8 @@ def jsonSave(contents: Union[list, dict], filepath: Union[str, Path]) -> None: with open(filepath, "w", encoding="utf8") as file: file.write(dumps(contents, ensure_ascii=False, indent=4)) file.close() - except Exception as exp: - logger.error("Could not save json file %s: %s\n%s", filepath, exp, format_exc()) + except Exception as exc: + logger.error("Could not save json file %s: %s\n%s", filepath, exc, format_exc()) return diff --git a/requirements.txt b/requirements.txt index 3bc682e..00b0667 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,9 @@ exif==1.6.0 fastapi[all]==0.101.1 opencv-python~=4.8.0.74 passlib~=1.7.4 -pymongo==4.4.1 python-jose[cryptography]~=3.3.0 python-magic~=0.4.27 scipy~=1.11.0 -ujson~=5.8.0 \ No newline at end of file +ujson~=5.8.0 +--extra-index-url https://git.end-play.xyz/api/packages/profitroll/pypi/simple +async_pymongo==0.1.4 \ No newline at end of file From 7011baff0fea4067773bb0de51eff8cc33fafd7f Mon Sep 17 00:00:00 2001 From: profitroll Date: Mon, 14 Aug 2023 13:51:18 +0200 Subject: [PATCH 10/25] Added db_client_sync --- modules/database.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/modules/database.py b/modules/database.py index 65fe87d..405c55c 100644 --- a/modules/database.py +++ b/modules/database.py @@ -1,5 +1,5 @@ from async_pymongo import AsyncClient -from pymongo import GEOSPHERE +from pymongo import GEOSPHERE, MongoClient from modules.utils import configGet @@ -19,6 +19,7 @@ else: ) db_client = AsyncClient(con_string) +db_client_sync = MongoClient(con_string) db = db_client.get_database(name=db_config["name"]) @@ -29,4 +30,4 @@ col_videos = db.get_collection("videos") col_tokens = db.get_collection("tokens") col_emails = db.get_collection("emails") -col_photos.create_index([("location", GEOSPHERE)]) +db_client_sync[db_config["name"]]["photos"].create_index([("location", GEOSPHERE)]) From c966a6de07558d33d79a48aeadd877fc673efc26 Mon Sep 17 00:00:00 2001 From: profitroll Date: Mon, 14 Aug 2023 13:55:49 +0200 Subject: [PATCH 11/25] Fixed direction errors --- extensions/photos.py | 2 +- extensions/videos.py | 32 ++++++++++++-------------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/extensions/photos.py b/extensions/photos.py index 3015bf4..c90e28f 100644 --- a/extensions/photos.py +++ b/extensions/photos.py @@ -621,7 +621,7 @@ async def photo_find( } async for image in col_photos.find(db_query, limit=page_size, skip=skip).sort( - "dates.uploaded", DESCENDING + "dates.uploaded", direction=DESCENDING ): output["results"].append( { diff --git a/extensions/videos.py b/extensions/videos.py index 031dcb9..671bcb8 100644 --- a/extensions/videos.py +++ b/extensions/videos.py @@ -393,31 +393,23 @@ async def video_find( "caption": re.compile(caption), } elif caption is None: - db_query = list( - col_videos.find( - {"user": current_user.user, "album": album, "filename": re.compile(q)}, - limit=page_size, - skip=skip, - ).sort("dates.uploaded", DESCENDING) - ) + db_query = { + "user": current_user.user, + "album": album, + "filename": re.compile(q), + } db_query_count = { "user": current_user.user, "album": album, "caption": re.compile(q), } else: - db_query = list( - col_videos.find( - { - "user": current_user.user, - "album": album, - "filename": re.compile(q), - "caption": re.compile(caption), - }, - limit=page_size, - skip=skip, - ).sort("dates.uploaded", DESCENDING) - ) + db_query = { + "user": current_user.user, + "album": album, + "filename": re.compile(q), + "caption": re.compile(caption), + } db_query_count = { "user": current_user.user, "album": album, @@ -426,7 +418,7 @@ async def video_find( } async for video in col_videos.find(db_query, limit=page_size, skip=skip).sort( - "dates.uploaded", DESCENDING + "dates.uploaded", direction=DESCENDING ): output["results"].append( { From 3569de9363b2f820d15c3e54f506c7cb9fd3bdbe Mon Sep 17 00:00:00 2001 From: profitroll Date: Mon, 14 Aug 2023 14:26:54 +0200 Subject: [PATCH 12/25] Added pymongo as a direct dependency --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 00b0667..94951b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,6 +4,7 @@ exif==1.6.0 fastapi[all]==0.101.1 opencv-python~=4.8.0.74 passlib~=1.7.4 +pymongo>=4.3.3 python-jose[cryptography]~=3.3.0 python-magic~=0.4.27 scipy~=1.11.0 From 91d5032fd22873437286336cde4f1dc57751fe40 Mon Sep 17 00:00:00 2001 From: Renovate Date: Fri, 25 Aug 2023 22:30:40 +0300 Subject: [PATCH 13/25] Update dependency fastapi to v0.102.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 94951b7..4fba89f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiofiles==23.2.1 apscheduler~=3.10.1 exif==1.6.0 -fastapi[all]==0.101.1 +fastapi[all]==0.102.0 opencv-python~=4.8.0.74 passlib~=1.7.4 pymongo>=4.3.3 From ee53a776913f5be13bc5b5e7003eb17396683afc Mon Sep 17 00:00:00 2001 From: Renovate Date: Sat, 26 Aug 2023 22:09:12 +0300 Subject: [PATCH 14/25] Update dependency fastapi to v0.103.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4fba89f..1443902 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiofiles==23.2.1 apscheduler~=3.10.1 exif==1.6.0 -fastapi[all]==0.102.0 +fastapi[all]==0.103.0 opencv-python~=4.8.0.74 passlib~=1.7.4 pymongo>=4.3.3 From 8347a4c7790ce2569ebfd7e8aab7d763110b914d Mon Sep 17 00:00:00 2001 From: Renovate Date: Sat, 2 Sep 2023 20:40:38 +0300 Subject: [PATCH 15/25] Update dependency fastapi to v0.103.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1443902..26abf85 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiofiles==23.2.1 apscheduler~=3.10.1 exif==1.6.0 -fastapi[all]==0.103.0 +fastapi[all]==0.103.1 opencv-python~=4.8.0.74 passlib~=1.7.4 pymongo>=4.3.3 From ac8f2b2ba6d2ed01692f86fe436fbbf02f967015 Mon Sep 17 00:00:00 2001 From: Renovate Date: Thu, 28 Sep 2023 14:20:54 +0300 Subject: [PATCH 16/25] Update dependency opencv-python to ~=4.8.1.78 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 26abf85..4da7e8b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ aiofiles==23.2.1 apscheduler~=3.10.1 exif==1.6.0 fastapi[all]==0.103.1 -opencv-python~=4.8.0.74 +opencv-python~=4.8.1.78 passlib~=1.7.4 pymongo>=4.3.3 python-jose[cryptography]~=3.3.0 From 0a30512dbcc59710f487747eb3c9e0e490d4df19 Mon Sep 17 00:00:00 2001 From: Renovate Date: Thu, 28 Sep 2023 23:38:38 +0300 Subject: [PATCH 17/25] Update dependency fastapi to v0.103.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 4da7e8b..6bf16de 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiofiles==23.2.1 apscheduler~=3.10.1 exif==1.6.0 -fastapi[all]==0.103.1 +fastapi[all]==0.103.2 opencv-python~=4.8.1.78 passlib~=1.7.4 pymongo>=4.3.3 From 728917b4b932d66eebd8902ce60d80ca701bb2e9 Mon Sep 17 00:00:00 2001 From: Renovate Date: Wed, 18 Oct 2023 16:08:54 +0300 Subject: [PATCH 18/25] Update dependency fastapi to v0.104.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6bf16de..d11be62 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiofiles==23.2.1 apscheduler~=3.10.1 exif==1.6.0 -fastapi[all]==0.103.2 +fastapi[all]==0.104.0 opencv-python~=4.8.1.78 passlib~=1.7.4 pymongo>=4.3.3 From 126c66637eb175f2a95ac1ca335d121a82bfa77e Mon Sep 17 00:00:00 2001 From: Renovate Date: Mon, 30 Oct 2023 12:18:40 +0200 Subject: [PATCH 19/25] Update dependency fastapi to v0.104.1 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index d11be62..7610f09 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ aiofiles==23.2.1 apscheduler~=3.10.1 exif==1.6.0 -fastapi[all]==0.104.0 +fastapi[all]==0.104.1 opencv-python~=4.8.1.78 passlib~=1.7.4 pymongo>=4.3.3 From b2146b965aa2f79fbad890d3b0870e11ef6f5f67 Mon Sep 17 00:00:00 2001 From: Profitroll Date: Fri, 24 Nov 2023 12:19:32 +0200 Subject: [PATCH 20/25] Fixed license link Signed-off-by: Profitroll --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e8c785d..45eabe5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@

Photos API

-License: GPL +License: GPL Code style: black

From 0f423166f1c366c874b16068c2487c8a6f5e9d61 Mon Sep 17 00:00:00 2001 From: profitroll Date: Sat, 25 Nov 2023 17:50:09 +0100 Subject: [PATCH 21/25] New secrets system and quotas (#35) --- .gitignore | 3 ++- README.md | 3 ++- classes/exceptions.py | 20 ++++++++++++++++++++ config_example.json | 2 ++ extensions/exceptions.py | 22 +++++++++++++++++----- extensions/photos.py | 11 ++++++++++- extensions/users.py | 1 + extensions/videos.py | 15 +++++++++++++-- modules/app.py | 2 +- modules/security.py | 31 +++++++++++++++++++++++++++++-- 10 files changed, 97 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 2e4bb6d..90f24e0 100644 --- a/.gitignore +++ b/.gitignore @@ -153,5 +153,6 @@ cython_debug/ #.idea/ # Custom -.vscode +data/ +.vscode/ config.json \ No newline at end of file diff --git a/README.md b/README.md index 45eabe5..4fc6a52 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,8 @@ First you need to have a Python interpreter, MongoDB and optionally git. You can 1. Copy file `config_example.json` to `config.json` 2. Open `config.json` using your favorite text editor. For example `nano config.json` 3. Change `"database"` keys to match your MongoDB setup - 4. Change `"external_address"` to the ip/http address you may get in responses. By default it's `"localhost"`. This is extremely useful when running behind reverse-proxy. + 4. Set the key `"secret"` to your JWT secret. You can type in anything, but long secrets are recommended. You can also set environment variable `PHOTOSAPI_SECRET` as an alternative + 5. Change `"external_address"` to the ip/http address you may get in responses. By default it's `"localhost"`. This is extremely useful when running behind reverse-proxy. After configuring everything listed above your API will be able to boot, however further configuration can be done. You can read about it in [repository's wiki](https://git.end-play.xyz/profitroll/PhotosAPI/wiki/Configuration). There's no need to focus on that now, it makes more sense to configure it afterwards. diff --git a/classes/exceptions.py b/classes/exceptions.py index 3866990..a4419e0 100644 --- a/classes/exceptions.py +++ b/classes/exceptions.py @@ -286,3 +286,23 @@ class UserCredentialsInvalid(HTTPException): status_code=401, detail=self.openapi["content"]["application/json"]["example"]["detail"], ) + + +class UserMediaQuotaReached(HTTPException): + """Raises HTTP 403 if user's quota has been reached.""" + + def __init__(self): + self.openapi = { + "description": "Media Quota Reached", + "content": { + "application/json": { + "example": { + "detail": "Media quota has been reached, media upload impossible." + } + } + }, + } + super().__init__( + status_code=403, + detail=self.openapi["content"]["application/json"]["example"]["detail"], + ) diff --git a/config_example.json b/config_example.json index 3c9a0ec..eee3bea 100644 --- a/config_example.json +++ b/config_example.json @@ -6,6 +6,7 @@ "user": null, "password": null }, + "secret": "", "messages": { "email_confirmed": "Email confirmed. You can now log in." }, @@ -14,6 +15,7 @@ "media_token_valid_hours": 12, "registration_enabled": true, "registration_requires_confirmation": false, + "default_user_quota": 10000, "mailer": { "smtp": { "host": "", diff --git a/extensions/exceptions.py b/extensions/exceptions.py index d8e7364..bd230ce 100644 --- a/extensions/exceptions.py +++ b/extensions/exceptions.py @@ -3,6 +3,7 @@ from fastapi.responses import UJSONResponse from starlette.status import ( HTTP_400_BAD_REQUEST, HTTP_401_UNAUTHORIZED, + HTTP_403_FORBIDDEN, HTTP_404_NOT_FOUND, HTTP_406_NOT_ACCEPTABLE, HTTP_409_CONFLICT, @@ -10,19 +11,20 @@ from starlette.status import ( ) from classes.exceptions import ( - AlbumNotFoundError, + AccessTokenInvalidError, AlbumAlreadyExistsError, AlbumIncorrectError, + AlbumNotFoundError, PhotoNotFoundError, PhotoSearchQueryEmptyError, - VideoNotFoundError, - VideoSearchQueryEmptyError, SearchPageInvalidError, SearchTokenInvalidError, - AccessTokenInvalidError, - UserEmailCodeInvalid, UserAlreadyExists, UserCredentialsInvalid, + UserEmailCodeInvalid, + UserMediaQuotaReached, + VideoNotFoundError, + VideoSearchQueryEmptyError, ) from modules.app import app @@ -155,3 +157,13 @@ async def user_credentials_invalid_exception_handler( status_code=HTTP_401_UNAUTHORIZED, content={"detail": "Invalid credentials."}, ) + + +@app.exception_handler(UserMediaQuotaReached) +async def user_media_quota_reached_exception_handler( + request: Request, exc: UserMediaQuotaReached +): + return UJSONResponse( + status_code=HTTP_403_FORBIDDEN, + content={"detail": "Media quota has been reached, media upload impossible."}, + ) diff --git a/extensions/photos.py b/extensions/photos.py index c90e28f..351075b 100644 --- a/extensions/photos.py +++ b/extensions/photos.py @@ -30,6 +30,7 @@ from classes.exceptions import ( SearchLimitInvalidError, SearchPageInvalidError, SearchTokenInvalidError, + UserMediaQuotaReached, ) from classes.models import ( Photo, @@ -38,7 +39,7 @@ from classes.models import ( SearchResultsPhoto, ) from modules.app import app -from modules.database import col_albums, col_photos, col_tokens +from modules.database import col_albums, col_photos, col_tokens, col_videos from modules.exif_reader import extract_location from modules.hasher import get_duplicates, get_phash from modules.scheduler import scheduler @@ -91,6 +92,7 @@ async def compress_image(image_path: str): photo_post_responses = { + 403: UserMediaQuotaReached().openapi, 404: AlbumNameNotFoundError("name").openapi, 409: { "description": "Image Duplicates Found", @@ -125,6 +127,13 @@ async def photo_upload( if (await col_albums.find_one({"user": current_user.user, "name": album})) is None: raise AlbumNameNotFoundError(album) + user_media_count = ( + await col_photos.count_documents({"user": current_user.user}) + ) + (await col_videos.count_documents({"user": current_user.user})) + + if user_media_count >= current_user.quota and not current_user.quota == -1: # type: ignore + raise UserMediaQuotaReached() + makedirs(Path(f"data/users/{current_user.user}/albums/{album}"), exist_ok=True) filename = file.filename diff --git a/extensions/users.py b/extensions/users.py index d53052e..8fe4bcb 100644 --- a/extensions/users.py +++ b/extensions/users.py @@ -109,6 +109,7 @@ if configGet("registration_enabled") is True: { "user": user, "email": email, + "quota": None, "hash": get_password_hash(password), "disabled": configGet("registration_requires_confirmation"), } diff --git a/extensions/videos.py b/extensions/videos.py index 671bcb8..6c200eb 100644 --- a/extensions/videos.py +++ b/extensions/videos.py @@ -21,6 +21,7 @@ from classes.exceptions import ( SearchLimitInvalidError, SearchPageInvalidError, SearchTokenInvalidError, + UserMediaQuotaReached, VideoNotFoundError, VideoSearchQueryEmptyError, ) @@ -31,10 +32,13 @@ from classes.models import ( VideoPublic, ) from modules.app import app -from modules.database import col_albums, col_tokens, col_videos +from modules.database import col_albums, col_photos, col_tokens, col_videos from modules.security import User, get_current_active_user -video_post_responses = {404: AlbumNameNotFoundError("name").openapi} +video_post_responses = { + 403: UserMediaQuotaReached().openapi, + 404: AlbumNameNotFoundError("name").openapi, +} @app.post( @@ -53,6 +57,13 @@ async def video_upload( if (await col_albums.find_one({"user": current_user.user, "name": album})) is None: raise AlbumNameNotFoundError(album) + user_media_count = ( + await col_videos.count_documents({"user": current_user.user}) + ) + (await col_photos.count_documents({"user": current_user.user})) + + if user_media_count >= current_user.quota and not current_user.quota == -1: # type: ignore + raise UserMediaQuotaReached() + makedirs(Path(f"data/users/{current_user.user}/albums/{album}"), exist_ok=True) filename = file.filename diff --git a/modules/app.py b/modules/app.py index 9a34960..3963e19 100644 --- a/modules/app.py +++ b/modules/app.py @@ -1,7 +1,7 @@ from fastapi import FastAPI from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html -app = FastAPI(title="END PLAY Photos", docs_url=None, redoc_url=None, version="0.5") +app = FastAPI(title="END PLAY Photos", docs_url=None, redoc_url=None, version="0.6") @app.get("/docs", include_in_schema=False) diff --git a/modules/security.py b/modules/security.py index f5e3ca8..9e75c7f 100644 --- a/modules/security.py +++ b/modules/security.py @@ -1,4 +1,5 @@ from datetime import datetime, timedelta, timezone +from os import getenv from typing import List, Union from fastapi import Depends, HTTPException, Security, status @@ -8,9 +9,26 @@ from passlib.context import CryptContext from pydantic import BaseModel, ValidationError from modules.database import col_users +from modules.utils import configGet + +try: + configGet("secret") +except KeyError as exc: + raise KeyError( + "PhotosAPI secret is not set. Secret key handling has changed in PhotosAPI 0.6.0, so you need to add the config key 'secret' to your config file." + ) from exc + +if configGet("secret") == "" and getenv("PHOTOSAPI_SECRET") is None: + raise KeyError( + "PhotosAPI secret is not set. Set the config key 'secret' or provide the environment variable 'PHOTOSAPI_SECRET' containing a secret string." + ) + +SECRET_KEY = ( + getenv("PHOTOSAPI_SECRET") + if getenv("PHOTOSAPI_SECRET") is not None + else configGet("secret") +) -with open("secret_key", "r", encoding="utf-8") as f: - SECRET_KEY = f.read() ALGORITHM = "HS256" ACCESS_TOKEN_EXPIRE_DAYS = 180 @@ -28,6 +46,7 @@ class TokenData(BaseModel): class User(BaseModel): user: str email: Union[str, None] = None + quota: Union[int, None] = None disabled: Union[bool, None] = None @@ -71,6 +90,7 @@ async def get_user(user: str) -> UserInDB: return UserInDB( user=found_user["user"], email=found_user["email"], + quota=found_user["quota"], disabled=found_user["disabled"], hash=found_user["hash"], ) @@ -87,13 +107,16 @@ def create_access_token( data: dict, expires_delta: Union[timedelta, None] = None ) -> str: to_encode = data.copy() + if expires_delta: expire = datetime.now(tz=timezone.utc) + expires_delta else: expire = datetime.now(tz=timezone.utc) + timedelta( days=ACCESS_TOKEN_EXPIRE_DAYS ) + to_encode["exp"] = expire + return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM) @@ -114,8 +137,10 @@ async def get_current_user( try: payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM]) user: str = payload.get("sub") + if user is None: raise credentials_exception + token_scopes = payload.get("scopes", []) token_data = TokenData(scopes=token_scopes, user=user) except (JWTError, ValidationError) as exc: @@ -133,6 +158,7 @@ async def get_current_user( detail="Not enough permissions", headers={"WWW-Authenticate": authenticate_value}, ) + return user_record @@ -141,4 +167,5 @@ async def get_current_active_user( ): if current_user.disabled: raise HTTPException(status_code=400, detail="Inactive user") + return current_user From 0043abdbad14fdd07e8d1211835a6397ba520799 Mon Sep 17 00:00:00 2001 From: profitroll Date: Sat, 25 Nov 2023 18:05:12 +0100 Subject: [PATCH 22/25] Migration for quotas added --- migrations/202311251700.py | 9 +++++++++ modules/migrator.py | 23 +++++++++++++++++++++++ photos_api.py | 14 ++++++++++++++ requirements.txt | 1 + 4 files changed, 47 insertions(+) create mode 100644 migrations/202311251700.py create mode 100644 modules/migrator.py diff --git a/migrations/202311251700.py b/migrations/202311251700.py new file mode 100644 index 0000000..2ef2cd3 --- /dev/null +++ b/migrations/202311251700.py @@ -0,0 +1,9 @@ +from mongodb_migrations.base import BaseMigration + + +class Migration(BaseMigration): + def upgrade(self): + self.db.users.update_many({}, {"$set": {"quota": None}}) + + def downgrade(self): + self.db.test_collection.update_many({}, {"$unset": "quota"}) diff --git a/modules/migrator.py b/modules/migrator.py new file mode 100644 index 0000000..06def68 --- /dev/null +++ b/modules/migrator.py @@ -0,0 +1,23 @@ +from typing import Any, Mapping + +from mongodb_migrations.cli import MigrationManager +from mongodb_migrations.config import Configuration + +from modules.utils import configGet + + +def migrate_database() -> None: + """Apply migrations from folder `migrations/` to the database""" + db_config: Mapping[str, Any] = configGet("database") + + manager_config = Configuration( + { + "mongo_host": db_config["host"], + "mongo_port": db_config["port"], + "mongo_database": db_config["name"], + "mongo_username": db_config["user"], + "mongo_password": db_config["password"], + } + ) + manager = MigrationManager(manager_config) + manager.run() diff --git a/photos_api.py b/photos_api.py index 6d5a497..45dfcc5 100644 --- a/photos_api.py +++ b/photos_api.py @@ -1,4 +1,5 @@ import logging +from argparse import ArgumentParser from os import makedirs from pathlib import Path @@ -6,6 +7,7 @@ from fastapi.responses import FileResponse from modules.app import app from modules.extensions_loader import dynamic_import_from_src +from modules.migrator import migrate_database from modules.scheduler import scheduler makedirs(Path("data/users"), exist_ok=True) @@ -27,3 +29,15 @@ dynamic_import_from_src("extensions", star_import=True) # ================================================================================= scheduler.start() + +parser = ArgumentParser( + prog="PhotosAPI", + description="Small and simple API server for saving photos and videos.", +) + +parser.add_argument("--migrate", action="store_true") + +args = parser.parse_args() + +if args.migrate: + migrate_database() diff --git a/requirements.txt b/requirements.txt index 7610f09..9e46266 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ aiofiles==23.2.1 apscheduler~=3.10.1 exif==1.6.0 fastapi[all]==0.104.1 +mongodb-migrations==1.3.0 opencv-python~=4.8.1.78 passlib~=1.7.4 pymongo>=4.3.3 From 5174602c31df6168a365f0045a1af4f10b13730b Mon Sep 17 00:00:00 2001 From: profitroll Date: Sat, 25 Nov 2023 18:12:01 +0100 Subject: [PATCH 23/25] Added upgrade section --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 4fc6a52..3959476 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,19 @@ First you need to have a Python interpreter, MongoDB and optionally git. You can Learn more about available uvicorn arguments using `uvicorn --help` +## Upgrading + +When a new version comes out, sometimes you want to upgrade your instance right away. Here's a checklist what to do: + +1. Carefully read the patch notes of the version you want to update to and all the versions that came out between the release of your version and the one you want to upgrade to. + Breaking changes will be marked so and config updates will also be described in the patch notes +2. Make a backup of your currently working instance. This includes both the PhotosAPI and the database +3. Download the latest version using git (`git pull` if you cloned the repo in the past) or from the releases +4. Reconfigure the config if needed and apply the changes from the patch notes +5. Upgrade the dependencies in your virtual environment using `pip install -r requirements.txt` +6. Start the migration using `python photos_api.py --migrate` from your virtual environment +7. Test if everything works and troubleshoot/rollback if not + ## Using as a service It's a good practice to use your API as a systemd service on Linux. Here's a quick overview how that can be done. From e5fad5ba928a4b052e8ad318e80cf027c241d2bd Mon Sep 17 00:00:00 2001 From: profitroll Date: Sat, 25 Nov 2023 18:14:30 +0100 Subject: [PATCH 24/25] Fixed "unrecognized arguments" error --- photos_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/photos_api.py b/photos_api.py index 45dfcc5..ec3da61 100644 --- a/photos_api.py +++ b/photos_api.py @@ -37,7 +37,7 @@ parser = ArgumentParser( parser.add_argument("--migrate", action="store_true") -args = parser.parse_args() +args, unknown = parser.parse_known_args() if args.migrate: migrate_database() From afefea6f68b7a878f5e3616940a70603880cdb6a Mon Sep 17 00:00:00 2001 From: profitroll Date: Sat, 25 Nov 2023 18:17:17 +0100 Subject: [PATCH 25/25] Fixed "TypeError" for UserInDB --- modules/security.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/security.py b/modules/security.py index 9e75c7f..f7599e9 100644 --- a/modules/security.py +++ b/modules/security.py @@ -90,7 +90,9 @@ async def get_user(user: str) -> UserInDB: return UserInDB( user=found_user["user"], email=found_user["email"], - quota=found_user["quota"], + quota=found_user["quota"] + if found_user["quota"] is not None + else configGet("default_user_quota"), disabled=found_user["disabled"], hash=found_user["hash"], )