profitroll/PhotosAPI
profitroll/PhotosAPI
1
0
Fork 0
Code Issues 4 Pull Requests Releases 7 Wiki Activity

WIP: Migration to async_pymongo

Browse Source
This commit is contained in:
Profitroll 2023-08-14 13:44:07 +02:00
parent 80ec8eb4f3
commit a1acaed6dd
Signed by: profitroll
GPG Key ID: FA35CAB49DACD3B2
13 changed files with 196 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
extensions/albums.py
View File

@ -47,12 +47,12 @@ async def album_create(
if 2 > len(title) > 40: if 2 > len(title) > 40:
raise AlbumIncorrectError("title", "must be >2 and <40 characters.") raise AlbumIncorrectError("title", "must be >2 and <40 characters.")
if col_albums.find_one({"name": name}) is not None: if (await col_albums.find_one({"name": name})) is not None:
raise AlbumAlreadyExistsError(name) raise AlbumAlreadyExistsError(name)
makedirs(Path(f"data/users/{current_user.user}/albums/{name}"), exist_ok=True) makedirs(Path(f"data/users/{current_user.user}/albums/{name}"), exist_ok=True)
uploaded = col_albums.insert_one( uploaded = await col_albums.insert_one(
{"user": current_user.user, "name": name, "title": title, "cover": None} {"user": current_user.user, "name": name, "title": title, "cover": None}
) )
@ -67,9 +67,10 @@ async def album_find(
current_user: User = Security(get_current_active_user, scopes=["albums.list"]), current_user: User = Security(get_current_active_user, scopes=["albums.list"]),
): ):
output = {"results": []} output = {"results": []}
albums = list(col_albums.find({"user": current_user.user, "name": re.compile(q)}))
for album in albums: async for album in col_albums.find(
{"user": current_user.user, "name": re.compile(q)}
):
output["results"].append( output["results"].append(
{ {
"id": album["_id"].__str__(), "id": album["_id"].__str__(),
@ -102,11 +103,11 @@ async def album_patch(
current_user: User = Security(get_current_active_user, scopes=["albums.write"]), current_user: User = Security(get_current_active_user, scopes=["albums.write"]),
): ):
try: try:
album = col_albums.find_one({"_id": ObjectId(id)}) album = await col_albums.find_one({"_id": ObjectId(id)})
if album is None: if album is None:
raise InvalidId(id) raise InvalidId(id)
except InvalidId: except InvalidId as exc:
raise AlbumNotFoundError(id) raise AlbumNotFoundError(id) from exc
if title is None: if title is None:
title = album["title"] title = album["title"]
@ -125,7 +126,7 @@ async def album_patch(
Path(f"data/users/{current_user.user}/albums/{album['name']}"), Path(f"data/users/{current_user.user}/albums/{album['name']}"),
Path(f"data/users/{current_user.user}/albums/{name}"), Path(f"data/users/{current_user.user}/albums/{name}"),
) )
col_photos.update_many( await col_photos.update_many(
{"user": current_user.user, "album": album["name"]}, {"user": current_user.user, "album": album["name"]},
{"$set": {"album": name}}, {"$set": {"album": name}},
) )
@ -133,12 +134,14 @@ async def album_patch(
name = album["name"] name = album["name"]
if cover is not None: if cover is not None:
image = col_photos.find_one({"_id": ObjectId(cover), "album": album["name"]}) image = await col_photos.find_one(
{"_id": ObjectId(cover), "album": album["name"]}
)
cover = image["_id"].__str__() if image is not None else album["cover"] cover = image["_id"].__str__() if image is not None else album["cover"]
else: else:
cover = album["cover"] cover = album["cover"]
col_albums.update_one( await col_albums.update_one(
{"_id": ObjectId(id)}, {"$set": {"name": name, "title": title, "cover": cover}} {"_id": ObjectId(id)}, {"$set": {"name": name, "title": title, "cover": cover}}
) )
@ -166,11 +169,11 @@ async def album_put(
current_user: User = Security(get_current_active_user, scopes=["albums.write"]), current_user: User = Security(get_current_active_user, scopes=["albums.write"]),
): ):
try: try:
album = col_albums.find_one({"_id": ObjectId(id)}) album = await col_albums.find_one({"_id": ObjectId(id)})
if album is None: if album is None:
raise InvalidId(id) raise InvalidId(id)
except InvalidId: except InvalidId as exc:
raise AlbumNotFoundError(id) raise AlbumNotFoundError(id) from exc
if re.search(re.compile("^[a-z,0-9,_]*$"), name) is False: if re.search(re.compile("^[a-z,0-9,_]*$"), name) is False:
raise AlbumIncorrectError("name", "can only contain a-z, 0-9 and _ characters.") raise AlbumIncorrectError("name", "can only contain a-z, 0-9 and _ characters.")
@ -181,7 +184,7 @@ async def album_put(
if 2 > len(title) > 40: if 2 > len(title) > 40:
raise AlbumIncorrectError("title", "must be >2 and <40 characters.") raise AlbumIncorrectError("title", "must be >2 and <40 characters.")
image = col_photos.find_one({"_id": ObjectId(cover), "album": album["name"]}) image = await col_photos.find_one({"_id": ObjectId(cover), "album": album["name"]})
cover = image["_id"].__str__() if image is not None else None # type: ignore cover = image["_id"].__str__() if image is not None else None # type: ignore
rename( rename(
@ -189,10 +192,10 @@ async def album_put(
Path(f"data/users/{current_user.user}/albums/{name}"), Path(f"data/users/{current_user.user}/albums/{name}"),
) )
col_photos.update_many( await col_photos.update_many(
{"user": current_user.user, "album": album["name"]}, {"$set": {"album": name}} {"user": current_user.user, "album": album["name"]}, {"$set": {"album": name}}
) )
col_albums.update_one( await col_albums.update_one(
{"_id": ObjectId(id)}, {"$set": {"name": name, "title": title, "cover": cover}} {"_id": ObjectId(id)}, {"$set": {"name": name, "title": title, "cover": cover}}
) )
@ -213,13 +216,13 @@ async def album_delete(
current_user: User = Security(get_current_active_user, scopes=["albums.write"]), current_user: User = Security(get_current_active_user, scopes=["albums.write"]),
): ):
try: try:
album = col_albums.find_one_and_delete({"_id": ObjectId(id)}) album = await col_albums.find_one_and_delete({"_id": ObjectId(id)})
if album is None: if album is None:
raise InvalidId(id) raise InvalidId(id)
except InvalidId: except InvalidId as exc:
raise AlbumNotFoundError(id) raise AlbumNotFoundError(id) from exc
col_photos.delete_many({"album": album["name"]}) await col_photos.delete_many({"album": album["name"]})
rmtree(Path(f"data/users/{current_user.user}/albums/{album['name']}")) rmtree(Path(f"data/users/{current_user.user}/albums/{album['name']}"))

111
extensions/photos.py
View File

@ -122,7 +122,7 @@ async def photo_upload(
caption: Union[str, None] = None, caption: Union[str, None] = None,
current_user: User = Security(get_current_active_user, scopes=["photos.write"]), current_user: User = Security(get_current_active_user, scopes=["photos.write"]),
): ):
if col_albums.find_one({"user": current_user.user, "name": album}) is None: if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album) raise AlbumNameNotFoundError(album)
makedirs(Path(f"data/users/{current_user.user}/albums/{album}"), exist_ok=True) makedirs(Path(f"data/users/{current_user.user}/albums/{album}"), exist_ok=True)
@ -158,7 +158,7 @@ async def photo_upload(
expires_delta=timedelta(hours=configGet("media_token_valid_hours")), expires_delta=timedelta(hours=configGet("media_token_valid_hours")),
) )
access_token_short = uuid4().hex[:12].lower() access_token_short = uuid4().hex[:12].lower()
col_tokens.insert_one( await col_tokens.insert_one(
{ {
"short": access_token_short, "short": access_token_short,
"access_token": access_token, "access_token": access_token,
@ -183,7 +183,7 @@ async def photo_upload(
except (UnpackError, ValueError): except (UnpackError, ValueError):
coords = {"lng": 0.0, "lat": 0.0, "alt": 0.0} coords = {"lng": 0.0, "lat": 0.0, "alt": 0.0}
uploaded = col_photos.insert_one( uploaded = await col_photos.insert_one(
{ {
"user": current_user.user, "user": current_user.user,
"album": album, "album": album,
@ -231,7 +231,7 @@ if configGet("media_token_access") is True:
responses=photo_get_token_responses, responses=photo_get_token_responses,
) )
async def photo_get_token(token: str, id: int): async def photo_get_token(token: str, id: int):
db_entry = col_tokens.find_one({"short": token}) db_entry = await col_tokens.find_one({"short": token})
if db_entry is None: if db_entry is None:
raise AccessTokenInvalidError() raise AccessTokenInvalidError()
@ -246,24 +246,23 @@ if configGet("media_token_access") is True:
raise AccessTokenInvalidError() raise AccessTokenInvalidError()
token_scopes = payload.get("scopes", []) token_scopes = payload.get("scopes", [])
token_data = TokenData(scopes=token_scopes, user=user) token_data = TokenData(scopes=token_scopes, user=user)
except (JWTError, ValidationError) as exp: except (JWTError, ValidationError) as exc:
print(exp, flush=True) raise AccessTokenInvalidError() from exc
raise AccessTokenInvalidError()
user = get_user(user=token_data.user) user_record = await get_user(user=token_data.user)
if id not in payload.get("allowed", []): if id not in payload.get("allowed", []):
raise AccessTokenInvalidError() raise AccessTokenInvalidError()
try: try:
image = col_photos.find_one({"_id": ObjectId(id)}) image = await col_photos.find_one({"_id": ObjectId(id)})
if image is None: if image is None:
raise InvalidId(id) raise InvalidId(id)
except InvalidId: except InvalidId as exc:
raise PhotoNotFoundError(id) raise PhotoNotFoundError(id) from exc
image_path = Path( image_path = Path(
f"data/users/{user.user}/albums/{image['album']}/{image['filename']}" f"data/users/{user_record.user}/albums/{image['album']}/{image['filename']}"
) )
mime = Magic(mime=True).from_file(image_path) mime = Magic(mime=True).from_file(image_path)
@ -301,11 +300,11 @@ async def photo_get(
current_user: User = Security(get_current_active_user, scopes=["photos.read"]), current_user: User = Security(get_current_active_user, scopes=["photos.read"]),
): ):
try: try:
image = col_photos.find_one({"_id": ObjectId(id)}) image = await col_photos.find_one({"_id": ObjectId(id)})
if image is None: if image is None:
raise InvalidId(id) raise InvalidId(id)
except InvalidId: except InvalidId as exc:
raise PhotoNotFoundError(id) raise PhotoNotFoundError(id) from exc
image_path = Path( image_path = Path(
f"data/users/{current_user.user}/albums/{image['album']}/{image['filename']}" f"data/users/{current_user.user}/albums/{image['album']}/{image['filename']}"
@ -334,13 +333,13 @@ async def photo_move(
current_user: User = Security(get_current_active_user, scopes=["photos.write"]), current_user: User = Security(get_current_active_user, scopes=["photos.write"]),
): ):
try: try:
image = col_photos.find_one({"_id": ObjectId(id)}) image = await col_photos.find_one({"_id": ObjectId(id)})
if image is None: if image is None:
raise InvalidId(id) raise InvalidId(id)
except InvalidId: except InvalidId as exc:
raise PhotoNotFoundError(id) raise PhotoNotFoundError(id) from exc
if col_albums.find_one({"user": current_user.user, "name": album}) is None: if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album) raise AlbumNameNotFoundError(album)
if Path( if Path(
@ -354,7 +353,7 @@ async def photo_move(
else: else:
filename = image["filename"] filename = image["filename"]
col_photos.find_one_and_update( await col_photos.find_one_and_update(
{"_id": ObjectId(id)}, {"_id": ObjectId(id)},
{ {
"$set": { "$set": {
@ -396,13 +395,13 @@ async def photo_patch(
current_user: User = Security(get_current_active_user, scopes=["photos.write"]), current_user: User = Security(get_current_active_user, scopes=["photos.write"]),
): ):
try: try:
image = col_photos.find_one({"_id": ObjectId(id)}) image = await col_photos.find_one({"_id": ObjectId(id)})
if image is None: if image is None:
raise InvalidId(id) raise InvalidId(id)
except InvalidId: except InvalidId as exc:
raise PhotoNotFoundError(id) raise PhotoNotFoundError(id) from exc
col_photos.find_one_and_update( await col_photos.find_one_and_update(
{"_id": ObjectId(id)}, {"_id": ObjectId(id)},
{"$set": {"caption": caption, "dates.modified": datetime.now(tz=timezone.utc)}}, {"$set": {"caption": caption, "dates.modified": datetime.now(tz=timezone.utc)}},
) )
@ -430,16 +429,16 @@ async def photo_delete(
current_user: User = Security(get_current_active_user, scopes=["photos.write"]), current_user: User = Security(get_current_active_user, scopes=["photos.write"]),
): ):
try: try:
image = col_photos.find_one_and_delete({"_id": ObjectId(id)}) image = await col_photos.find_one_and_delete({"_id": ObjectId(id)})
if image is None: if image is None:
raise InvalidId(id) raise InvalidId(id)
except InvalidId: except InvalidId as exc:
raise PhotoNotFoundError(id) raise PhotoNotFoundError(id) from exc
album = col_albums.find_one({"name": image["album"]}) album = await col_albums.find_one({"name": image["album"]})
if album is not None and album["cover"] == image["_id"].__str__(): if album is not None and album["cover"] == image["_id"].__str__():
col_albums.update_one({"name": image["album"]}, {"$set": {"cover": None}}) await col_albums.update_one({"name": image["album"]}, {"$set": {"cover": None}})
remove( remove(
Path( Path(
@ -469,7 +468,7 @@ async def photo_random(
limit: int = 100, limit: int = 100,
current_user: User = Security(get_current_active_user, scopes=["photos.list"]), current_user: User = Security(get_current_active_user, scopes=["photos.list"]),
): ):
if col_albums.find_one({"user": current_user.user, "name": album}) is None: if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album) raise AlbumNameNotFoundError(album)
if limit <= 0: if limit <= 0:
@ -490,20 +489,16 @@ async def photo_random(
} }
) )
documents_count = col_photos.count_documents(db_query) documents_count = await col_photos.count_documents(db_query)
skip = randint(0, documents_count - 1) if documents_count > 1 else 0 skip = randint(0, documents_count - 1) if documents_count > 1 else 0
images = list( async for image in col_photos.aggregate(
col_photos.aggregate( [
[ {"$match": db_query},
{"$match": db_query}, {"$skip": skip},
{"$skip": skip}, {"$limit": limit},
{"$limit": limit}, ]
] ):
)
)
for image in images:
output["results"].append( output["results"].append(
{ {
"id": image["_id"].__str__(), "id": image["_id"].__str__(),
@ -543,7 +538,7 @@ async def photo_find(
current_user: User = Security(get_current_active_user, scopes=["photos.list"]), current_user: User = Security(get_current_active_user, scopes=["photos.list"]),
): ):
if token is not None: if token is not None:
found_record = col_tokens.find_one({"token": token}) found_record = await col_tokens.find_one({"token": token})
if found_record is None: if found_record is None:
raise SearchTokenInvalidError() raise SearchTokenInvalidError()
@ -560,7 +555,7 @@ async def photo_find(
current_user=current_user, current_user=current_user,
) )
if col_albums.find_one({"user": current_user.user, "name": album}) is None: if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album) raise AlbumNameNotFoundError(album)
if page <= 0 or page_size <= 0: if page <= 0 or page_size <= 0:
@ -612,16 +607,22 @@ async def photo_find(
"filename": re.compile(q), "filename": re.compile(q),
} }
else: else:
db_query = {"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)} # type: ignore db_query = {
db_query_count = {"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)} # type: ignore "user": current_user.user,
"album": album,
"filename": re.compile(q),
"caption": re.compile(caption),
}
db_query_count = {
"user": current_user.user,
"album": album,
"filename": re.compile(q),
"caption": re.compile(caption),
}
images = list( async for image in col_photos.find(db_query, limit=page_size, skip=skip).sort(
col_photos.find(db_query, limit=page_size, skip=skip).sort( "dates.uploaded", DESCENDING
"dates.uploaded", DESCENDING ):
)
)
for image in images:
output["results"].append( output["results"].append(
{ {
"id": image["_id"].__str__(), "id": image["_id"].__str__(),
@ -630,9 +631,9 @@ async def photo_find(
} }
) )
if col_photos.count_documents(db_query_count) > page * page_size: if (await col_photos.count_documents(db_query_count)) > page * page_size:
token = str(token_urlsafe(32)) token = str(token_urlsafe(32))
col_tokens.insert_one( await col_tokens.insert_one(
{ {
"token": token, "token": token,
"query": q, "query": q,

2
extensions/security.py
View File

@ -17,7 +17,7 @@ token_post_responses = {401: UserCredentialsInvalid().openapi}
@app.post("/token", response_model=Token, responses=token_post_responses) @app.post("/token", response_model=Token, responses=token_post_responses)
async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends()): async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends()):
user = authenticate_user(form_data.username, form_data.password) user = await authenticate_user(form_data.username, form_data.password)
if not user: if not user:
raise UserCredentialsInvalid() raise UserCredentialsInvalid()
access_token_expires = timedelta(days=ACCESS_TOKEN_EXPIRE_DAYS) access_token_expires = timedelta(days=ACCESS_TOKEN_EXPIRE_DAYS)

28
extensions/users.py
View File

@ -41,14 +41,14 @@ async def send_confirmation(user: str, email: str):
+ f"/users/{user}/confirm?code={confirmation_code}" + f"/users/{user}/confirm?code={confirmation_code}"
), ),
) )
col_emails.insert_one( await col_emails.insert_one(
{"user": user, "email": email, "used": False, "code": confirmation_code} {"user": user, "email": email, "used": False, "code": confirmation_code}
) )
logger.info( logger.info(
"Sent confirmation email to '%s' with code %s", email, confirmation_code "Sent confirmation email to '%s' with code %s", email, confirmation_code
) )
except Exception as exp: except Exception as exc:
logger.error("Could not send confirmation email to '%s' due to: %s", email, exp) logger.error("Could not send confirmation email to '%s' due to: %s", email, exc)
@app.get("/users/me/", response_model=User) @app.get("/users/me/", response_model=User)
@ -80,15 +80,15 @@ if configGet("registration_requires_confirmation") is True:
responses=user_confirm_responses, responses=user_confirm_responses,
) )
async def user_confirm(user: str, code: str): async def user_confirm(user: str, code: str):
confirm_record = col_emails.find_one( confirm_record = await col_emails.find_one(
{"user": user, "code": code, "used": False} {"user": user, "code": code, "used": False}
) )
if confirm_record is None: if confirm_record is None:
raise UserEmailCodeInvalid() raise UserEmailCodeInvalid()
col_emails.find_one_and_update( await col_emails.find_one_and_update(
{"_id": confirm_record["_id"]}, {"$set": {"used": True}} {"_id": confirm_record["_id"]}, {"$set": {"used": True}}
) )
col_users.find_one_and_update( await col_users.find_one_and_update(
{"user": confirm_record["user"]}, {"$set": {"disabled": False}} {"user": confirm_record["user"]}, {"$set": {"disabled": False}}
) )
return UJSONResponse({"detail": configGet("email_confirmed", "messages")}) return UJSONResponse({"detail": configGet("email_confirmed", "messages")})
@ -103,9 +103,9 @@ if configGet("registration_enabled") is True:
async def user_create( async def user_create(
user: str = Form(), email: str = Form(), password: str = Form() user: str = Form(), email: str = Form(), password: str = Form()
): ):
if col_users.find_one({"user": user}) is not None: if (await col_users.find_one({"user": user})) is not None:
raise UserAlreadyExists() raise UserAlreadyExists()
col_users.insert_one( await col_users.insert_one(
{ {
"user": user, "user": user,
"email": email, "email": email,
@ -132,14 +132,14 @@ user_delete_responses = {401: UserCredentialsInvalid().openapi}
async def user_delete( async def user_delete(
password: str = Form(), current_user: User = Depends(get_current_active_user) password: str = Form(), current_user: User = Depends(get_current_active_user)
): ):
user = get_user(current_user.user) user = await get_user(current_user.user)
if not user: if not user:
return False return False
if not verify_password(password, user.hash): if not verify_password(password, user.hash):
raise UserCredentialsInvalid() raise UserCredentialsInvalid()
col_users.delete_many({"user": current_user.user}) await col_users.delete_many({"user": current_user.user})
col_emails.delete_many({"user": current_user.user}) await col_emails.delete_many({"user": current_user.user})
col_photos.delete_many({"user": current_user.user}) await col_photos.delete_many({"user": current_user.user})
col_videos.delete_many({"user": current_user.user}) await col_videos.delete_many({"user": current_user.user})
col_albums.delete_many({"user": current_user.user}) await col_albums.delete_many({"user": current_user.user})
return Response(status_code=HTTP_204_NO_CONTENT) return Response(status_code=HTTP_204_NO_CONTENT)

94
extensions/videos.py
View File

@ -50,7 +50,7 @@ async def video_upload(
caption: Union[str, None] = None, caption: Union[str, None] = None,
current_user: User = Security(get_current_active_user, scopes=["videos.write"]), current_user: User = Security(get_current_active_user, scopes=["videos.write"]),
): ):
if col_albums.find_one({"user": current_user.user, "name": album}) is None: if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album) raise AlbumNameNotFoundError(album)
makedirs(Path(f"data/users/{current_user.user}/albums/{album}"), exist_ok=True) makedirs(Path(f"data/users/{current_user.user}/albums/{album}"), exist_ok=True)
@ -73,7 +73,7 @@ async def video_upload(
# Coords extraction should be here # Coords extraction should be here
uploaded = col_videos.insert_one( uploaded = await col_videos.insert_one(
{ {
"user": current_user.user, "user": current_user.user,
"album": album, "album": album,
@ -123,11 +123,11 @@ async def video_get(
current_user: User = Security(get_current_active_user, scopes=["videos.read"]), current_user: User = Security(get_current_active_user, scopes=["videos.read"]),
): ):
try: try:
video = col_videos.find_one({"_id": ObjectId(id)}) video = await col_videos.find_one({"_id": ObjectId(id)})
if video is None: if video is None:
raise InvalidId(id) raise InvalidId(id)
except InvalidId: except InvalidId as exc:
raise VideoNotFoundError(id) raise VideoNotFoundError(id) from exc
video_path = Path( video_path = Path(
f"data/users/{current_user.user}/albums/{video['album']}/{video['filename']}" f"data/users/{current_user.user}/albums/{video['album']}/{video['filename']}"
@ -156,13 +156,13 @@ async def video_move(
current_user: User = Security(get_current_active_user, scopes=["videos.write"]), current_user: User = Security(get_current_active_user, scopes=["videos.write"]),
): ):
try: try:
video = col_videos.find_one({"_id": ObjectId(id)}) video = await col_videos.find_one({"_id": ObjectId(id)})
if video is None: if video is None:
raise InvalidId(id) raise InvalidId(id)
except InvalidId: except InvalidId as exc:
raise VideoNotFoundError(id) raise VideoNotFoundError(id) from exc
if col_albums.find_one({"user": current_user.user, "name": album}) is None: if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album) raise AlbumNameNotFoundError(album)
if Path( if Path(
@ -176,7 +176,7 @@ async def video_move(
else: else:
filename = video["filename"] filename = video["filename"]
col_videos.find_one_and_update( await col_videos.find_one_and_update(
{"_id": ObjectId(id)}, {"_id": ObjectId(id)},
{ {
"$set": { "$set": {
@ -218,13 +218,13 @@ async def video_patch(
current_user: User = Security(get_current_active_user, scopes=["videos.write"]), current_user: User = Security(get_current_active_user, scopes=["videos.write"]),
): ):
try: try:
video = col_videos.find_one({"_id": ObjectId(id)}) video = await col_videos.find_one({"_id": ObjectId(id)})
if video is None: if video is None:
raise InvalidId(id) raise InvalidId(id)
except InvalidId: except InvalidId as exc:
raise VideoNotFoundError(id) raise VideoNotFoundError(id) from exc
col_videos.find_one_and_update( await col_videos.find_one_and_update(
{"_id": ObjectId(id)}, {"_id": ObjectId(id)},
{"$set": {"caption": caption, "dates.modified": datetime.now(tz=timezone.utc)}}, {"$set": {"caption": caption, "dates.modified": datetime.now(tz=timezone.utc)}},
) )
@ -252,13 +252,13 @@ async def video_delete(
current_user: User = Security(get_current_active_user, scopes=["videos.write"]), current_user: User = Security(get_current_active_user, scopes=["videos.write"]),
): ):
try: try:
video = col_videos.find_one_and_delete({"_id": ObjectId(id)}) video = await col_videos.find_one_and_delete({"_id": ObjectId(id)})
if video is None: if video is None:
raise InvalidId(id) raise InvalidId(id)
except InvalidId: except InvalidId as exc:
raise VideoNotFoundError(id) raise VideoNotFoundError(id) from exc
album = col_albums.find_one({"name": video["album"]}) album = await col_albums.find_one({"name": video["album"]})
remove( remove(
Path( Path(
@ -288,7 +288,7 @@ async def video_random(
limit: int = 100, limit: int = 100,
current_user: User = Security(get_current_active_user, scopes=["videos.list"]), current_user: User = Security(get_current_active_user, scopes=["videos.list"]),
): ):
if col_albums.find_one({"user": current_user.user, "name": album}) is None: if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album) raise AlbumNameNotFoundError(album)
if limit <= 0: if limit <= 0:
@ -309,20 +309,16 @@ async def video_random(
} }
) )
documents_count = col_videos.count_documents(db_query) documents_count = await col_videos.count_documents(db_query)
skip = randint(0, documents_count - 1) if documents_count > 1 else 0 skip = randint(0, documents_count - 1) if documents_count > 1 else 0
videos = list( async for video in col_videos.aggregate(
col_videos.aggregate( [
[ {"$match": db_query},
{"$match": db_query}, {"$skip": skip},
{"$skip": skip}, {"$limit": limit},
{"$limit": limit}, ]
] ):
)
)
for video in videos:
output["results"].append( output["results"].append(
{ {
"id": video["_id"].__str__(), "id": video["_id"].__str__(),
@ -359,7 +355,7 @@ async def video_find(
current_user: User = Security(get_current_active_user, scopes=["videos.list"]), current_user: User = Security(get_current_active_user, scopes=["videos.list"]),
): ):
if token is not None: if token is not None:
found_record = col_tokens.find_one({"token": token}) found_record = await col_tokens.find_one({"token": token})
if found_record is None: if found_record is None:
raise SearchTokenInvalidError() raise SearchTokenInvalidError()
@ -373,7 +369,7 @@ async def video_find(
current_user=current_user, current_user=current_user,
) )
if col_albums.find_one({"user": current_user.user, "name": album}) is None: if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album) raise AlbumNameNotFoundError(album)
if page <= 0 or page_size <= 0: if page <= 0 or page_size <= 0:
@ -410,16 +406,28 @@ async def video_find(
"caption": re.compile(q), "caption": re.compile(q),
} }
else: else:
db_query = list(col_videos.find({"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)}, limit=page_size, skip=skip).sort("dates.uploaded", DESCENDING)) # type: ignore db_query = list(
db_query_count = {"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)} # type: ignore col_videos.find(
{
videos = list( "user": current_user.user,
col_videos.find(db_query, limit=page_size, skip=skip).sort( "album": album,
"dates.uploaded", DESCENDING "filename": re.compile(q),
"caption": re.compile(caption),
},
limit=page_size,
skip=skip,
).sort("dates.uploaded", DESCENDING)
) )
) db_query_count = {
"user": current_user.user,
"album": album,
"filename": re.compile(q),
"caption": re.compile(caption),
}
for video in videos: async for video in col_videos.find(db_query, limit=page_size, skip=skip).sort(
"dates.uploaded", DESCENDING
):
output["results"].append( output["results"].append(
{ {
"id": video["_id"].__str__(), "id": video["_id"].__str__(),
@ -428,9 +436,9 @@ async def video_find(
} }
) )
if col_videos.count_documents(db_query_count) > page * page_size: if (await col_videos.count_documents(db_query_count)) > page * page_size:
token = str(token_urlsafe(32)) token = str(token_urlsafe(32))
col_tokens.insert_one( await col_tokens.insert_one(
{ {
"token": token, "token": token,
"query": q, "query": q,

11
modules/database.py
View File

@ -1,4 +1,5 @@
from pymongo import GEOSPHERE, MongoClient from async_pymongo import AsyncClient
from pymongo import GEOSPHERE
from modules.utils import configGet from modules.utils import configGet
@ -17,16 +18,10 @@ else:
db_config["host"], db_config["port"], db_config["name"] db_config["host"], db_config["port"], db_config["name"]
) )
db_client = MongoClient(con_string) db_client = AsyncClient(con_string)
db = db_client.get_database(name=db_config["name"]) db = db_client.get_database(name=db_config["name"])
collections = db.list_collection_names()
for collection in ["users", "albums", "photos", "videos", "tokens", "emails"]:
if collection not in collections:
db.create_collection(collection)
col_users = db.get_collection("users") col_users = db.get_collection("users")
col_albums = db.get_collection("albums") col_albums = db.get_collection("albums")
col_photos = db.get_collection("photos") col_photos = db.get_collection("photos")

4
modules/exif_reader.py
View File

@ -1,4 +1,6 @@
import contextlib import contextlib
from pathlib import Path
from typing import Mapping, Union
from exif import Image from exif import Image
@ -21,7 +23,7 @@ def decimal_coords(coords: float, ref: str) -> float:
return round(decimal_degrees, 5) return round(decimal_degrees, 5)
def extract_location(filepath: str) -> dict: def extract_location(filepath: Union[str, Path]) -> Mapping[str, float]:
"""Get location data from image """Get location data from image
### Args: ### Args:

17
modules/extensions_loader.py
View File

@ -1,6 +1,7 @@
from importlib.util import module_from_spec, spec_from_file_location from importlib.util import module_from_spec, spec_from_file_location
from os import getcwd, path, walk from os import getcwd, path, walk
from pathlib import Path from pathlib import Path
from typing import Union
# ================================================================================= # =================================================================================
@ -17,11 +18,15 @@ def get_py_files(src):
return py_files return py_files
def dynamic_import(module_name, py_path): def dynamic_import(module_name: str, py_path: str):
try: try:
module_spec = spec_from_file_location(module_name, py_path) module_spec = spec_from_file_location(module_name, py_path)
module = module_from_spec(module_spec) # type: ignore if module_spec is None:
module_spec.loader.exec_module(module) # type: ignore raise RuntimeError(
f"Module spec from module name {module_name} and path {py_path} is None"
)
module = module_from_spec(module_spec)
module_spec.loader.exec_module(module)
return module return module
except SyntaxError: except SyntaxError:
print( print(
@ -29,12 +34,12 @@ def dynamic_import(module_name, py_path):
flush=True, flush=True,
) )
return return
except Exception as exp: except Exception as exc:
print(f"Could not load extension {module_name} due to {exp}", flush=True) print(f"Could not load extension {module_name} due to {exc}", flush=True)
return return
def dynamic_import_from_src(src, star_import=False): def dynamic_import_from_src(src: Union[str, Path], star_import=False):
my_py_files = get_py_files(src) my_py_files = get_py_files(src)
for py_file in my_py_files: for py_file in my_py_files:
module_name = Path(py_file).stem module_name = Path(py_file).stem

12
modules/hasher.py
View File

@ -1,5 +1,5 @@
from pathlib import Path from pathlib import Path
from typing import Union from typing import Any, List, Mapping, Union
import cv2 import cv2
import numpy as np import numpy as np
@ -9,7 +9,7 @@ from scipy import spatial
from modules.database import col_photos from modules.database import col_photos
def hash_array_to_hash_hex(hash_array): def hash_array_to_hash_hex(hash_array) -> str:
# convert hash array of 0 or 1 to hash string in hex # convert hash array of 0 or 1 to hash string in hex
hash_array = np.array(hash_array, dtype=np.uint8) hash_array = np.array(hash_array, dtype=np.uint8)
hash_str = "".join(str(i) for i in 1 * hash_array.flatten()) hash_str = "".join(str(i) for i in 1 * hash_array.flatten())
@ -23,10 +23,10 @@ def hash_hex_to_hash_array(hash_hex) -> NDArray:
return np.array(list(array_str), dtype=np.float32) return np.array(list(array_str), dtype=np.float32)
def get_duplicates_cache(album: str) -> dict: async def get_duplicates_cache(album: str) -> Mapping[str, Any]:
return { return {
photo["filename"]: [photo["_id"].__str__(), photo["hash"]] photo["filename"]: [photo["_id"].__str__(), photo["hash"]]
for photo in col_photos.find({"album": album}) async for photo in col_photos.find({"album": album})
} }
@ -52,9 +52,9 @@ async def get_phash(filepath: Union[str, Path]) -> str:
return hash_array_to_hash_hex(dct_block.flatten()) return hash_array_to_hash_hex(dct_block.flatten())
async def get_duplicates(hash_string: str, album: str) -> list: async def get_duplicates(hash_string: str, album: str) -> List[Mapping[str, Any]]:
duplicates = [] duplicates = []
cache = get_duplicates_cache(album) cache = await get_duplicates_cache(album)
for image_name, image_object in cache.items(): for image_name, image_object in cache.items():
try: try:
distance = spatial.distance.hamming( distance = spatial.distance.hamming(

8
modules/mailer.py
View File

@ -28,8 +28,8 @@ try:
) )
mail_sender.ehlo() mail_sender.ehlo()
logger.info("Initialized SMTP connection") logger.info("Initialized SMTP connection")
except Exception as exp: except Exception as exc:
logger.error("Could not initialize SMTP connection to: %s", exp) logger.error("Could not initialize SMTP connection to: %s", exc)
print_exc() print_exc()
try: try:
@ -37,5 +37,5 @@ try:
configGet("login", "mailer", "smtp"), configGet("password", "mailer", "smtp") configGet("login", "mailer", "smtp"), configGet("password", "mailer", "smtp")
) )
logger.info("Successfully initialized mailer") logger.info("Successfully initialized mailer")
except Exception as exp: except Exception as exc:
logger.error("Could not login into provided SMTP account due to: %s", exp) logger.error("Could not login into provided SMTP account due to: %s", exc)

32
modules/security.py
View File

@ -54,16 +54,20 @@ oauth2_scheme = OAuth2PasswordBearer(
) )
def verify_password(plain_password, hashed_password): def verify_password(plain_password, hashed_password) -> bool:
return pwd_context.verify(plain_password, hashed_password) return pwd_context.verify(plain_password, hashed_password)
def get_password_hash(password): def get_password_hash(password) -> str:
return pwd_context.hash(password) return pwd_context.hash(password)
def get_user(user: str): async def get_user(user: str) -> UserInDB:
found_user = col_users.find_one({"user": user}) found_user = await col_users.find_one({"user": user})
if found_user is None:
raise RuntimeError(f"User {user} does not exist")
return UserInDB( return UserInDB(
user=found_user["user"], user=found_user["user"],
email=found_user["email"], email=found_user["email"],
@ -72,14 +76,16 @@ def get_user(user: str):
) )
def authenticate_user(user_name: str, password: str): async def authenticate_user(user_name: str, password: str) -> Union[UserInDB, bool]:
if user := get_user(user_name): if user := await get_user(user_name):
return user if verify_password(password, user.hash) else False return user if verify_password(password, user.hash) else False
else: else:
return False return False
def create_access_token(data: dict, expires_delta: Union[timedelta, None] = None): def create_access_token(
data: dict, expires_delta: Union[timedelta, None] = None
) -> str:
to_encode = data.copy() to_encode = data.copy()
if expires_delta: if expires_delta:
expire = datetime.now(tz=timezone.utc) + expires_delta expire = datetime.now(tz=timezone.utc) + expires_delta
@ -93,7 +99,7 @@ def create_access_token(data: dict, expires_delta: Union[timedelta, None] = None
async def get_current_user( async def get_current_user(
security_scopes: SecurityScopes, token: str = Depends(oauth2_scheme) security_scopes: SecurityScopes, token: str = Depends(oauth2_scheme)
): ) -> UserInDB:
if security_scopes.scopes: if security_scopes.scopes:
authenticate_value = f'Bearer scope="{security_scopes.scope_str}"' authenticate_value = f'Bearer scope="{security_scopes.scope_str}"'
else: else:
@ -112,12 +118,12 @@ async def get_current_user(
raise credentials_exception raise credentials_exception
token_scopes = payload.get("scopes", []) token_scopes = payload.get("scopes", [])
token_data = TokenData(scopes=token_scopes, user=user) token_data = TokenData(scopes=token_scopes, user=user)
except (JWTError, ValidationError): except (JWTError, ValidationError) as exc:
raise credentials_exception raise credentials_exception from exc
user = get_user(user=token_data.user) user_record = await get_user(user=token_data.user)
if user is None: if user_record is None:
raise credentials_exception raise credentials_exception
for scope in security_scopes.scopes: for scope in security_scopes.scopes:
@ -127,7 +133,7 @@ async def get_current_user(
detail="Not enough permissions", detail="Not enough permissions",
headers={"WWW-Authenticate": authenticate_value}, headers={"WWW-Authenticate": authenticate_value},
) )
return user return user_record
async def get_current_active_user( async def get_current_active_user(

4
modules/utils.py
View File

@ -49,8 +49,8 @@ def jsonSave(contents: Union[list, dict], filepath: Union[str, Path]) -> None:
with open(filepath, "w", encoding="utf8") as file: with open(filepath, "w", encoding="utf8") as file:
file.write(dumps(contents, ensure_ascii=False, indent=4)) file.write(dumps(contents, ensure_ascii=False, indent=4))
file.close() file.close()
except Exception as exp: except Exception as exc:
logger.error("Could not save json file %s: %s\n%s", filepath, exp, format_exc()) logger.error("Could not save json file %s: %s\n%s", filepath, exc, format_exc())
return return

3
requirements.txt
View File

@ -4,8 +4,9 @@ exif==1.6.0
fastapi[all]==0.101.1 fastapi[all]==0.101.1
opencv-python~=4.8.0.74 opencv-python~=4.8.0.74
passlib~=1.7.4 passlib~=1.7.4
pymongo==4.4.1
python-jose[cryptography]~=3.3.0 python-jose[cryptography]~=3.3.0
python-magic~=0.4.27 python-magic~=0.4.27
scipy~=1.11.0 scipy~=1.11.0
ujson~=5.8.0 ujson~=5.8.0
--extra-index-url https://git.end-play.xyz/api/packages/profitroll/pypi/simple
async_pymongo==0.1.4