profitroll/PhotosAPI
profitroll/PhotosAPI
1
0
Fork 0
Code Issues 4 Pull Requests Releases 7 Wiki Activity

Merge branch 'dev'

Browse Source
This commit is contained in:
Profitroll 2023-11-25 19:12:05 +01:00
commit 4d6efac3c4
22 changed files with 361 additions and 200 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -153,5 +153,6 @@ cython_debug/
#.idea/
# Custom
.vscode
data/
.vscode/
config.json

18
README.md
View File

@ -1,7 +1,7 @@
<h1 align="center">Photos API</h1>
<p align="center">
<a href="https://git.end-play.xyz/profitroll/PhotosAPILICENSE"><img alt="License: GPL" src="https://img.shields.io/badge/License-GPL-blue"></a>
<a href="https://git.end-play.xyz/profitroll/PhotosAPI/src/branch/master/README.md"><img alt="License: GPL" src="https://img.shields.io/badge/License-GPL-blue"></a>
<a href="https://git.end-play.xyz/profitroll/PhotosAPI"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
</p>
@ -47,7 +47,8 @@ First you need to have a Python interpreter, MongoDB and optionally git. You can
1. Copy file `config_example.json` to `config.json`
2. Open `config.json` using your favorite text editor. For example `nano config.json`
3. Change `"database"` keys to match your MongoDB setup
4. Change `"external_address"` to the ip/http address you may get in responses. By default it's `"localhost"`. This is extremely useful when running behind reverse-proxy.
4. Set the key `"secret"` to your JWT secret. You can type in anything, but long secrets are recommended. You can also set environment variable `PHOTOSAPI_SECRET` as an alternative
5. Change `"external_address"` to the ip/http address you may get in responses. By default it's `"localhost"`. This is extremely useful when running behind reverse-proxy.
After configuring everything listed above your API will be able to boot, however further configuration can be done. You can read about it in [repository's wiki](https://git.end-play.xyz/profitroll/PhotosAPI/wiki/Configuration). There's no need to focus on that now, it makes more sense to configure it afterwards.
@ -58,6 +59,19 @@ First you need to have a Python interpreter, MongoDB and optionally git. You can
Learn more about available uvicorn arguments using `uvicorn --help`
## Upgrading
When a new version comes out, sometimes you want to upgrade your instance right away. Here's a checklist what to do:
1. Carefully read the patch notes of the version you want to update to and all the versions that came out between the release of your version and the one you want to upgrade to.
Breaking changes will be marked so and config updates will also be described in the patch notes
2. Make a backup of your currently working instance. This includes both the PhotosAPI and the database
3. Download the latest version using git (`git pull` if you cloned the repo in the past) or from the releases
4. Reconfigure the config if needed and apply the changes from the patch notes
5. Upgrade the dependencies in your virtual environment using `pip install -r requirements.txt`
6. Start the migration using `python photos_api.py --migrate` from your virtual environment
7. Test if everything works and troubleshoot/rollback if not
## Using as a service
It's a good practice to use your API as a systemd service on Linux. Here's a quick overview how that can be done.

20
classes/exceptions.py
View File

@ -286,3 +286,23 @@ class UserCredentialsInvalid(HTTPException):
status_code=401,
detail=self.openapi["content"]["application/json"]["example"]["detail"],
)
class UserMediaQuotaReached(HTTPException):
"""Raises HTTP 403 if user's quota has been reached."""
def __init__(self):
self.openapi = {
"description": "Media Quota Reached",
"content": {
"application/json": {
"example": {
"detail": "Media quota has been reached, media upload impossible."
}
}
},
}
super().__init__(
status_code=403,
detail=self.openapi["content"]["application/json"]["example"]["detail"],
)

2
config_example.json
View File

@ -6,6 +6,7 @@
"user": null,
"password": null
},
"secret": "",
"messages": {
"email_confirmed": "Email confirmed. You can now log in."
},
@ -14,6 +15,7 @@
"media_token_valid_hours": 12,
"registration_enabled": true,
"registration_requires_confirmation": false,
"default_user_quota": 10000,
"mailer": {
"smtp": {
"host": "",

43
extensions/albums.py
View File

@ -47,12 +47,12 @@ async def album_create(
if 2 > len(title) > 40:
raise AlbumIncorrectError("title", "must be >2 and <40 characters.")
if col_albums.find_one({"name": name}) is not None:
if (await col_albums.find_one({"name": name})) is not None:
raise AlbumAlreadyExistsError(name)
makedirs(Path(f"data/users/{current_user.user}/albums/{name}"), exist_ok=True)
uploaded = col_albums.insert_one(
uploaded = await col_albums.insert_one(
{"user": current_user.user, "name": name, "title": title, "cover": None}
)
@ -67,9 +67,10 @@ async def album_find(
current_user: User = Security(get_current_active_user, scopes=["albums.list"]),
):
output = {"results": []}
albums = list(col_albums.find({"user": current_user.user, "name": re.compile(q)}))
for album in albums:
async for album in col_albums.find(
{"user": current_user.user, "name": re.compile(q)}
):
output["results"].append(
{
"id": album["_id"].__str__(),
@ -102,11 +103,11 @@ async def album_patch(
current_user: User = Security(get_current_active_user, scopes=["albums.write"]),
):
try:
album = col_albums.find_one({"_id": ObjectId(id)})
album = await col_albums.find_one({"_id": ObjectId(id)})
if album is None:
raise InvalidId(id)
except InvalidId:
raise AlbumNotFoundError(id)
except InvalidId as exc:
raise AlbumNotFoundError(id) from exc
if title is None:
title = album["title"]
@ -125,7 +126,7 @@ async def album_patch(
Path(f"data/users/{current_user.user}/albums/{album['name']}"),
Path(f"data/users/{current_user.user}/albums/{name}"),
)
col_photos.update_many(
await col_photos.update_many(
{"user": current_user.user, "album": album["name"]},
{"$set": {"album": name}},
)
@ -133,12 +134,14 @@ async def album_patch(
name = album["name"]
if cover is not None:
image = col_photos.find_one({"_id": ObjectId(cover), "album": album["name"]})
image = await col_photos.find_one(
{"_id": ObjectId(cover), "album": album["name"]}
)
cover = image["_id"].__str__() if image is not None else album["cover"]
else:
cover = album["cover"]
col_albums.update_one(
await col_albums.update_one(
{"_id": ObjectId(id)}, {"$set": {"name": name, "title": title, "cover": cover}}
)
@ -166,11 +169,11 @@ async def album_put(
current_user: User = Security(get_current_active_user, scopes=["albums.write"]),
):
try:
album = col_albums.find_one({"_id": ObjectId(id)})
album = await col_albums.find_one({"_id": ObjectId(id)})
if album is None:
raise InvalidId(id)
except InvalidId:
raise AlbumNotFoundError(id)
except InvalidId as exc:
raise AlbumNotFoundError(id) from exc
if re.search(re.compile("^[a-z,0-9,_]*$"), name) is False:
raise AlbumIncorrectError("name", "can only contain a-z, 0-9 and _ characters.")
@ -181,7 +184,7 @@ async def album_put(
if 2 > len(title) > 40:
raise AlbumIncorrectError("title", "must be >2 and <40 characters.")
image = col_photos.find_one({"_id": ObjectId(cover), "album": album["name"]})
image = await col_photos.find_one({"_id": ObjectId(cover), "album": album["name"]})
cover = image["_id"].__str__() if image is not None else None # type: ignore
rename(
@ -189,10 +192,10 @@ async def album_put(
Path(f"data/users/{current_user.user}/albums/{name}"),
)
col_photos.update_many(
await col_photos.update_many(
{"user": current_user.user, "album": album["name"]}, {"$set": {"album": name}}
)
col_albums.update_one(
await col_albums.update_one(
{"_id": ObjectId(id)}, {"$set": {"name": name, "title": title, "cover": cover}}
)
@ -213,13 +216,13 @@ async def album_delete(
current_user: User = Security(get_current_active_user, scopes=["albums.write"]),
):
try:
album = col_albums.find_one_and_delete({"_id": ObjectId(id)})
album = await col_albums.find_one_and_delete({"_id": ObjectId(id)})
if album is None:
raise InvalidId(id)
except InvalidId:
raise AlbumNotFoundError(id)
except InvalidId as exc:
raise AlbumNotFoundError(id) from exc
col_photos.delete_many({"album": album["name"]})
await col_photos.delete_many({"album": album["name"]})
rmtree(Path(f"data/users/{current_user.user}/albums/{album['name']}"))

22
extensions/exceptions.py
View File

@ -3,6 +3,7 @@ from fastapi.responses import UJSONResponse
from starlette.status import (
HTTP_400_BAD_REQUEST,
HTTP_401_UNAUTHORIZED,
HTTP_403_FORBIDDEN,
HTTP_404_NOT_FOUND,
HTTP_406_NOT_ACCEPTABLE,
HTTP_409_CONFLICT,
@ -10,19 +11,20 @@ from starlette.status import (
)
from classes.exceptions import (
AlbumNotFoundError,
AccessTokenInvalidError,
AlbumAlreadyExistsError,
AlbumIncorrectError,
AlbumNotFoundError,
PhotoNotFoundError,
PhotoSearchQueryEmptyError,
VideoNotFoundError,
VideoSearchQueryEmptyError,
SearchPageInvalidError,
SearchTokenInvalidError,
AccessTokenInvalidError,
UserEmailCodeInvalid,
UserAlreadyExists,
UserCredentialsInvalid,
UserEmailCodeInvalid,
UserMediaQuotaReached,
VideoNotFoundError,
VideoSearchQueryEmptyError,
)
from modules.app import app
@ -155,3 +157,13 @@ async def user_credentials_invalid_exception_handler(
status_code=HTTP_401_UNAUTHORIZED,
content={"detail": "Invalid credentials."},
)
@app.exception_handler(UserMediaQuotaReached)
async def user_media_quota_reached_exception_handler(
request: Request, exc: UserMediaQuotaReached
):
return UJSONResponse(
status_code=HTTP_403_FORBIDDEN,
content={"detail": "Media quota has been reached, media upload impossible."},
)

122
extensions/photos.py
View File

@ -30,6 +30,7 @@ from classes.exceptions import (
SearchLimitInvalidError,
SearchPageInvalidError,
SearchTokenInvalidError,
UserMediaQuotaReached,
)
from classes.models import (
Photo,
@ -38,7 +39,7 @@ from classes.models import (
SearchResultsPhoto,
)
from modules.app import app
from modules.database import col_albums, col_photos, col_tokens
from modules.database import col_albums, col_photos, col_tokens, col_videos
from modules.exif_reader import extract_location
from modules.hasher import get_duplicates, get_phash
from modules.scheduler import scheduler
@ -91,6 +92,7 @@ async def compress_image(image_path: str):
photo_post_responses = {
403: UserMediaQuotaReached().openapi,
404: AlbumNameNotFoundError("name").openapi,
409: {
"description": "Image Duplicates Found",
@ -122,9 +124,16 @@ async def photo_upload(
caption: Union[str, None] = None,
current_user: User = Security(get_current_active_user, scopes=["photos.write"]),
):
if col_albums.find_one({"user": current_user.user, "name": album}) is None:
if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album)
user_media_count = (
await col_photos.count_documents({"user": current_user.user})
) + (await col_videos.count_documents({"user": current_user.user}))
if user_media_count >= current_user.quota and not current_user.quota == -1: # type: ignore
raise UserMediaQuotaReached()
makedirs(Path(f"data/users/{current_user.user}/albums/{album}"), exist_ok=True)
filename = file.filename
@ -158,7 +167,7 @@ async def photo_upload(
expires_delta=timedelta(hours=configGet("media_token_valid_hours")),
)
access_token_short = uuid4().hex[:12].lower()
col_tokens.insert_one(
await col_tokens.insert_one(
{
"short": access_token_short,
"access_token": access_token,
@ -183,7 +192,7 @@ async def photo_upload(
except (UnpackError, ValueError):
coords = {"lng": 0.0, "lat": 0.0, "alt": 0.0}
uploaded = col_photos.insert_one(
uploaded = await col_photos.insert_one(
{
"user": current_user.user,
"album": album,
@ -231,7 +240,7 @@ if configGet("media_token_access") is True:
responses=photo_get_token_responses,
)
async def photo_get_token(token: str, id: int):
db_entry = col_tokens.find_one({"short": token})
db_entry = await col_tokens.find_one({"short": token})
if db_entry is None:
raise AccessTokenInvalidError()
@ -246,24 +255,23 @@ if configGet("media_token_access") is True:
raise AccessTokenInvalidError()
token_scopes = payload.get("scopes", [])
token_data = TokenData(scopes=token_scopes, user=user)
except (JWTError, ValidationError) as exp:
print(exp, flush=True)
raise AccessTokenInvalidError()
except (JWTError, ValidationError) as exc:
raise AccessTokenInvalidError() from exc
user = get_user(user=token_data.user)
user_record = await get_user(user=token_data.user)
if id not in payload.get("allowed", []):
raise AccessTokenInvalidError()
try:
image = col_photos.find_one({"_id": ObjectId(id)})
image = await col_photos.find_one({"_id": ObjectId(id)})
if image is None:
raise InvalidId(id)
except InvalidId:
raise PhotoNotFoundError(id)
except InvalidId as exc:
raise PhotoNotFoundError(id) from exc
image_path = Path(
f"data/users/{user.user}/albums/{image['album']}/{image['filename']}"
f"data/users/{user_record.user}/albums/{image['album']}/{image['filename']}"
)
mime = Magic(mime=True).from_file(image_path)
@ -301,11 +309,11 @@ async def photo_get(
current_user: User = Security(get_current_active_user, scopes=["photos.read"]),
):
try:
image = col_photos.find_one({"_id": ObjectId(id)})
image = await col_photos.find_one({"_id": ObjectId(id)})
if image is None:
raise InvalidId(id)
except InvalidId:
raise PhotoNotFoundError(id)
except InvalidId as exc:
raise PhotoNotFoundError(id) from exc
image_path = Path(
f"data/users/{current_user.user}/albums/{image['album']}/{image['filename']}"
@ -334,13 +342,13 @@ async def photo_move(
current_user: User = Security(get_current_active_user, scopes=["photos.write"]),
):
try:
image = col_photos.find_one({"_id": ObjectId(id)})
image = await col_photos.find_one({"_id": ObjectId(id)})
if image is None:
raise InvalidId(id)
except InvalidId:
raise PhotoNotFoundError(id)
except InvalidId as exc:
raise PhotoNotFoundError(id) from exc
if col_albums.find_one({"user": current_user.user, "name": album}) is None:
if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album)
if Path(
@ -354,7 +362,7 @@ async def photo_move(
else:
filename = image["filename"]
col_photos.find_one_and_update(
await col_photos.find_one_and_update(
{"_id": ObjectId(id)},
{
"$set": {
@ -396,13 +404,13 @@ async def photo_patch(
current_user: User = Security(get_current_active_user, scopes=["photos.write"]),
):
try:
image = col_photos.find_one({"_id": ObjectId(id)})
image = await col_photos.find_one({"_id": ObjectId(id)})
if image is None:
raise InvalidId(id)
except InvalidId:
raise PhotoNotFoundError(id)
except InvalidId as exc:
raise PhotoNotFoundError(id) from exc
col_photos.find_one_and_update(
await col_photos.find_one_and_update(
{"_id": ObjectId(id)},
{"$set": {"caption": caption, "dates.modified": datetime.now(tz=timezone.utc)}},
)
@ -430,16 +438,16 @@ async def photo_delete(
current_user: User = Security(get_current_active_user, scopes=["photos.write"]),
):
try:
image = col_photos.find_one_and_delete({"_id": ObjectId(id)})
image = await col_photos.find_one_and_delete({"_id": ObjectId(id)})
if image is None:
raise InvalidId(id)
except InvalidId:
raise PhotoNotFoundError(id)
except InvalidId as exc:
raise PhotoNotFoundError(id) from exc
album = col_albums.find_one({"name": image["album"]})
album = await col_albums.find_one({"name": image["album"]})
if album is not None and album["cover"] == image["_id"].__str__():
col_albums.update_one({"name": image["album"]}, {"$set": {"cover": None}})
await col_albums.update_one({"name": image["album"]}, {"$set": {"cover": None}})
remove(
Path(
@ -469,7 +477,7 @@ async def photo_random(
limit: int = 100,
current_user: User = Security(get_current_active_user, scopes=["photos.list"]),
):
if col_albums.find_one({"user": current_user.user, "name": album}) is None:
if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album)
if limit <= 0:
@ -490,20 +498,16 @@ async def photo_random(
}
)
documents_count = col_photos.count_documents(db_query)
documents_count = await col_photos.count_documents(db_query)
skip = randint(0, documents_count - 1) if documents_count > 1 else 0
images = list(
col_photos.aggregate(
[
{"$match": db_query},
{"$skip": skip},
{"$limit": limit},
]
)
)
for image in images:
async for image in col_photos.aggregate(
[
{"$match": db_query},
{"$skip": skip},
{"$limit": limit},
]
):
output["results"].append(
{
"id": image["_id"].__str__(),
@ -543,7 +547,7 @@ async def photo_find(
current_user: User = Security(get_current_active_user, scopes=["photos.list"]),
):
if token is not None:
found_record = col_tokens.find_one({"token": token})
found_record = await col_tokens.find_one({"token": token})
if found_record is None:
raise SearchTokenInvalidError()
@ -560,7 +564,7 @@ async def photo_find(
current_user=current_user,
)
if col_albums.find_one({"user": current_user.user, "name": album}) is None:
if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album)
if page <= 0 or page_size <= 0:
@ -612,16 +616,22 @@ async def photo_find(
"filename": re.compile(q),
}
else:
db_query = {"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)} # type: ignore
db_query_count = {"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)} # type: ignore
db_query = {
"user": current_user.user,
"album": album,
"filename": re.compile(q),
"caption": re.compile(caption),
}
db_query_count = {
"user": current_user.user,
"album": album,
"filename": re.compile(q),
"caption": re.compile(caption),
}
images = list(
col_photos.find(db_query, limit=page_size, skip=skip).sort(
"dates.uploaded", DESCENDING
)
)
for image in images:
async for image in col_photos.find(db_query, limit=page_size, skip=skip).sort(
"dates.uploaded", direction=DESCENDING
):
output["results"].append(
{
"id": image["_id"].__str__(),
@ -630,9 +640,9 @@ async def photo_find(
}
)
if col_photos.count_documents(db_query_count) > page * page_size:
if (await col_photos.count_documents(db_query_count)) > page * page_size:
token = str(token_urlsafe(32))
col_tokens.insert_one(
await col_tokens.insert_one(
{
"token": token,
"query": q,

2
extensions/security.py
View File

@ -17,7 +17,7 @@ token_post_responses = {401: UserCredentialsInvalid().openapi}
@app.post("/token", response_model=Token, responses=token_post_responses)
async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends()):
user = authenticate_user(form_data.username, form_data.password)
user = await authenticate_user(form_data.username, form_data.password)
if not user:
raise UserCredentialsInvalid()
access_token_expires = timedelta(days=ACCESS_TOKEN_EXPIRE_DAYS)

29
extensions/users.py
View File

@ -41,14 +41,14 @@ async def send_confirmation(user: str, email: str):
+ f"/users/{user}/confirm?code={confirmation_code}"
),
)
col_emails.insert_one(
await col_emails.insert_one(
{"user": user, "email": email, "used": False, "code": confirmation_code}
)
logger.info(
"Sent confirmation email to '%s' with code %s", email, confirmation_code
)
except Exception as exp:
logger.error("Could not send confirmation email to '%s' due to: %s", email, exp)
except Exception as exc:
logger.error("Could not send confirmation email to '%s' due to: %s", email, exc)
@app.get("/users/me/", response_model=User)
@ -80,15 +80,15 @@ if configGet("registration_requires_confirmation") is True:
responses=user_confirm_responses,
)
async def user_confirm(user: str, code: str):
confirm_record = col_emails.find_one(
confirm_record = await col_emails.find_one(
{"user": user, "code": code, "used": False}
)
if confirm_record is None:
raise UserEmailCodeInvalid()
col_emails.find_one_and_update(
await col_emails.find_one_and_update(
{"_id": confirm_record["_id"]}, {"$set": {"used": True}}
)
col_users.find_one_and_update(
await col_users.find_one_and_update(
{"user": confirm_record["user"]}, {"$set": {"disabled": False}}
)
return UJSONResponse({"detail": configGet("email_confirmed", "messages")})
@ -103,12 +103,13 @@ if configGet("registration_enabled") is True:
async def user_create(
user: str = Form(), email: str = Form(), password: str = Form()
):
if col_users.find_one({"user": user}) is not None:
if (await col_users.find_one({"user": user})) is not None:
raise UserAlreadyExists()
col_users.insert_one(
await col_users.insert_one(
{
"user": user,
"email": email,
"quota": None,
"hash": get_password_hash(password),
"disabled": configGet("registration_requires_confirmation"),
}
@ -132,14 +133,14 @@ user_delete_responses = {401: UserCredentialsInvalid().openapi}
async def user_delete(
password: str = Form(), current_user: User = Depends(get_current_active_user)
):
user = get_user(current_user.user)
user = await get_user(current_user.user)
if not user:
return False
if not verify_password(password, user.hash):
raise UserCredentialsInvalid()
col_users.delete_many({"user": current_user.user})
col_emails.delete_many({"user": current_user.user})
col_photos.delete_many({"user": current_user.user})
col_videos.delete_many({"user": current_user.user})
col_albums.delete_many({"user": current_user.user})
await col_users.delete_many({"user": current_user.user})
await col_emails.delete_many({"user": current_user.user})
await col_photos.delete_many({"user": current_user.user})
await col_videos.delete_many({"user": current_user.user})
await col_albums.delete_many({"user": current_user.user})
return Response(status_code=HTTP_204_NO_CONTENT)

117
extensions/videos.py
View File

@ -21,6 +21,7 @@ from classes.exceptions import (
SearchLimitInvalidError,
SearchPageInvalidError,
SearchTokenInvalidError,
UserMediaQuotaReached,
VideoNotFoundError,
VideoSearchQueryEmptyError,
)
@ -31,10 +32,13 @@ from classes.models import (
VideoPublic,
)
from modules.app import app
from modules.database import col_albums, col_tokens, col_videos
from modules.database import col_albums, col_photos, col_tokens, col_videos
from modules.security import User, get_current_active_user
video_post_responses = {404: AlbumNameNotFoundError("name").openapi}
video_post_responses = {
403: UserMediaQuotaReached().openapi,
404: AlbumNameNotFoundError("name").openapi,
}
@app.post(
@ -50,9 +54,16 @@ async def video_upload(
caption: Union[str, None] = None,
current_user: User = Security(get_current_active_user, scopes=["videos.write"]),
):
if col_albums.find_one({"user": current_user.user, "name": album}) is None:
if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album)
user_media_count = (
await col_videos.count_documents({"user": current_user.user})
) + (await col_photos.count_documents({"user": current_user.user}))
if user_media_count >= current_user.quota and not current_user.quota == -1: # type: ignore
raise UserMediaQuotaReached()
makedirs(Path(f"data/users/{current_user.user}/albums/{album}"), exist_ok=True)
filename = file.filename
@ -73,7 +84,7 @@ async def video_upload(
# Coords extraction should be here
uploaded = col_videos.insert_one(
uploaded = await col_videos.insert_one(
{
"user": current_user.user,
"album": album,
@ -123,11 +134,11 @@ async def video_get(
current_user: User = Security(get_current_active_user, scopes=["videos.read"]),
):
try:
video = col_videos.find_one({"_id": ObjectId(id)})
video = await col_videos.find_one({"_id": ObjectId(id)})
if video is None:
raise InvalidId(id)
except InvalidId:
raise VideoNotFoundError(id)
except InvalidId as exc:
raise VideoNotFoundError(id) from exc
video_path = Path(
f"data/users/{current_user.user}/albums/{video['album']}/{video['filename']}"
@ -156,13 +167,13 @@ async def video_move(
current_user: User = Security(get_current_active_user, scopes=["videos.write"]),
):
try:
video = col_videos.find_one({"_id": ObjectId(id)})
video = await col_videos.find_one({"_id": ObjectId(id)})
if video is None:
raise InvalidId(id)
except InvalidId:
raise VideoNotFoundError(id)
except InvalidId as exc:
raise VideoNotFoundError(id) from exc
if col_albums.find_one({"user": current_user.user, "name": album}) is None:
if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album)
if Path(
@ -176,7 +187,7 @@ async def video_move(
else:
filename = video["filename"]
col_videos.find_one_and_update(
await col_videos.find_one_and_update(
{"_id": ObjectId(id)},
{
"$set": {
@ -218,13 +229,13 @@ async def video_patch(
current_user: User = Security(get_current_active_user, scopes=["videos.write"]),
):
try:
video = col_videos.find_one({"_id": ObjectId(id)})
video = await col_videos.find_one({"_id": ObjectId(id)})
if video is None:
raise InvalidId(id)
except InvalidId:
raise VideoNotFoundError(id)
except InvalidId as exc:
raise VideoNotFoundError(id) from exc
col_videos.find_one_and_update(
await col_videos.find_one_and_update(
{"_id": ObjectId(id)},
{"$set": {"caption": caption, "dates.modified": datetime.now(tz=timezone.utc)}},
)
@ -252,13 +263,13 @@ async def video_delete(
current_user: User = Security(get_current_active_user, scopes=["videos.write"]),
):
try:
video = col_videos.find_one_and_delete({"_id": ObjectId(id)})
video = await col_videos.find_one_and_delete({"_id": ObjectId(id)})
if video is None:
raise InvalidId(id)
except InvalidId:
raise VideoNotFoundError(id)
except InvalidId as exc:
raise VideoNotFoundError(id) from exc
album = col_albums.find_one({"name": video["album"]})
album = await col_albums.find_one({"name": video["album"]})
remove(
Path(
@ -288,7 +299,7 @@ async def video_random(
limit: int = 100,
current_user: User = Security(get_current_active_user, scopes=["videos.list"]),
):
if col_albums.find_one({"user": current_user.user, "name": album}) is None:
if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album)
if limit <= 0:
@ -309,20 +320,16 @@ async def video_random(
}
)
documents_count = col_videos.count_documents(db_query)
documents_count = await col_videos.count_documents(db_query)
skip = randint(0, documents_count - 1) if documents_count > 1 else 0
videos = list(
col_videos.aggregate(
[
{"$match": db_query},
{"$skip": skip},
{"$limit": limit},
]
)
)
for video in videos:
async for video in col_videos.aggregate(
[
{"$match": db_query},
{"$skip": skip},
{"$limit": limit},
]
):
output["results"].append(
{
"id": video["_id"].__str__(),
@ -359,7 +366,7 @@ async def video_find(
current_user: User = Security(get_current_active_user, scopes=["videos.list"]),
):
if token is not None:
found_record = col_tokens.find_one({"token": token})
found_record = await col_tokens.find_one({"token": token})
if found_record is None:
raise SearchTokenInvalidError()
@ -373,7 +380,7 @@ async def video_find(
current_user=current_user,
)
if col_albums.find_one({"user": current_user.user, "name": album}) is None:
if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
raise AlbumNameNotFoundError(album)
if page <= 0 or page_size <= 0:
@ -397,29 +404,33 @@ async def video_find(
"caption": re.compile(caption),
}
elif caption is None:
db_query = list(
col_videos.find(
{"user": current_user.user, "album": album, "filename": re.compile(q)},
limit=page_size,
skip=skip,
).sort("dates.uploaded", DESCENDING)
)
db_query = {
"user": current_user.user,
"album": album,
"filename": re.compile(q),
}
db_query_count = {
"user": current_user.user,
"album": album,
"caption": re.compile(q),
}
else:
db_query = list(col_videos.find({"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)}, limit=page_size, skip=skip).sort("dates.uploaded", DESCENDING)) # type: ignore
db_query_count = {"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)} # type: ignore
db_query = {
"user": current_user.user,
"album": album,
"filename": re.compile(q),
"caption": re.compile(caption),
}
db_query_count = {
"user": current_user.user,
"album": album,
"filename": re.compile(q),
"caption": re.compile(caption),
}
videos = list(
col_videos.find(db_query, limit=page_size, skip=skip).sort(
"dates.uploaded", DESCENDING
)
)
for video in videos:
async for video in col_videos.find(db_query, limit=page_size, skip=skip).sort(
"dates.uploaded", direction=DESCENDING
):
output["results"].append(
{
"id": video["_id"].__str__(),
@ -428,9 +439,9 @@ async def video_find(
}
)
if col_videos.count_documents(db_query_count) > page * page_size:
if (await col_videos.count_documents(db_query_count)) > page * page_size:
token = str(token_urlsafe(32))
col_tokens.insert_one(
await col_tokens.insert_one(
{
"token": token,
"query": q,

9
migrations/202311251700.py Normal file
View File

@ -0,0 +1,9 @@
from mongodb_migrations.base import BaseMigration
class Migration(BaseMigration):
def upgrade(self):
self.db.users.update_many({}, {"$set": {"quota": None}})
def downgrade(self):
self.db.test_collection.update_many({}, {"$unset": "quota"})

2
modules/app.py
View File

@ -1,7 +1,7 @@
from fastapi import FastAPI
from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
app = FastAPI(title="END PLAY Photos", docs_url=None, redoc_url=None, version="0.5")
app = FastAPI(title="END PLAY Photos", docs_url=None, redoc_url=None, version="0.6")
@app.get("/docs", include_in_schema=False)

12
modules/database.py
View File

@ -1,3 +1,4 @@
from async_pymongo import AsyncClient
from pymongo import GEOSPHERE, MongoClient
from modules.utils import configGet
@ -17,16 +18,11 @@ else:
db_config["host"], db_config["port"], db_config["name"]
)
db_client = MongoClient(con_string)
db_client = AsyncClient(con_string)
db_client_sync = MongoClient(con_string)
db = db_client.get_database(name=db_config["name"])
collections = db.list_collection_names()
for collection in ["users", "albums", "photos", "videos", "tokens", "emails"]:
if collection not in collections:
db.create_collection(collection)
col_users = db.get_collection("users")
col_albums = db.get_collection("albums")
col_photos = db.get_collection("photos")
@ -34,4 +30,4 @@ col_videos = db.get_collection("videos")
col_tokens = db.get_collection("tokens")
col_emails = db.get_collection("emails")
col_photos.create_index([("location", GEOSPHERE)])
db_client_sync[db_config["name"]]["photos"].create_index([("location", GEOSPHERE)])

4
modules/exif_reader.py
View File

@ -1,4 +1,6 @@
import contextlib
from pathlib import Path
from typing import Mapping, Union
from exif import Image
@ -21,7 +23,7 @@ def decimal_coords(coords: float, ref: str) -> float:
return round(decimal_degrees, 5)
def extract_location(filepath: str) -> dict:
def extract_location(filepath: Union[str, Path]) -> Mapping[str, float]:
"""Get location data from image
### Args:

17
modules/extensions_loader.py
View File

@ -1,6 +1,7 @@
from importlib.util import module_from_spec, spec_from_file_location
from os import getcwd, path, walk
from pathlib import Path
from typing import Union
# =================================================================================
@ -17,11 +18,15 @@ def get_py_files(src):
return py_files
def dynamic_import(module_name, py_path):
def dynamic_import(module_name: str, py_path: str):
try:
module_spec = spec_from_file_location(module_name, py_path)
module = module_from_spec(module_spec) # type: ignore
module_spec.loader.exec_module(module) # type: ignore
if module_spec is None:
raise RuntimeError(
f"Module spec from module name {module_name} and path {py_path} is None"
)
module = module_from_spec(module_spec)
module_spec.loader.exec_module(module)
return module
except SyntaxError:
print(
@ -29,12 +34,12 @@ def dynamic_import(module_name, py_path):
flush=True,
)
return
except Exception as exp:
print(f"Could not load extension {module_name} due to {exp}", flush=True)
except Exception as exc:
print(f"Could not load extension {module_name} due to {exc}", flush=True)
return
def dynamic_import_from_src(src, star_import=False):
def dynamic_import_from_src(src: Union[str, Path], star_import=False):
my_py_files = get_py_files(src)
for py_file in my_py_files:
module_name = Path(py_file).stem

12
modules/hasher.py
View File

@ -1,5 +1,5 @@
from pathlib import Path
from typing import Union
from typing import Any, List, Mapping, Union
import cv2
import numpy as np
@ -9,7 +9,7 @@ from scipy import spatial
from modules.database import col_photos
def hash_array_to_hash_hex(hash_array):
def hash_array_to_hash_hex(hash_array) -> str:
# convert hash array of 0 or 1 to hash string in hex
hash_array = np.array(hash_array, dtype=np.uint8)
hash_str = "".join(str(i) for i in 1 * hash_array.flatten())
@ -23,10 +23,10 @@ def hash_hex_to_hash_array(hash_hex) -> NDArray:
return np.array(list(array_str), dtype=np.float32)
def get_duplicates_cache(album: str) -> dict:
async def get_duplicates_cache(album: str) -> Mapping[str, Any]:
return {
photo["filename"]: [photo["_id"].__str__(), photo["hash"]]
for photo in col_photos.find({"album": album})
async for photo in col_photos.find({"album": album})
}
@ -52,9 +52,9 @@ async def get_phash(filepath: Union[str, Path]) -> str:
return hash_array_to_hash_hex(dct_block.flatten())
async def get_duplicates(hash_string: str, album: str) -> list:
async def get_duplicates(hash_string: str, album: str) -> List[Mapping[str, Any]]:
duplicates = []
cache = get_duplicates_cache(album)
cache = await get_duplicates_cache(album)
for image_name, image_object in cache.items():
try:
distance = spatial.distance.hamming(

8
modules/mailer.py
View File

@ -28,8 +28,8 @@ try:
)
mail_sender.ehlo()
logger.info("Initialized SMTP connection")
except Exception as exp:
logger.error("Could not initialize SMTP connection to: %s", exp)
except Exception as exc:
logger.error("Could not initialize SMTP connection to: %s", exc)
print_exc()
try:
@ -37,5 +37,5 @@ try:
configGet("login", "mailer", "smtp"), configGet("password", "mailer", "smtp")
)
logger.info("Successfully initialized mailer")
except Exception as exp:
logger.error("Could not login into provided SMTP account due to: %s", exp)
except Exception as exc:
logger.error("Could not login into provided SMTP account due to: %s", exc)

23
modules/migrator.py Normal file
View File

@ -0,0 +1,23 @@
from typing import Any, Mapping
from mongodb_migrations.cli import MigrationManager
from mongodb_migrations.config import Configuration
from modules.utils import configGet
def migrate_database() -> None:
"""Apply migrations from folder `migrations/` to the database"""
db_config: Mapping[str, Any] = configGet("database")
manager_config = Configuration(
{
"mongo_host": db_config["host"],
"mongo_port": db_config["port"],
"mongo_database": db_config["name"],
"mongo_username": db_config["user"],
"mongo_password": db_config["password"],
}
)
manager = MigrationManager(manager_config)
manager.run()

65
modules/security.py
View File

@ -1,4 +1,5 @@
from datetime import datetime, timedelta, timezone
from os import getenv
from typing import List, Union
from fastapi import Depends, HTTPException, Security, status
@ -8,9 +9,26 @@ from passlib.context import CryptContext
from pydantic import BaseModel, ValidationError
from modules.database import col_users
from modules.utils import configGet
try:
configGet("secret")
except KeyError as exc:
raise KeyError(
"PhotosAPI secret is not set. Secret key handling has changed in PhotosAPI 0.6.0, so you need to add the config key 'secret' to your config file."
) from exc
if configGet("secret") == "" and getenv("PHOTOSAPI_SECRET") is None:
raise KeyError(
"PhotosAPI secret is not set. Set the config key 'secret' or provide the environment variable 'PHOTOSAPI_SECRET' containing a secret string."
)
SECRET_KEY = (
getenv("PHOTOSAPI_SECRET")
if getenv("PHOTOSAPI_SECRET") is not None
else configGet("secret")
)
with open("secret_key", "r", encoding="utf-8") as f:
SECRET_KEY = f.read()
ALGORITHM = "HS256"
ACCESS_TOKEN_EXPIRE_DAYS = 180
@ -28,6 +46,7 @@ class TokenData(BaseModel):
class User(BaseModel):
user: str
email: Union[str, None] = None
quota: Union[int, None] = None
disabled: Union[bool, None] = None
@ -54,46 +73,58 @@ oauth2_scheme = OAuth2PasswordBearer(
)
def verify_password(plain_password, hashed_password):
def verify_password(plain_password, hashed_password) -> bool:
return pwd_context.verify(plain_password, hashed_password)
def get_password_hash(password):
def get_password_hash(password) -> str:
return pwd_context.hash(password)
def get_user(user: str):
found_user = col_users.find_one({"user": user})
async def get_user(user: str) -> UserInDB:
found_user = await col_users.find_one({"user": user})
if found_user is None:
raise RuntimeError(f"User {user} does not exist")
return UserInDB(
user=found_user["user"],
email=found_user["email"],
quota=found_user["quota"]
if found_user["quota"] is not None
else configGet("default_user_quota"),
disabled=found_user["disabled"],
hash=found_user["hash"],
)
def authenticate_user(user_name: str, password: str):
if user := get_user(user_name):
async def authenticate_user(user_name: str, password: str) -> Union[UserInDB, bool]:
if user := await get_user(user_name):
return user if verify_password(password, user.hash) else False
else:
return False
def create_access_token(data: dict, expires_delta: Union[timedelta, None] = None):
def create_access_token(
data: dict, expires_delta: Union[timedelta, None] = None
) -> str:
to_encode = data.copy()
if expires_delta:
expire = datetime.now(tz=timezone.utc) + expires_delta
else:
expire = datetime.now(tz=timezone.utc) + timedelta(
days=ACCESS_TOKEN_EXPIRE_DAYS
)
to_encode["exp"] = expire
return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
async def get_current_user(
security_scopes: SecurityScopes, token: str = Depends(oauth2_scheme)
):
) -> UserInDB:
if security_scopes.scopes:
authenticate_value = f'Bearer scope="{security_scopes.scope_str}"'
else:
@ -108,16 +139,18 @@ async def get_current_user(
try:
payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
user: str = payload.get("sub")
if user is None:
raise credentials_exception
token_scopes = payload.get("scopes", [])
token_data = TokenData(scopes=token_scopes, user=user)
except (JWTError, ValidationError):
raise credentials_exception
except (JWTError, ValidationError) as exc:
raise credentials_exception from exc
user = get_user(user=token_data.user)
user_record = await get_user(user=token_data.user)
if user is None:
if user_record is None:
raise credentials_exception
for scope in security_scopes.scopes:
@ -127,7 +160,8 @@ async def get_current_user(
detail="Not enough permissions",
headers={"WWW-Authenticate": authenticate_value},
)
return user
return user_record
async def get_current_active_user(
@ -135,4 +169,5 @@ async def get_current_active_user(
):
if current_user.disabled:
raise HTTPException(status_code=400, detail="Inactive user")
return current_user

4
modules/utils.py
View File

@ -49,8 +49,8 @@ def jsonSave(contents: Union[list, dict], filepath: Union[str, Path]) -> None:
with open(filepath, "w", encoding="utf8") as file:
file.write(dumps(contents, ensure_ascii=False, indent=4))
file.close()
except Exception as exp:
logger.error("Could not save json file %s: %s\n%s", filepath, exp, format_exc())
except Exception as exc:
logger.error("Could not save json file %s: %s\n%s", filepath, exc, format_exc())
return

14
photos_api.py
View File

@ -1,4 +1,5 @@
import logging
from argparse import ArgumentParser
from os import makedirs
from pathlib import Path
@ -6,6 +7,7 @@ from fastapi.responses import FileResponse
from modules.app import app
from modules.extensions_loader import dynamic_import_from_src
from modules.migrator import migrate_database
from modules.scheduler import scheduler
makedirs(Path("data/users"), exist_ok=True)
@ -27,3 +29,15 @@ dynamic_import_from_src("extensions", star_import=True)
# =================================================================================
scheduler.start()
parser = ArgumentParser(
prog="PhotosAPI",
description="Small and simple API server for saving photos and videos.",
)
parser.add_argument("--migrate", action="store_true")
args, unknown = parser.parse_known_args()
if args.migrate:
migrate_database()

13
requirements.txt
View File

@ -1,11 +1,14 @@
aiofiles==23.1.0
aiofiles==23.2.1
apscheduler~=3.10.1
exif==1.6.0
fastapi[all]==0.98.0
opencv-python~=4.7.0.72
fastapi[all]==0.104.1
mongodb-migrations==1.3.0
opencv-python~=4.8.1.78
passlib~=1.7.4
pymongo==4.4.0
pymongo>=4.3.3
python-jose[cryptography]~=3.3.0
python-magic~=0.4.27
scipy~=1.11.0
ujson~=5.8.0
ujson~=5.8.0
--extra-index-url https://git.end-play.xyz/api/packages/profitroll/pypi/simple
async_pymongo==0.1.4