Merge branch 'dev'

.gitignore

@@ -153,5 +153,6 @@ cython_debug/
 #.idea/
 
 # Custom
-.vscode
+data/
+.vscode/
 config.json

README.md

@@ -1,7 +1,7 @@
 <h1 align="center">Photos API</h1>
 
 <p align="center">
-<a href="https://git.end-play.xyz/profitroll/PhotosAPILICENSE"><img alt="License: GPL" src="https://img.shields.io/badge/License-GPL-blue"></a>
+<a href="https://git.end-play.xyz/profitroll/PhotosAPI/src/branch/master/README.md"><img alt="License: GPL" src="https://img.shields.io/badge/License-GPL-blue"></a>
 <a href="https://git.end-play.xyz/profitroll/PhotosAPI"><img alt="Code style: black" src="https://img.shields.io/badge/code%20style-black-000000.svg"></a>
 </p>  
 
@@ -47,7 +47,8 @@ First you need to have a Python interpreter, MongoDB and optionally git. You can
     1. Copy file `config_example.json` to `config.json`
     2. Open `config.json` using your favorite text editor. For example `nano config.json`
     3. Change `"database"` keys to match your MongoDB setup
-    4. Change `"external_address"` to the ip/http address you may get in responses. By default it's `"localhost"`. This is extremely useful when running behind reverse-proxy.  
+    4. Set the key `"secret"` to your JWT secret. You can type in anything, but long secrets are recommended. You can also set environment variable `PHOTOSAPI_SECRET` as an alternative
+    5. Change `"external_address"` to the ip/http address you may get in responses. By default it's `"localhost"`. This is extremely useful when running behind reverse-proxy.  
 
     After configuring everything listed above your API will be able to boot, however further configuration can be done. You can read about it in [repository's wiki](https://git.end-play.xyz/profitroll/PhotosAPI/wiki/Configuration). There's no need to focus on that now, it makes more sense to configure it afterwards.
 
@@ -58,6 +59,19 @@ First you need to have a Python interpreter, MongoDB and optionally git. You can
 
     Learn more about available uvicorn arguments using `uvicorn --help`
 
+## Upgrading
+
+When a new version comes out, sometimes you want to upgrade your instance right away. Here's a checklist what to do:
+
+1. Carefully read the patch notes of the version you want to update to and all the versions that came out between the release of your version and the one you want to upgrade to.
+   Breaking changes will be marked so and config updates will also be described in the patch notes
+2. Make a backup of your currently working instance. This includes both the PhotosAPI and the database
+3. Download the latest version using git (`git pull` if you cloned the repo in the past) or from the releases
+4. Reconfigure the config if needed and apply the changes from the patch notes
+5. Upgrade the dependencies in your virtual environment using `pip install -r requirements.txt`
+6. Start the migration using `python photos_api.py --migrate` from your virtual environment
+7. Test if everything works and troubleshoot/rollback if not
+
 ## Using as a service
 
 It's a good practice to use your API as a systemd service on Linux. Here's a quick overview how that can be done.  

classes/exceptions.py

@@ -286,3 +286,23 @@ class UserCredentialsInvalid(HTTPException):
             status_code=401,
             detail=self.openapi["content"]["application/json"]["example"]["detail"],
         )
+
+
+class UserMediaQuotaReached(HTTPException):
+    """Raises HTTP 403 if user's quota has been reached."""
+
+    def __init__(self):
+        self.openapi = {
+            "description": "Media Quota Reached",
+            "content": {
+                "application/json": {
+                    "example": {
+                        "detail": "Media quota has been reached, media upload impossible."
+                    }
+                }
+            },
+        }
+        super().__init__(
+            status_code=403,
+            detail=self.openapi["content"]["application/json"]["example"]["detail"],
+        )

config_example.json

@@ -6,6 +6,7 @@
         "user": null,
         "password": null
     },
+    "secret": "",
     "messages": {
         "email_confirmed": "Email confirmed. You can now log in."
     },
@@ -14,6 +15,7 @@
     "media_token_valid_hours": 12,
     "registration_enabled": true,
     "registration_requires_confirmation": false,
+    "default_user_quota": 10000,
     "mailer": {
         "smtp": {
             "host": "",

extensions/albums.py

@@ -47,12 +47,12 @@ async def album_create(
     if 2 > len(title) > 40:
         raise AlbumIncorrectError("title", "must be >2 and <40 characters.")
 
-    if col_albums.find_one({"name": name}) is not None:
+    if (await col_albums.find_one({"name": name})) is not None:
         raise AlbumAlreadyExistsError(name)
 
     makedirs(Path(f"data/users/{current_user.user}/albums/{name}"), exist_ok=True)
 
-    uploaded = col_albums.insert_one(
+    uploaded = await col_albums.insert_one(
         {"user": current_user.user, "name": name, "title": title, "cover": None}
     )
 
@@ -67,9 +67,10 @@ async def album_find(
     current_user: User = Security(get_current_active_user, scopes=["albums.list"]),
 ):
     output = {"results": []}
-    albums = list(col_albums.find({"user": current_user.user, "name": re.compile(q)}))
 
-    for album in albums:
+    async for album in col_albums.find(
+        {"user": current_user.user, "name": re.compile(q)}
+    ):
         output["results"].append(
             {
                 "id": album["_id"].__str__(),
@@ -102,11 +103,11 @@ async def album_patch(
     current_user: User = Security(get_current_active_user, scopes=["albums.write"]),
 ):
     try:
-        album = col_albums.find_one({"_id": ObjectId(id)})
+        album = await col_albums.find_one({"_id": ObjectId(id)})
         if album is None:
             raise InvalidId(id)
-    except InvalidId:
+    except InvalidId as exc:
-        raise AlbumNotFoundError(id)
+        raise AlbumNotFoundError(id) from exc
 
     if title is None:
         title = album["title"]
@@ -125,7 +126,7 @@ async def album_patch(
             Path(f"data/users/{current_user.user}/albums/{album['name']}"),
             Path(f"data/users/{current_user.user}/albums/{name}"),
         )
-        col_photos.update_many(
+        await col_photos.update_many(
             {"user": current_user.user, "album": album["name"]},
             {"$set": {"album": name}},
         )
@@ -133,12 +134,14 @@ async def album_patch(
         name = album["name"]
 
     if cover is not None:
-        image = col_photos.find_one({"_id": ObjectId(cover), "album": album["name"]})
+        image = await col_photos.find_one(
+            {"_id": ObjectId(cover), "album": album["name"]}
+        )
         cover = image["_id"].__str__() if image is not None else album["cover"]
     else:
         cover = album["cover"]
 
-    col_albums.update_one(
+    await col_albums.update_one(
         {"_id": ObjectId(id)}, {"$set": {"name": name, "title": title, "cover": cover}}
     )
 
@@ -166,11 +169,11 @@ async def album_put(
     current_user: User = Security(get_current_active_user, scopes=["albums.write"]),
 ):
     try:
-        album = col_albums.find_one({"_id": ObjectId(id)})
+        album = await col_albums.find_one({"_id": ObjectId(id)})
         if album is None:
             raise InvalidId(id)
-    except InvalidId:
+    except InvalidId as exc:
-        raise AlbumNotFoundError(id)
+        raise AlbumNotFoundError(id) from exc
 
     if re.search(re.compile("^[a-z,0-9,_]*$"), name) is False:
         raise AlbumIncorrectError("name", "can only contain a-z, 0-9 and _ characters.")
@@ -181,7 +184,7 @@ async def album_put(
     if 2 > len(title) > 40:
         raise AlbumIncorrectError("title", "must be >2 and <40 characters.")
 
-    image = col_photos.find_one({"_id": ObjectId(cover), "album": album["name"]})
+    image = await col_photos.find_one({"_id": ObjectId(cover), "album": album["name"]})
     cover = image["_id"].__str__() if image is not None else None  # type: ignore
 
     rename(
@@ -189,10 +192,10 @@ async def album_put(
         Path(f"data/users/{current_user.user}/albums/{name}"),
     )
 
-    col_photos.update_many(
+    await col_photos.update_many(
         {"user": current_user.user, "album": album["name"]}, {"$set": {"album": name}}
     )
-    col_albums.update_one(
+    await col_albums.update_one(
         {"_id": ObjectId(id)}, {"$set": {"name": name, "title": title, "cover": cover}}
     )
 
@@ -213,13 +216,13 @@ async def album_delete(
     current_user: User = Security(get_current_active_user, scopes=["albums.write"]),
 ):
     try:
-        album = col_albums.find_one_and_delete({"_id": ObjectId(id)})
+        album = await col_albums.find_one_and_delete({"_id": ObjectId(id)})
         if album is None:
             raise InvalidId(id)
-    except InvalidId:
+    except InvalidId as exc:
-        raise AlbumNotFoundError(id)
+        raise AlbumNotFoundError(id) from exc
 
-    col_photos.delete_many({"album": album["name"]})
+    await col_photos.delete_many({"album": album["name"]})
 
     rmtree(Path(f"data/users/{current_user.user}/albums/{album['name']}"))
 

extensions/exceptions.py

@@ -3,6 +3,7 @@ from fastapi.responses import UJSONResponse
 from starlette.status import (
     HTTP_400_BAD_REQUEST,
     HTTP_401_UNAUTHORIZED,
+    HTTP_403_FORBIDDEN,
     HTTP_404_NOT_FOUND,
     HTTP_406_NOT_ACCEPTABLE,
     HTTP_409_CONFLICT,
@@ -10,19 +11,20 @@ from starlette.status import (
 )
 
 from classes.exceptions import (
-    AlbumNotFoundError,
+    AccessTokenInvalidError,
     AlbumAlreadyExistsError,
     AlbumIncorrectError,
+    AlbumNotFoundError,
     PhotoNotFoundError,
     PhotoSearchQueryEmptyError,
-    VideoNotFoundError,
-    VideoSearchQueryEmptyError,
     SearchPageInvalidError,
     SearchTokenInvalidError,
-    AccessTokenInvalidError,
-    UserEmailCodeInvalid,
     UserAlreadyExists,
     UserCredentialsInvalid,
+    UserEmailCodeInvalid,
+    UserMediaQuotaReached,
+    VideoNotFoundError,
+    VideoSearchQueryEmptyError,
 )
 from modules.app import app
 
@@ -155,3 +157,13 @@ async def user_credentials_invalid_exception_handler(
         status_code=HTTP_401_UNAUTHORIZED,
         content={"detail": "Invalid credentials."},
     )
+
+
+@app.exception_handler(UserMediaQuotaReached)
+async def user_media_quota_reached_exception_handler(
+    request: Request, exc: UserMediaQuotaReached
+):
+    return UJSONResponse(
+        status_code=HTTP_403_FORBIDDEN,
+        content={"detail": "Media quota has been reached, media upload impossible."},
+    )

extensions/photos.py

@@ -30,6 +30,7 @@ from classes.exceptions import (
     SearchLimitInvalidError,
     SearchPageInvalidError,
     SearchTokenInvalidError,
+    UserMediaQuotaReached,
 )
 from classes.models import (
     Photo,
@@ -38,7 +39,7 @@ from classes.models import (
     SearchResultsPhoto,
 )
 from modules.app import app
-from modules.database import col_albums, col_photos, col_tokens
+from modules.database import col_albums, col_photos, col_tokens, col_videos
 from modules.exif_reader import extract_location
 from modules.hasher import get_duplicates, get_phash
 from modules.scheduler import scheduler
@@ -91,6 +92,7 @@ async def compress_image(image_path: str):
 
 
 photo_post_responses = {
+    403: UserMediaQuotaReached().openapi,
     404: AlbumNameNotFoundError("name").openapi,
     409: {
         "description": "Image Duplicates Found",
@@ -122,9 +124,16 @@ async def photo_upload(
     caption: Union[str, None] = None,
     current_user: User = Security(get_current_active_user, scopes=["photos.write"]),
 ):
-    if col_albums.find_one({"user": current_user.user, "name": album}) is None:
+    if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
         raise AlbumNameNotFoundError(album)
 
+    user_media_count = (
+        await col_photos.count_documents({"user": current_user.user})
+    ) + (await col_videos.count_documents({"user": current_user.user}))
+
+    if user_media_count >= current_user.quota and not current_user.quota == -1:  # type: ignore
+        raise UserMediaQuotaReached()
+
     makedirs(Path(f"data/users/{current_user.user}/albums/{album}"), exist_ok=True)
 
     filename = file.filename
@@ -158,7 +167,7 @@ async def photo_upload(
                 expires_delta=timedelta(hours=configGet("media_token_valid_hours")),
             )
             access_token_short = uuid4().hex[:12].lower()
-            col_tokens.insert_one(
+            await col_tokens.insert_one(
                 {
                     "short": access_token_short,
                     "access_token": access_token,
@@ -183,7 +192,7 @@ async def photo_upload(
     except (UnpackError, ValueError):
         coords = {"lng": 0.0, "lat": 0.0, "alt": 0.0}
 
-    uploaded = col_photos.insert_one(
+    uploaded = await col_photos.insert_one(
         {
             "user": current_user.user,
             "album": album,
@@ -231,7 +240,7 @@ if configGet("media_token_access") is True:
         responses=photo_get_token_responses,
     )
     async def photo_get_token(token: str, id: int):
-        db_entry = col_tokens.find_one({"short": token})
+        db_entry = await col_tokens.find_one({"short": token})
 
         if db_entry is None:
             raise AccessTokenInvalidError()
@@ -246,24 +255,23 @@ if configGet("media_token_access") is True:
                 raise AccessTokenInvalidError()
             token_scopes = payload.get("scopes", [])
             token_data = TokenData(scopes=token_scopes, user=user)
-        except (JWTError, ValidationError) as exp:
+        except (JWTError, ValidationError) as exc:
-            print(exp, flush=True)
+            raise AccessTokenInvalidError() from exc
-            raise AccessTokenInvalidError()
 
-        user = get_user(user=token_data.user)
+        user_record = await get_user(user=token_data.user)
 
         if id not in payload.get("allowed", []):
             raise AccessTokenInvalidError()
 
         try:
-            image = col_photos.find_one({"_id": ObjectId(id)})
+            image = await col_photos.find_one({"_id": ObjectId(id)})
             if image is None:
                 raise InvalidId(id)
-        except InvalidId:
+        except InvalidId as exc:
-            raise PhotoNotFoundError(id)
+            raise PhotoNotFoundError(id) from exc
 
         image_path = Path(
-            f"data/users/{user.user}/albums/{image['album']}/{image['filename']}"
+            f"data/users/{user_record.user}/albums/{image['album']}/{image['filename']}"
         )
 
         mime = Magic(mime=True).from_file(image_path)
@@ -301,11 +309,11 @@ async def photo_get(
     current_user: User = Security(get_current_active_user, scopes=["photos.read"]),
 ):
     try:
-        image = col_photos.find_one({"_id": ObjectId(id)})
+        image = await col_photos.find_one({"_id": ObjectId(id)})
         if image is None:
             raise InvalidId(id)
-    except InvalidId:
+    except InvalidId as exc:
-        raise PhotoNotFoundError(id)
+        raise PhotoNotFoundError(id) from exc
 
     image_path = Path(
         f"data/users/{current_user.user}/albums/{image['album']}/{image['filename']}"
@@ -334,13 +342,13 @@ async def photo_move(
     current_user: User = Security(get_current_active_user, scopes=["photos.write"]),
 ):
     try:
-        image = col_photos.find_one({"_id": ObjectId(id)})
+        image = await col_photos.find_one({"_id": ObjectId(id)})
         if image is None:
             raise InvalidId(id)
-    except InvalidId:
+    except InvalidId as exc:
-        raise PhotoNotFoundError(id)
+        raise PhotoNotFoundError(id) from exc
 
-    if col_albums.find_one({"user": current_user.user, "name": album}) is None:
+    if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
         raise AlbumNameNotFoundError(album)
 
     if Path(
@@ -354,7 +362,7 @@ async def photo_move(
     else:
         filename = image["filename"]
 
-    col_photos.find_one_and_update(
+    await col_photos.find_one_and_update(
         {"_id": ObjectId(id)},
         {
             "$set": {
@@ -396,13 +404,13 @@ async def photo_patch(
     current_user: User = Security(get_current_active_user, scopes=["photos.write"]),
 ):
     try:
-        image = col_photos.find_one({"_id": ObjectId(id)})
+        image = await col_photos.find_one({"_id": ObjectId(id)})
         if image is None:
             raise InvalidId(id)
-    except InvalidId:
+    except InvalidId as exc:
-        raise PhotoNotFoundError(id)
+        raise PhotoNotFoundError(id) from exc
 
-    col_photos.find_one_and_update(
+    await col_photos.find_one_and_update(
         {"_id": ObjectId(id)},
         {"$set": {"caption": caption, "dates.modified": datetime.now(tz=timezone.utc)}},
     )
@@ -430,16 +438,16 @@ async def photo_delete(
     current_user: User = Security(get_current_active_user, scopes=["photos.write"]),
 ):
     try:
-        image = col_photos.find_one_and_delete({"_id": ObjectId(id)})
+        image = await col_photos.find_one_and_delete({"_id": ObjectId(id)})
         if image is None:
             raise InvalidId(id)
-    except InvalidId:
+    except InvalidId as exc:
-        raise PhotoNotFoundError(id)
+        raise PhotoNotFoundError(id) from exc
 
-    album = col_albums.find_one({"name": image["album"]})
+    album = await col_albums.find_one({"name": image["album"]})
 
     if album is not None and album["cover"] == image["_id"].__str__():
-        col_albums.update_one({"name": image["album"]}, {"$set": {"cover": None}})
+        await col_albums.update_one({"name": image["album"]}, {"$set": {"cover": None}})
 
     remove(
         Path(
@@ -469,7 +477,7 @@ async def photo_random(
     limit: int = 100,
     current_user: User = Security(get_current_active_user, scopes=["photos.list"]),
 ):
-    if col_albums.find_one({"user": current_user.user, "name": album}) is None:
+    if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
         raise AlbumNameNotFoundError(album)
 
     if limit <= 0:
@@ -490,20 +498,16 @@ async def photo_random(
         }
     )
 
-    documents_count = col_photos.count_documents(db_query)
+    documents_count = await col_photos.count_documents(db_query)
     skip = randint(0, documents_count - 1) if documents_count > 1 else 0
 
-    images = list(
+    async for image in col_photos.aggregate(
-        col_photos.aggregate(
+        [
-            [
+            {"$match": db_query},
-                {"$match": db_query},
+            {"$skip": skip},
-                {"$skip": skip},
+            {"$limit": limit},
-                {"$limit": limit},
+        ]
-            ]
+    ):
-        )
-    )
-
-    for image in images:
         output["results"].append(
             {
                 "id": image["_id"].__str__(),
@@ -543,7 +547,7 @@ async def photo_find(
     current_user: User = Security(get_current_active_user, scopes=["photos.list"]),
 ):
     if token is not None:
-        found_record = col_tokens.find_one({"token": token})
+        found_record = await col_tokens.find_one({"token": token})
 
         if found_record is None:
             raise SearchTokenInvalidError()
@@ -560,7 +564,7 @@ async def photo_find(
             current_user=current_user,
         )
 
-    if col_albums.find_one({"user": current_user.user, "name": album}) is None:
+    if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
         raise AlbumNameNotFoundError(album)
 
     if page <= 0 or page_size <= 0:
@@ -612,16 +616,22 @@ async def photo_find(
             "filename": re.compile(q),
         }
     else:
-        db_query = {"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)}  # type: ignore
+        db_query = {
-        db_query_count = {"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)}  # type: ignore
+            "user": current_user.user,
+            "album": album,
+            "filename": re.compile(q),
+            "caption": re.compile(caption),
+        }
+        db_query_count = {
+            "user": current_user.user,
+            "album": album,
+            "filename": re.compile(q),
+            "caption": re.compile(caption),
+        }
 
-    images = list(
+    async for image in col_photos.find(db_query, limit=page_size, skip=skip).sort(
-        col_photos.find(db_query, limit=page_size, skip=skip).sort(
+        "dates.uploaded", direction=DESCENDING
-            "dates.uploaded", DESCENDING
+    ):
-        )
-    )
-
-    for image in images:
         output["results"].append(
             {
                 "id": image["_id"].__str__(),
@@ -630,9 +640,9 @@ async def photo_find(
             }
         )
 
-    if col_photos.count_documents(db_query_count) > page * page_size:
+    if (await col_photos.count_documents(db_query_count)) > page * page_size:
         token = str(token_urlsafe(32))
-        col_tokens.insert_one(
+        await col_tokens.insert_one(
             {
                 "token": token,
                 "query": q,

extensions/security.py

@@ -17,7 +17,7 @@ token_post_responses = {401: UserCredentialsInvalid().openapi}
 
 @app.post("/token", response_model=Token, responses=token_post_responses)
 async def login_for_access_token(form_data: OAuth2PasswordRequestForm = Depends()):
-    user = authenticate_user(form_data.username, form_data.password)
+    user = await authenticate_user(form_data.username, form_data.password)
     if not user:
         raise UserCredentialsInvalid()
     access_token_expires = timedelta(days=ACCESS_TOKEN_EXPIRE_DAYS)

extensions/users.py

@@ -41,14 +41,14 @@ async def send_confirmation(user: str, email: str):
                 + f"/users/{user}/confirm?code={confirmation_code}"
             ),
         )
-        col_emails.insert_one(
+        await col_emails.insert_one(
             {"user": user, "email": email, "used": False, "code": confirmation_code}
         )
         logger.info(
             "Sent confirmation email to '%s' with code %s", email, confirmation_code
         )
-    except Exception as exp:
+    except Exception as exc:
-        logger.error("Could not send confirmation email to '%s' due to: %s", email, exp)
+        logger.error("Could not send confirmation email to '%s' due to: %s", email, exc)
 
 
 @app.get("/users/me/", response_model=User)
@@ -80,15 +80,15 @@ if configGet("registration_requires_confirmation") is True:
         responses=user_confirm_responses,
     )
     async def user_confirm(user: str, code: str):
-        confirm_record = col_emails.find_one(
+        confirm_record = await col_emails.find_one(
             {"user": user, "code": code, "used": False}
         )
         if confirm_record is None:
             raise UserEmailCodeInvalid()
-        col_emails.find_one_and_update(
+        await col_emails.find_one_and_update(
             {"_id": confirm_record["_id"]}, {"$set": {"used": True}}
         )
-        col_users.find_one_and_update(
+        await col_users.find_one_and_update(
             {"user": confirm_record["user"]}, {"$set": {"disabled": False}}
         )
         return UJSONResponse({"detail": configGet("email_confirmed", "messages")})
@@ -103,12 +103,13 @@ if configGet("registration_enabled") is True:
     async def user_create(
         user: str = Form(), email: str = Form(), password: str = Form()
     ):
-        if col_users.find_one({"user": user}) is not None:
+        if (await col_users.find_one({"user": user})) is not None:
             raise UserAlreadyExists()
-        col_users.insert_one(
+        await col_users.insert_one(
             {
                 "user": user,
                 "email": email,
+                "quota": None,
                 "hash": get_password_hash(password),
                 "disabled": configGet("registration_requires_confirmation"),
             }
@@ -132,14 +133,14 @@ user_delete_responses = {401: UserCredentialsInvalid().openapi}
 async def user_delete(
     password: str = Form(), current_user: User = Depends(get_current_active_user)
 ):
-    user = get_user(current_user.user)
+    user = await get_user(current_user.user)
     if not user:
         return False
     if not verify_password(password, user.hash):
         raise UserCredentialsInvalid()
-    col_users.delete_many({"user": current_user.user})
+    await col_users.delete_many({"user": current_user.user})
-    col_emails.delete_many({"user": current_user.user})
+    await col_emails.delete_many({"user": current_user.user})
-    col_photos.delete_many({"user": current_user.user})
+    await col_photos.delete_many({"user": current_user.user})
-    col_videos.delete_many({"user": current_user.user})
+    await col_videos.delete_many({"user": current_user.user})
-    col_albums.delete_many({"user": current_user.user})
+    await col_albums.delete_many({"user": current_user.user})
     return Response(status_code=HTTP_204_NO_CONTENT)

extensions/videos.py

@@ -21,6 +21,7 @@ from classes.exceptions import (
     SearchLimitInvalidError,
     SearchPageInvalidError,
     SearchTokenInvalidError,
+    UserMediaQuotaReached,
     VideoNotFoundError,
     VideoSearchQueryEmptyError,
 )
@@ -31,10 +32,13 @@ from classes.models import (
     VideoPublic,
 )
 from modules.app import app
-from modules.database import col_albums, col_tokens, col_videos
+from modules.database import col_albums, col_photos, col_tokens, col_videos
 from modules.security import User, get_current_active_user
 
-video_post_responses = {404: AlbumNameNotFoundError("name").openapi}
+video_post_responses = {
+    403: UserMediaQuotaReached().openapi,
+    404: AlbumNameNotFoundError("name").openapi,
+}
 
 
 @app.post(
@@ -50,9 +54,16 @@ async def video_upload(
     caption: Union[str, None] = None,
     current_user: User = Security(get_current_active_user, scopes=["videos.write"]),
 ):
-    if col_albums.find_one({"user": current_user.user, "name": album}) is None:
+    if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
         raise AlbumNameNotFoundError(album)
 
+    user_media_count = (
+        await col_videos.count_documents({"user": current_user.user})
+    ) + (await col_photos.count_documents({"user": current_user.user}))
+
+    if user_media_count >= current_user.quota and not current_user.quota == -1:  # type: ignore
+        raise UserMediaQuotaReached()
+
     makedirs(Path(f"data/users/{current_user.user}/albums/{album}"), exist_ok=True)
 
     filename = file.filename
@@ -73,7 +84,7 @@ async def video_upload(
 
     # Coords extraction should be here
 
-    uploaded = col_videos.insert_one(
+    uploaded = await col_videos.insert_one(
         {
             "user": current_user.user,
             "album": album,
@@ -123,11 +134,11 @@ async def video_get(
     current_user: User = Security(get_current_active_user, scopes=["videos.read"]),
 ):
     try:
-        video = col_videos.find_one({"_id": ObjectId(id)})
+        video = await col_videos.find_one({"_id": ObjectId(id)})
         if video is None:
             raise InvalidId(id)
-    except InvalidId:
+    except InvalidId as exc:
-        raise VideoNotFoundError(id)
+        raise VideoNotFoundError(id) from exc
 
     video_path = Path(
         f"data/users/{current_user.user}/albums/{video['album']}/{video['filename']}"
@@ -156,13 +167,13 @@ async def video_move(
     current_user: User = Security(get_current_active_user, scopes=["videos.write"]),
 ):
     try:
-        video = col_videos.find_one({"_id": ObjectId(id)})
+        video = await col_videos.find_one({"_id": ObjectId(id)})
         if video is None:
             raise InvalidId(id)
-    except InvalidId:
+    except InvalidId as exc:
-        raise VideoNotFoundError(id)
+        raise VideoNotFoundError(id) from exc
 
-    if col_albums.find_one({"user": current_user.user, "name": album}) is None:
+    if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
         raise AlbumNameNotFoundError(album)
 
     if Path(
@@ -176,7 +187,7 @@ async def video_move(
     else:
         filename = video["filename"]
 
-    col_videos.find_one_and_update(
+    await col_videos.find_one_and_update(
         {"_id": ObjectId(id)},
         {
             "$set": {
@@ -218,13 +229,13 @@ async def video_patch(
     current_user: User = Security(get_current_active_user, scopes=["videos.write"]),
 ):
     try:
-        video = col_videos.find_one({"_id": ObjectId(id)})
+        video = await col_videos.find_one({"_id": ObjectId(id)})
         if video is None:
             raise InvalidId(id)
-    except InvalidId:
+    except InvalidId as exc:
-        raise VideoNotFoundError(id)
+        raise VideoNotFoundError(id) from exc
 
-    col_videos.find_one_and_update(
+    await col_videos.find_one_and_update(
         {"_id": ObjectId(id)},
         {"$set": {"caption": caption, "dates.modified": datetime.now(tz=timezone.utc)}},
     )
@@ -252,13 +263,13 @@ async def video_delete(
     current_user: User = Security(get_current_active_user, scopes=["videos.write"]),
 ):
     try:
-        video = col_videos.find_one_and_delete({"_id": ObjectId(id)})
+        video = await col_videos.find_one_and_delete({"_id": ObjectId(id)})
         if video is None:
             raise InvalidId(id)
-    except InvalidId:
+    except InvalidId as exc:
-        raise VideoNotFoundError(id)
+        raise VideoNotFoundError(id) from exc
 
-    album = col_albums.find_one({"name": video["album"]})
+    album = await col_albums.find_one({"name": video["album"]})
 
     remove(
         Path(
@@ -288,7 +299,7 @@ async def video_random(
     limit: int = 100,
     current_user: User = Security(get_current_active_user, scopes=["videos.list"]),
 ):
-    if col_albums.find_one({"user": current_user.user, "name": album}) is None:
+    if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
         raise AlbumNameNotFoundError(album)
 
     if limit <= 0:
@@ -309,20 +320,16 @@ async def video_random(
         }
     )
 
-    documents_count = col_videos.count_documents(db_query)
+    documents_count = await col_videos.count_documents(db_query)
     skip = randint(0, documents_count - 1) if documents_count > 1 else 0
 
-    videos = list(
+    async for video in col_videos.aggregate(
-        col_videos.aggregate(
+        [
-            [
+            {"$match": db_query},
-                {"$match": db_query},
+            {"$skip": skip},
-                {"$skip": skip},
+            {"$limit": limit},
-                {"$limit": limit},
+        ]
-            ]
+    ):
-        )
-    )
-
-    for video in videos:
         output["results"].append(
             {
                 "id": video["_id"].__str__(),
@@ -359,7 +366,7 @@ async def video_find(
     current_user: User = Security(get_current_active_user, scopes=["videos.list"]),
 ):
     if token is not None:
-        found_record = col_tokens.find_one({"token": token})
+        found_record = await col_tokens.find_one({"token": token})
 
         if found_record is None:
             raise SearchTokenInvalidError()
@@ -373,7 +380,7 @@ async def video_find(
             current_user=current_user,
         )
 
-    if col_albums.find_one({"user": current_user.user, "name": album}) is None:
+    if (await col_albums.find_one({"user": current_user.user, "name": album})) is None:
         raise AlbumNameNotFoundError(album)
 
     if page <= 0 or page_size <= 0:
@@ -397,29 +404,33 @@ async def video_find(
             "caption": re.compile(caption),
         }
     elif caption is None:
-        db_query = list(
+        db_query = {
-            col_videos.find(
+            "user": current_user.user,
-                {"user": current_user.user, "album": album, "filename": re.compile(q)},
+            "album": album,
-                limit=page_size,
+            "filename": re.compile(q),
-                skip=skip,
+        }
-            ).sort("dates.uploaded", DESCENDING)
-        )
         db_query_count = {
             "user": current_user.user,
             "album": album,
             "caption": re.compile(q),
         }
     else:
-        db_query = list(col_videos.find({"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)}, limit=page_size, skip=skip).sort("dates.uploaded", DESCENDING))  # type: ignore
+        db_query = {
-        db_query_count = {"user": current_user.user, "album": album, "filename": re.compile(q), "caption": re.compile(caption)}  # type: ignore
+            "user": current_user.user,
+            "album": album,
+            "filename": re.compile(q),
+            "caption": re.compile(caption),
+        }
+        db_query_count = {
+            "user": current_user.user,
+            "album": album,
+            "filename": re.compile(q),
+            "caption": re.compile(caption),
+        }
 
-    videos = list(
+    async for video in col_videos.find(db_query, limit=page_size, skip=skip).sort(
-        col_videos.find(db_query, limit=page_size, skip=skip).sort(
+        "dates.uploaded", direction=DESCENDING
-            "dates.uploaded", DESCENDING
+    ):
-        )
-    )
-
-    for video in videos:
         output["results"].append(
             {
                 "id": video["_id"].__str__(),
@@ -428,9 +439,9 @@ async def video_find(
             }
         )
 
-    if col_videos.count_documents(db_query_count) > page * page_size:
+    if (await col_videos.count_documents(db_query_count)) > page * page_size:
         token = str(token_urlsafe(32))
-        col_tokens.insert_one(
+        await col_tokens.insert_one(
             {
                 "token": token,
                 "query": q,

migrations/202311251700.py

@@ -0,0 +1,9 @@
+from mongodb_migrations.base import BaseMigration
+
+
+class Migration(BaseMigration):
+    def upgrade(self):
+        self.db.users.update_many({}, {"$set": {"quota": None}})
+
+    def downgrade(self):
+        self.db.test_collection.update_many({}, {"$unset": "quota"})

modules/app.py

@@ -1,7 +1,7 @@
 from fastapi import FastAPI
 from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
 
-app = FastAPI(title="END PLAY Photos", docs_url=None, redoc_url=None, version="0.5")
+app = FastAPI(title="END PLAY Photos", docs_url=None, redoc_url=None, version="0.6")
 
 
 @app.get("/docs", include_in_schema=False)

modules/database.py

@@ -1,3 +1,4 @@
+from async_pymongo import AsyncClient
 from pymongo import GEOSPHERE, MongoClient
 
 from modules.utils import configGet
@@ -17,16 +18,11 @@ else:
         db_config["host"], db_config["port"], db_config["name"]
     )
 
-db_client = MongoClient(con_string)
+db_client = AsyncClient(con_string)
+db_client_sync = MongoClient(con_string)
 
 db = db_client.get_database(name=db_config["name"])
 
-collections = db.list_collection_names()
-
-for collection in ["users", "albums", "photos", "videos", "tokens", "emails"]:
-    if collection not in collections:
-        db.create_collection(collection)
-
 col_users = db.get_collection("users")
 col_albums = db.get_collection("albums")
 col_photos = db.get_collection("photos")
@@ -34,4 +30,4 @@ col_videos = db.get_collection("videos")
 col_tokens = db.get_collection("tokens")
 col_emails = db.get_collection("emails")
 
-col_photos.create_index([("location", GEOSPHERE)])
+db_client_sync[db_config["name"]]["photos"].create_index([("location", GEOSPHERE)])

modules/exif_reader.py

@@ -1,4 +1,6 @@
 import contextlib
+from pathlib import Path
+from typing import Mapping, Union
 
 from exif import Image
 
@@ -21,7 +23,7 @@ def decimal_coords(coords: float, ref: str) -> float:
     return round(decimal_degrees, 5)
 
 
-def extract_location(filepath: str) -> dict:
+def extract_location(filepath: Union[str, Path]) -> Mapping[str, float]:
     """Get location data from image
 
     ### Args:

modules/extensions_loader.py

@@ -1,6 +1,7 @@
 from importlib.util import module_from_spec, spec_from_file_location
 from os import getcwd, path, walk
 from pathlib import Path
+from typing import Union
 
 # =================================================================================
 
@@ -17,11 +18,15 @@ def get_py_files(src):
     return py_files
 
 
-def dynamic_import(module_name, py_path):
+def dynamic_import(module_name: str, py_path: str):
     try:
         module_spec = spec_from_file_location(module_name, py_path)
-        module = module_from_spec(module_spec)  # type: ignore
+        if module_spec is None:
-        module_spec.loader.exec_module(module)  # type: ignore
+            raise RuntimeError(
+                f"Module spec from module name {module_name} and path {py_path} is None"
+            )
+        module = module_from_spec(module_spec)
+        module_spec.loader.exec_module(module)
         return module
     except SyntaxError:
         print(
@@ -29,12 +34,12 @@ def dynamic_import(module_name, py_path):
             flush=True,
         )
         return
-    except Exception as exp:
+    except Exception as exc:
-        print(f"Could not load extension {module_name} due to {exp}", flush=True)
+        print(f"Could not load extension {module_name} due to {exc}", flush=True)
         return
 
 
-def dynamic_import_from_src(src, star_import=False):
+def dynamic_import_from_src(src: Union[str, Path], star_import=False):
     my_py_files = get_py_files(src)
     for py_file in my_py_files:
         module_name = Path(py_file).stem

modules/hasher.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Union
+from typing import Any, List, Mapping, Union
 
 import cv2
 import numpy as np
@@ -9,7 +9,7 @@ from scipy import spatial
 from modules.database import col_photos
 
 
-def hash_array_to_hash_hex(hash_array):
+def hash_array_to_hash_hex(hash_array) -> str:
     # convert hash array of 0 or 1 to hash string in hex
     hash_array = np.array(hash_array, dtype=np.uint8)
     hash_str = "".join(str(i) for i in 1 * hash_array.flatten())
@@ -23,10 +23,10 @@ def hash_hex_to_hash_array(hash_hex) -> NDArray:
     return np.array(list(array_str), dtype=np.float32)
 
 
-def get_duplicates_cache(album: str) -> dict:
+async def get_duplicates_cache(album: str) -> Mapping[str, Any]:
     return {
         photo["filename"]: [photo["_id"].__str__(), photo["hash"]]
-        for photo in col_photos.find({"album": album})
+        async for photo in col_photos.find({"album": album})
     }
 
 
@@ -52,9 +52,9 @@ async def get_phash(filepath: Union[str, Path]) -> str:
     return hash_array_to_hash_hex(dct_block.flatten())
 
 
-async def get_duplicates(hash_string: str, album: str) -> list:
+async def get_duplicates(hash_string: str, album: str) -> List[Mapping[str, Any]]:
     duplicates = []
-    cache = get_duplicates_cache(album)
+    cache = await get_duplicates_cache(album)
     for image_name, image_object in cache.items():
         try:
             distance = spatial.distance.hamming(

modules/mailer.py

@@ -28,8 +28,8 @@ try:
         )
         mail_sender.ehlo()
         logger.info("Initialized SMTP connection")
-except Exception as exp:
+except Exception as exc:
-    logger.error("Could not initialize SMTP connection to: %s", exp)
+    logger.error("Could not initialize SMTP connection to: %s", exc)
     print_exc()
 
 try:
@@ -37,5 +37,5 @@ try:
         configGet("login", "mailer", "smtp"), configGet("password", "mailer", "smtp")
     )
     logger.info("Successfully initialized mailer")
-except Exception as exp:
+except Exception as exc:
-    logger.error("Could not login into provided SMTP account due to: %s", exp)
+    logger.error("Could not login into provided SMTP account due to: %s", exc)

modules/migrator.py

@@ -0,0 +1,23 @@
+from typing import Any, Mapping
+
+from mongodb_migrations.cli import MigrationManager
+from mongodb_migrations.config import Configuration
+
+from modules.utils import configGet
+
+
+def migrate_database() -> None:
+    """Apply migrations from folder `migrations/` to the database"""
+    db_config: Mapping[str, Any] = configGet("database")
+
+    manager_config = Configuration(
+        {
+            "mongo_host": db_config["host"],
+            "mongo_port": db_config["port"],
+            "mongo_database": db_config["name"],
+            "mongo_username": db_config["user"],
+            "mongo_password": db_config["password"],
+        }
+    )
+    manager = MigrationManager(manager_config)
+    manager.run()

modules/security.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timedelta, timezone
+from os import getenv
 from typing import List, Union
 
 from fastapi import Depends, HTTPException, Security, status
@@ -8,9 +9,26 @@ from passlib.context import CryptContext
 from pydantic import BaseModel, ValidationError
 
 from modules.database import col_users
+from modules.utils import configGet
+
+try:
+    configGet("secret")
+except KeyError as exc:
+    raise KeyError(
+        "PhotosAPI secret is not set. Secret key handling has changed in PhotosAPI 0.6.0, so you need to add the config key 'secret' to your config file."
+    ) from exc
+
+if configGet("secret") == "" and getenv("PHOTOSAPI_SECRET") is None:
+    raise KeyError(
+        "PhotosAPI secret is not set. Set the config key 'secret' or provide the environment variable 'PHOTOSAPI_SECRET' containing a secret string."
+    )
+
+SECRET_KEY = (
+    getenv("PHOTOSAPI_SECRET")
+    if getenv("PHOTOSAPI_SECRET") is not None
+    else configGet("secret")
+)
 
-with open("secret_key", "r", encoding="utf-8") as f:
-    SECRET_KEY = f.read()
 ALGORITHM = "HS256"
 ACCESS_TOKEN_EXPIRE_DAYS = 180
 
@@ -28,6 +46,7 @@ class TokenData(BaseModel):
 class User(BaseModel):
     user: str
     email: Union[str, None] = None
+    quota: Union[int, None] = None
     disabled: Union[bool, None] = None
 
 
@@ -54,46 +73,58 @@ oauth2_scheme = OAuth2PasswordBearer(
 )
 
 
-def verify_password(plain_password, hashed_password):
+def verify_password(plain_password, hashed_password) -> bool:
     return pwd_context.verify(plain_password, hashed_password)
 
 
-def get_password_hash(password):
+def get_password_hash(password) -> str:
     return pwd_context.hash(password)
 
 
-def get_user(user: str):
+async def get_user(user: str) -> UserInDB:
-    found_user = col_users.find_one({"user": user})
+    found_user = await col_users.find_one({"user": user})
+
+    if found_user is None:
+        raise RuntimeError(f"User {user} does not exist")
+
     return UserInDB(
         user=found_user["user"],
         email=found_user["email"],
+        quota=found_user["quota"]
+        if found_user["quota"] is not None
+        else configGet("default_user_quota"),
         disabled=found_user["disabled"],
         hash=found_user["hash"],
     )
 
 
-def authenticate_user(user_name: str, password: str):
+async def authenticate_user(user_name: str, password: str) -> Union[UserInDB, bool]:
-    if user := get_user(user_name):
+    if user := await get_user(user_name):
         return user if verify_password(password, user.hash) else False
     else:
         return False
 
 
-def create_access_token(data: dict, expires_delta: Union[timedelta, None] = None):
+def create_access_token(
+    data: dict, expires_delta: Union[timedelta, None] = None
+) -> str:
     to_encode = data.copy()
+
     if expires_delta:
         expire = datetime.now(tz=timezone.utc) + expires_delta
     else:
         expire = datetime.now(tz=timezone.utc) + timedelta(
             days=ACCESS_TOKEN_EXPIRE_DAYS
         )
+
     to_encode["exp"] = expire
+
     return jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
 
 
 async def get_current_user(
     security_scopes: SecurityScopes, token: str = Depends(oauth2_scheme)
-):
+) -> UserInDB:
     if security_scopes.scopes:
         authenticate_value = f'Bearer scope="{security_scopes.scope_str}"'
     else:
@@ -108,16 +139,18 @@ async def get_current_user(
     try:
         payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
         user: str = payload.get("sub")
+
         if user is None:
             raise credentials_exception
+
         token_scopes = payload.get("scopes", [])
         token_data = TokenData(scopes=token_scopes, user=user)
-    except (JWTError, ValidationError):
+    except (JWTError, ValidationError) as exc:
-        raise credentials_exception
+        raise credentials_exception from exc
 
-    user = get_user(user=token_data.user)
+    user_record = await get_user(user=token_data.user)
 
-    if user is None:
+    if user_record is None:
         raise credentials_exception
 
     for scope in security_scopes.scopes:
@@ -127,7 +160,8 @@ async def get_current_user(
                 detail="Not enough permissions",
                 headers={"WWW-Authenticate": authenticate_value},
             )
-    return user
+
+    return user_record
 
 
 async def get_current_active_user(
@@ -135,4 +169,5 @@ async def get_current_active_user(
 ):
     if current_user.disabled:
         raise HTTPException(status_code=400, detail="Inactive user")
+
     return current_user

modules/utils.py

@@ -49,8 +49,8 @@ def jsonSave(contents: Union[list, dict], filepath: Union[str, Path]) -> None:
         with open(filepath, "w", encoding="utf8") as file:
             file.write(dumps(contents, ensure_ascii=False, indent=4))
             file.close()
-    except Exception as exp:
+    except Exception as exc:
-        logger.error("Could not save json file %s: %s\n%s", filepath, exp, format_exc())
+        logger.error("Could not save json file %s: %s\n%s", filepath, exc, format_exc())
     return
 
 

photos_api.py

@@ -1,4 +1,5 @@
 import logging
+from argparse import ArgumentParser
 from os import makedirs
 from pathlib import Path
 
@@ -6,6 +7,7 @@ from fastapi.responses import FileResponse
 
 from modules.app import app
 from modules.extensions_loader import dynamic_import_from_src
+from modules.migrator import migrate_database
 from modules.scheduler import scheduler
 
 makedirs(Path("data/users"), exist_ok=True)
@@ -27,3 +29,15 @@ dynamic_import_from_src("extensions", star_import=True)
 # =================================================================================
 
 scheduler.start()
+
+parser = ArgumentParser(
+    prog="PhotosAPI",
+    description="Small and simple API server for saving photos and videos.",
+)
+
+parser.add_argument("--migrate", action="store_true")
+
+args, unknown = parser.parse_known_args()
+
+if args.migrate:
+    migrate_database()

requirements.txt

@@ -1,11 +1,14 @@
-aiofiles==23.1.0
+aiofiles==23.2.1
 apscheduler~=3.10.1
 exif==1.6.0
-fastapi[all]==0.98.0
+fastapi[all]==0.104.1
-opencv-python~=4.7.0.72
+mongodb-migrations==1.3.0
+opencv-python~=4.8.1.78
 passlib~=1.7.4
-pymongo==4.4.0
+pymongo>=4.3.3
 python-jose[cryptography]~=3.3.0
 python-magic~=0.4.27
 scipy~=1.11.0
-ujson~=5.8.0
+ujson~=5.8.0
+--extra-index-url https://git.end-play.xyz/api/packages/profitroll/pypi/simple
+async_pymongo==0.1.4