Migrated from main API

modules/app.py

@@ -0,0 +1,79 @@
+from os import sep
+from fastapi import FastAPI, Security, HTTPException
+from starlette.status import HTTP_401_UNAUTHORIZED, HTTP_403_FORBIDDEN
+from fastapi.security import APIKeyQuery, APIKeyHeader, APIKeyCookie
+from fastapi.openapi.docs import get_swagger_ui_html, get_redoc_html
+from starlette.status import HTTP_401_UNAUTHORIZED
+from fastapi.openapi.models import APIKey
+
+from modules.utils import configGet, jsonLoad
+
+app = FastAPI(title="END PLAY Photos", docs_url=None, redoc_url=None, version="2.0")
+
+api_key_query = APIKeyQuery(name="apikey", auto_error=False)
+api_key_header = APIKeyHeader(name="apikey", auto_error=False)
+api_key_cookie = APIKeyCookie(name="apikey", auto_error=False)
+
+
+def get_all_api_keys():
+    return jsonLoad(f'{configGet("data_location")}{sep}api_keys.json')
+
+def get_all_expired_keys():
+    return jsonLoad(f'{configGet("data_location")}{sep}expired_keys.json')
+    
+def check_project_key(project: str, apikey: APIKey) -> bool:
+    keys = jsonLoad(f'{configGet("data_location")}{sep}api_keys.json')
+    if apikey in keys:
+        if keys[apikey] != []:
+            if project in keys[apikey]:
+                return True
+            else:
+                return False
+        else:
+            return False
+    else:
+        return False
+
+
+async def get_api_key(
+    api_key_query: str = Security(api_key_query),
+    api_key_header: str = Security(api_key_header),
+    api_key_cookie: str = Security(api_key_cookie),
+):
+
+    keys = get_all_api_keys()
+    expired = get_all_expired_keys()
+
+    def is_valid(key):
+        if (key in keys) or (key == "publickey"):
+            return True
+        else:
+            return False
+
+    if is_valid(api_key_query):
+        return api_key_query
+    elif is_valid(api_key_header):
+        return api_key_header
+    elif is_valid(api_key_cookie):
+        return api_key_cookie
+    else:
+        if (api_key_query in expired) or (api_key_header in expired) or (api_key_cookie in expired):
+            raise HTTPException(status_code=HTTP_403_FORBIDDEN, detail=configGet("key_expired", "messages"))
+        else:
+            raise HTTPException(status_code=HTTP_401_UNAUTHORIZED, detail=configGet("key_invalid", "messages"))
+
+@app.get("/docs", include_in_schema=False)
+async def custom_swagger_ui_html():
+    return get_swagger_ui_html(
+        openapi_url=app.openapi_url, # type: ignore
+        title=app.title + " - Documentation",
+        swagger_favicon_url="/favicon.ico"
+    )
+
+@app.get("/redoc", include_in_schema=False)
+async def custom_redoc_html():
+    return get_redoc_html(
+        openapi_url=app.openapi_url, # type: ignore
+        title=app.title + " - Documentation",
+        redoc_favicon_url="/favicon.ico"
+    )

modules/database.py

@@ -0,0 +1,33 @@
+from modules.utils import configGet
+from pymongo import MongoClient
+
+db_config = configGet("database")
+
+if db_config["user"] is not None and db_config["password"] is not None:
+    con_string = 'mongodb://{0}:{1}@{2}:{3}/{4}'.format(
+        db_config["user"],
+        db_config["password"],
+        db_config["host"],
+        db_config["port"],
+        db_config["name"]
+    )
+else:
+    con_string = 'mongodb://{0}:{1}/{2}'.format(
+        db_config["host"],
+        db_config["port"],
+        db_config["name"]
+    )
+
+db_client = MongoClient(con_string)
+
+db = db_client.get_database(name=db_config["name"])
+
+collections = db.list_collection_names()
+
+for collection in ["albums", "photos", "tokens"]:
+    if not collection in collections:
+        db.create_collection(collection)
+
+col_albums = db.get_collection("albums")
+col_photos = db.get_collection("photos")
+col_tokens = db.get_collection("tokens")

modules/extensions_loader.py

@@ -0,0 +1,47 @@
+from importlib.util import module_from_spec, spec_from_file_location
+from os import getcwd, path, walk
+
+#=================================================================================
+
+# Import functions
+# Took from https://stackoverflow.com/a/57892961
+def get_py_files(src):
+    cwd = getcwd() # Current Working directory
+    py_files = [] 
+    for root, dirs, files in walk(src):
+        for file in files:
+            if file.endswith(".py"):
+                py_files.append(path.join(cwd, root, file))
+    return py_files
+
+
+def dynamic_import(module_name, py_path):
+    try:
+        module_spec = spec_from_file_location(module_name, py_path)
+        module = module_from_spec(module_spec) # type: ignore
+        module_spec.loader.exec_module(module) # type: ignore
+        return module
+    except SyntaxError:
+        print(f"Could not load extension {module_name} due to invalid syntax. Check logs/errors.log for details.", flush=True)
+        return
+    except Exception as exp:
+        print(f"Could not load extension {module_name} due to {exp}", flush=True)
+        return
+
+
+def dynamic_import_from_src(src, star_import = False):
+    my_py_files = get_py_files(src)
+    for py_file in my_py_files:
+        module_name = path.split(py_file)[-1][:-3]
+        print(f"Importing {module_name} extension...", flush=True)
+        imported_module = dynamic_import(module_name, py_file)
+        if imported_module != None:
+            if star_import:
+                for obj in dir(imported_module):
+                    globals()[obj] = imported_module.__dict__[obj]
+            else:
+                globals()[module_name] = imported_module
+        print(f"Successfully loaded {module_name} extension", flush=True)
+    return
+
+#=================================================================================

modules/hasher.py

@@ -0,0 +1,55 @@
+from modules.database import col_photos
+import numpy as np
+from numpy.typing import NDArray
+from scipy import spatial
+import cv2
+
+def hash_array_to_hash_hex(hash_array):
+  # convert hash array of 0 or 1 to hash string in hex
+  hash_array = np.array(hash_array, dtype = np.uint8)
+  hash_str = ''.join(str(i) for i in 1 * hash_array.flatten())
+  return (hex(int(hash_str, 2)))
+
+def hash_hex_to_hash_array(hash_hex) -> NDArray:
+  # convert hash string in hex to hash values of 0 or 1
+  hash_str = int(hash_hex, 16)
+  array_str = bin(hash_str)[2:]
+  return np.array([i for i in array_str], dtype = np.float32)
+
+def get_duplicates_cache(album: str) -> dict:
+    output = {}
+    for photo in col_photos.find( {"album": album} ):
+        output[photo["filename"]] = [photo["_id"].__str__(), photo["hash"]]
+    return output
+
+async def get_phash(filepath: str) -> str:
+    img = cv2.imread(filepath)
+    # resize image and convert to gray scale
+    img = cv2.resize(img, (64, 64))
+    img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    img = np.array(img, dtype = np.float32)
+    # calculate dct of image 
+    dct = cv2.dct(img)
+    # to reduce hash length take only 8*8 top-left block 
+    # as this block has more information than the rest
+    dct_block = dct[: 8, : 8]
+    # caclulate mean of dct block excluding first term i.e, dct(0, 0)
+    dct_average = (dct_block.mean() * dct_block.size - dct_block[0, 0]) / (dct_block.size - 1)
+    # convert dct block to binary values based on dct_average
+    dct_block[dct_block < dct_average] = 0.0
+    dct_block[dct_block != 0] = 1.0
+    # store hash value
+    return hash_array_to_hash_hex(dct_block.flatten())
+
+async def get_duplicates(hash: str, album: str) -> list:
+    duplicates = []
+    cache = get_duplicates_cache(album)
+    for image_name in cache.keys():
+        distance = spatial.distance.hamming(
+            hash_hex_to_hash_array(cache[image_name][1]), 
+            hash_hex_to_hash_array(hash)
+        )
+        print("{0:<30} {1}".format(image_name, distance), flush=True)
+        if distance <= 0.25:
+            duplicates.append({"id": cache[image_name][0], "filename": image_name, "difference": distance})
+    return duplicates

modules/utils.py

@@ -0,0 +1,75 @@
+from typing import Any, Union
+from ujson import loads, dumps, JSONDecodeError
+from traceback import print_exc
+
+# Print to stdout and then to log
+def logWrite(message: str, debug: bool = False) -> None:
+    # save to log file and rotation is to be done
+    # logAppend(f'{message}', debug=debug)
+    print(f"{message}", flush=True)
+
+def jsonLoad(filepath: str) -> Any:
+    """Load json file
+
+    ### Args:
+        * filepath (`str`): Path to input file
+
+    ### Returns:
+        * `Any`: Some json deserializable
+    """    
+    with open(filepath, "r", encoding='utf8') as file:
+        try:
+            output = loads(file.read())
+        except JSONDecodeError:
+            logWrite(f"Could not load json file {filepath}: file seems to be incorrect!\n{print_exc()}")
+            raise
+        except FileNotFoundError:
+            logWrite(f"Could not load json file {filepath}: file does not seem to exist!\n{print_exc()}")
+            raise
+        file.close()
+    return output
+
+def jsonSave(contents: Union[list, dict], filepath: str) -> None:
+    """Save contents into json file
+
+    ### Args:
+        * contents (`Union[list, dict]`): Some json serializable
+        * filepath (`str`): Path to output file
+    """    
+    try:
+        with open(filepath, "w", encoding='utf8') as file:
+            file.write(dumps(contents, ensure_ascii=False, indent=4))
+            file.close()
+    except Exception as exp:
+        logWrite(f"Could not save json file {filepath}: {exp}\n{print_exc()}")
+    return
+
+def configGet(key: str, *args: str) -> Any:
+    """Get value of the config key
+
+    ### Args:
+        * key (`str`): The last key of the keys path.
+        * *args (`str`): Path to key like: dict[args][key].
+
+    ### Returns:
+        * `Any`: Value of provided key
+    """    
+    this_dict = jsonLoad("config.json")
+    this_key = this_dict
+    for dict_key in args:
+        this_key = this_key[dict_key]
+    return this_key[key]
+
+def apiKeyInvalid(obj):
+    obj.send_response(401)
+    obj.send_header('Content-type', 'application/json; charset=utf-8')
+    obj.end_headers()
+    obj.wfile.write(b'{"code":401, "message": "Invalid API key"}')
+    return
+
+def apiKeyExpired(obj):
+    obj.send_response(403)
+    obj.send_header('Content-type', 'application/json; charset=utf-8')
+    obj.end_headers()
+    obj.wfile.write(b'{"code":403, "message": "API key expired"}')
+    return