From fec40b1c447ded8e3e027d2737761f307725ef5a Mon Sep 17 00:00:00 2001 From: Renovate Date: Wed, 8 Jan 2025 08:41:33 +0200 Subject: [PATCH 01/24] Update dependency pytest-asyncio to v0.25.2 --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 1411061..42930c1 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -3,7 +3,7 @@ build==1.2.2.post1 isort==5.13.2 mypy==1.14.1 pylint==3.3.3 -pytest-asyncio==0.25.1 +pytest-asyncio==0.25.2 pytest-cov==6.0.0 pytest==8.3.4 tox==4.23.2 From a1d0b988581091c65bcbc27f686e121b4c7989a0 Mon Sep 17 00:00:00 2001 From: profitroll Date: Thu, 9 Jan 2025 22:18:57 +0100 Subject: [PATCH 02/24] Replaced ubuntu-latest with ubuntu-24.04 --- .gitea/workflows/analysis.yml | 5 ++--- .gitea/workflows/publish.yml | 33 +++++---------------------------- .gitea/workflows/tests.yml | 4 +--- 3 files changed, 8 insertions(+), 34 deletions(-) diff --git a/.gitea/workflows/analysis.yml b/.gitea/workflows/analysis.yml index 50053fc..18fc3b4 100644 --- a/.gitea/workflows/analysis.yml +++ b/.gitea/workflows/analysis.yml @@ -6,13 +6,12 @@ on: - main - dev pull_request: - types: [opened, synchronize, reopened] + types: [ opened, synchronize, reopened ] jobs: sonarcloud: name: SonarCloud - runs-on: ubuntu-latest - container: catthehacker/ubuntu:act-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: diff --git a/.gitea/workflows/publish.yml b/.gitea/workflows/publish.yml index 5809a53..678d7db 100644 --- a/.gitea/workflows/publish.yml +++ b/.gitea/workflows/publish.yml @@ -9,81 +9,58 @@ permissions: jobs: release-build: - runs-on: ubuntu-latest - container: catthehacker/ubuntu:act-latest - + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 with: python-version: "3.x" - - name: Build release distributions run: | python -m pip install build python -m build - - name: Upload distributions uses: christopherhx/gitea-upload-artifact@v4 with: name: release-dists path: dist/ - gitea-publish: - runs-on: ubuntu-latest - container: catthehacker/ubuntu:act-latest - - needs: - - release-build - + runs-on: ubuntu-24.04 + needs: release-build permissions: id-token: write - environment: name: gitea url: https://git.end-play.xyz/profitroll/-/packages/pypi/libbot - env: GITHUB_WORKFLOW_REF: ${{ gitea.workflow_ref }} INPUT_REPOSITORY_URL: https://git.end-play.xyz/api/packages/profitroll/pypi - steps: - name: Retrieve release distributions uses: christopherhx/gitea-download-artifact@v4 with: name: release-dists path: dist/ - - name: Publish package distributions to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_GITEA_API_TOKEN }} repository-url: https://git.end-play.xyz/api/packages/profitroll/pypi - pypi-publish: - runs-on: ubuntu-latest - container: catthehacker/ubuntu:act-latest - - needs: - - release-build - + runs-on: ubuntu-24.04 + needs: release-build permissions: id-token: write - environment: name: pypi - env: GITHUB_WORKFLOW_REF: ${{ gitea.workflow_ref }} - steps: - name: Retrieve release distributions uses: christopherhx/gitea-download-artifact@v4 with: name: release-dists path: dist/ - - name: Publish package distributions to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index 0fa3d13..1f19e51 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -11,12 +11,10 @@ on: jobs: test: name: Build and Test - runs-on: ubuntu-latest - container: catthehacker/ubuntu:act-latest + runs-on: ubuntu-24.04 strategy: matrix: python-version: [ "3.11", "3.12", "3.13" ] - steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} From f8c6b782a1533326a16c8fed6111b72ca8f6c170 Mon Sep 17 00:00:00 2001 From: profitroll Date: Thu, 9 Jan 2025 22:32:24 +0100 Subject: [PATCH 03/24] SonarCloud doesn't like 24.04, trying 22.04 instead --- .gitea/workflows/analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/analysis.yml b/.gitea/workflows/analysis.yml index 18fc3b4..6195b46 100644 --- a/.gitea/workflows/analysis.yml +++ b/.gitea/workflows/analysis.yml @@ -11,7 +11,7 @@ on: jobs: sonarcloud: name: SonarCloud - runs-on: ubuntu-24.04 + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 with: From 651022ab6e06795bc24a5fd228c72cd368802fd0 Mon Sep 17 00:00:00 2001 From: profitroll Date: Thu, 9 Jan 2025 22:33:38 +0100 Subject: [PATCH 04/24] SonarCloud doesn't like 22.04 either, trying ubuntu-latest instead --- .gitea/workflows/analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/analysis.yml b/.gitea/workflows/analysis.yml index 6195b46..cf7d38a 100644 --- a/.gitea/workflows/analysis.yml +++ b/.gitea/workflows/analysis.yml @@ -11,7 +11,7 @@ on: jobs: sonarcloud: name: SonarCloud - runs-on: ubuntu-22.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: From 9021eac87b4549450e3f5a35b1ac1263656c301f Mon Sep 17 00:00:00 2001 From: profitroll Date: Thu, 9 Jan 2025 22:35:53 +0100 Subject: [PATCH 05/24] SonarCloud action is deprecated, replacing with sonarqube one and returning to ubuntu-24.04 --- .gitea/workflows/analysis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/analysis.yml b/.gitea/workflows/analysis.yml index cf7d38a..691cf90 100644 --- a/.gitea/workflows/analysis.yml +++ b/.gitea/workflows/analysis.yml @@ -11,13 +11,13 @@ on: jobs: sonarcloud: name: SonarCloud - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - name: SonarCloud Scan - uses: SonarSource/sonarcloud-github-action@master + uses: SonarSource/sonarqube-scan-action@v4.2.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file From 82542de0bbeeb512eef544b3bb218846dcb5bba9 Mon Sep 17 00:00:00 2001 From: profitroll Date: Thu, 9 Jan 2025 22:40:32 +0100 Subject: [PATCH 06/24] SonarQube doesn't seem to work, one more try with latest --- .gitea/workflows/analysis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/analysis.yml b/.gitea/workflows/analysis.yml index 691cf90..4df6aeb 100644 --- a/.gitea/workflows/analysis.yml +++ b/.gitea/workflows/analysis.yml @@ -11,12 +11,12 @@ on: jobs: sonarcloud: name: SonarCloud - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: SonarCloud Scan + - name: SonarQube Scan uses: SonarSource/sonarqube-scan-action@v4.2.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} From ed7fa50dbd440a6ef98f65ee8352ad26143a7499 Mon Sep 17 00:00:00 2001 From: profitroll Date: Thu, 9 Jan 2025 22:54:21 +0100 Subject: [PATCH 07/24] SonarQube works like shit, switching back to the old SonarCloud action --- .gitea/workflows/analysis.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.gitea/workflows/analysis.yml b/.gitea/workflows/analysis.yml index 4df6aeb..3fe3ce3 100644 --- a/.gitea/workflows/analysis.yml +++ b/.gitea/workflows/analysis.yml @@ -12,12 +12,13 @@ jobs: sonarcloud: name: SonarCloud runs-on: ubuntu-latest + container: catthehacker/ubuntu:act-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: SonarQube Scan - uses: SonarSource/sonarqube-scan-action@v4.2.1 + - name: SonarCloud Scan + uses: SonarSource/sonarcloud-github-action@master env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file From 974aebfd1ae572164b868b8f18c85df5dc7f5eb8 Mon Sep 17 00:00:00 2001 From: profitroll Date: Thu, 9 Jan 2025 23:08:09 +0100 Subject: [PATCH 08/24] Bruh, works exactly as bad. I give up... Let's cache this shit. --- .gitea/workflows/analysis.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/analysis.yml b/.gitea/workflows/analysis.yml index 3fe3ce3..4e10a71 100644 --- a/.gitea/workflows/analysis.yml +++ b/.gitea/workflows/analysis.yml @@ -11,14 +11,18 @@ on: jobs: sonarcloud: name: SonarCloud - runs-on: ubuntu-latest - container: catthehacker/ubuntu:act-latest + runs-on: ubuntu-24.04 steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: SonarCloud Scan - uses: SonarSource/sonarcloud-github-action@master + - name: SonarQube Cache + uses: actions/cache@v4 + with: + path: ${{ gitea.workspace }}/.sonar/cache + key: ${{ runner.os }}-sonar + - name: SonarQube Scan + uses: SonarSource/sonarqube-scan-action@v4.2.1 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file From 1ca126829b04035290a4733aac0b0a872eb69f6e Mon Sep 17 00:00:00 2001 From: profitroll Date: Fri, 10 Jan 2025 00:09:05 +0100 Subject: [PATCH 09/24] Hopefully fixed caching --- .gitea/workflows/analysis.yml | 6 +----- .gitea/workflows/publish.yml | 1 + .gitea/workflows/tests.yml | 1 + 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.gitea/workflows/analysis.yml b/.gitea/workflows/analysis.yml index 4e10a71..fa7e2b9 100644 --- a/.gitea/workflows/analysis.yml +++ b/.gitea/workflows/analysis.yml @@ -16,11 +16,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: SonarQube Cache - uses: actions/cache@v4 - with: - path: ${{ gitea.workspace }}/.sonar/cache - key: ${{ runner.os }}-sonar + - uses: SonarActions/cache@v1 - name: SonarQube Scan uses: SonarSource/sonarqube-scan-action@v4.2.1 env: diff --git a/.gitea/workflows/publish.yml b/.gitea/workflows/publish.yml index 678d7db..ffe785f 100644 --- a/.gitea/workflows/publish.yml +++ b/.gitea/workflows/publish.yml @@ -15,6 +15,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.x" + cache: 'pip' - name: Build release distributions run: | python -m pip install build diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index 1f19e51..1a6db50 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -21,6 +21,7 @@ jobs: uses: actions/setup-python@v3 with: python-version: ${{ matrix.python-version }} + cache: 'pip' env: AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache - name: Install dependencies From 129cbd923b78fd8272ae0e677255b9f7744eaa00 Mon Sep 17 00:00:00 2001 From: Profitroll Date: Fri, 10 Jan 2025 11:22:33 +0200 Subject: [PATCH 10/24] Updated cache path for tests --- .gitea/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitea/workflows/tests.yml b/.gitea/workflows/tests.yml index 1a6db50..11e5a27 100644 --- a/.gitea/workflows/tests.yml +++ b/.gitea/workflows/tests.yml @@ -22,6 +22,7 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' + cache-dependency-path: './requirements/*' env: AGENT_TOOLSDIRECTORY: /opt/hostedtoolcache - name: Install dependencies From c5e83c17d39f35e6a5448ed85e11fbaff54d7353 Mon Sep 17 00:00:00 2001 From: Profitroll Date: Fri, 10 Jan 2025 11:23:35 +0200 Subject: [PATCH 11/24] Disabled pip cache for publish because dependencies are inline --- .gitea/workflows/publish.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitea/workflows/publish.yml b/.gitea/workflows/publish.yml index ffe785f..678d7db 100644 --- a/.gitea/workflows/publish.yml +++ b/.gitea/workflows/publish.yml @@ -15,7 +15,6 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.x" - cache: 'pip' - name: Build release distributions run: | python -m pip install build From 44d07dc56a0342f59803fa993597f31c6dcecc2e Mon Sep 17 00:00:00 2001 From: Renovate Date: Tue, 21 Jan 2025 20:32:47 +0200 Subject: [PATCH 12/24] Update dependency tox to v4.24.0 --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 42930c1..12f526e 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,7 +6,7 @@ pylint==3.3.3 pytest-asyncio==0.25.2 pytest-cov==6.0.0 pytest==8.3.4 -tox==4.23.2 +tox==4.24.0 twine==6.0.1 types-aiofiles==24.1.0.20241221 types-ujson==5.10.0.20240515 \ No newline at end of file From 475eaf9ff36bd730610c5b07c3f8986195b31673 Mon Sep 17 00:00:00 2001 From: Renovate Date: Tue, 21 Jan 2025 21:37:09 +0200 Subject: [PATCH 13/24] Update dependency twine to v6.1.0 --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 12f526e..eac4e6f 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -7,6 +7,6 @@ pytest-asyncio==0.25.2 pytest-cov==6.0.0 pytest==8.3.4 tox==4.24.0 -twine==6.0.1 +twine==6.1.0 types-aiofiles==24.1.0.20241221 types-ujson==5.10.0.20240515 \ No newline at end of file From 258b46d829e2c750fd18474ef3a95b0ac1a9e20b Mon Sep 17 00:00:00 2001 From: Renovate Date: Tue, 28 Jan 2025 15:50:37 +0200 Subject: [PATCH 14/24] Update dependency pylint to v3.3.4 --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index eac4e6f..87747c9 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,7 +2,7 @@ black==24.10.0 build==1.2.2.post1 isort==5.13.2 mypy==1.14.1 -pylint==3.3.3 +pylint==3.3.4 pytest-asyncio==0.25.2 pytest-cov==6.0.0 pytest==8.3.4 From 8562d7e84cb182b8a9300114b68b1681c70fc7ec Mon Sep 17 00:00:00 2001 From: Renovate Date: Tue, 28 Jan 2025 21:04:58 +0200 Subject: [PATCH 15/24] Update dependency pytest-asyncio to v0.25.3 --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 87747c9..03ab9ac 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -3,7 +3,7 @@ build==1.2.2.post1 isort==5.13.2 mypy==1.14.1 pylint==3.3.4 -pytest-asyncio==0.25.2 +pytest-asyncio==0.25.3 pytest-cov==6.0.0 pytest==8.3.4 tox==4.24.0 From 5fc8ae6a6ed841bf59d978292cb1aed5e52d0a11 Mon Sep 17 00:00:00 2001 From: Renovate Date: Wed, 29 Jan 2025 06:29:48 +0200 Subject: [PATCH 16/24] Update dependency black to v25 --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 87747c9..df3a771 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,4 +1,4 @@ -black==24.10.0 +black==25.1.0 build==1.2.2.post1 isort==5.13.2 mypy==1.14.1 From ad70648ea217e0231c267341c3082565f5838567 Mon Sep 17 00:00:00 2001 From: Renovate Date: Wed, 5 Feb 2025 06:31:00 +0200 Subject: [PATCH 17/24] Update dependency mypy to v1.15.0 --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 680894c..167d6ef 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,7 +1,7 @@ black==25.1.0 build==1.2.2.post1 isort==5.13.2 -mypy==1.14.1 +mypy==1.15.0 pylint==3.3.4 pytest-asyncio==0.25.3 pytest-cov==6.0.0 From 6b2be480529c57e4a60e8e21c6579be794b25c30 Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 9 Feb 2025 18:40:30 +0100 Subject: [PATCH 18/24] Closes #187 and improves documentation --- src/libbot/utils/config/_functions.py | 12 ++++----- src/libbot/utils/misc/_functions.py | 36 ++++++++++++++------------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src/libbot/utils/config/_functions.py b/src/libbot/utils/config/_functions.py index 8e67437..7aeb285 100644 --- a/src/libbot/utils/config/_functions.py +++ b/src/libbot/utils/config/_functions.py @@ -20,7 +20,7 @@ def config_get(key: str, *path: str, config_file: str | Path = DEFAULT_CONFIG_LO ### Args: * key (`str`): Key that contains the value - * *path (`str`): Path to the key that contains the value + * *path (`str`): Path to the key that contains the value (pass *[] or don't pass anything at all to get on the top/root level) * config_file (`str | Path`, *optional*): Path-like object or path as a string of a location of the config file. Defaults to `"config.json"` ### Returns: @@ -59,7 +59,7 @@ async def config_get(key: str, *path: str, config_file: str | Path = DEFAULT_CON ### Args: * key (`str`): Key that contains the value - * *path (`str`): Path to the key that contains the value + * *path (`str`): Path to the key that contains the value (pass *[] or don't pass anything at all to get on the top/root level) * config_file (`str | Path`, *optional*): Path-like object or path as a string of a location of the config file. Defaults to `"config.json"` ### Returns: @@ -98,7 +98,7 @@ def config_set(key: str, value: Any, *path: str, config_file: str | Path = DEFAU ### Args: * key (`str`): Key that leads to the value * value (`Any`): Any JSON serializable data - * *path (`str`): Path to the key of the target + * *path (`str`): Path to the key of the target (pass *[] or don't pass anything at all to set on the top/root level) * config_file (`str | Path`, *optional*): Path-like object or path as a string of a location of the config file. Defaults to `"config.json"` ### Raises: @@ -116,7 +116,7 @@ async def config_set( ### Args: * key (`str`): Key that leads to the value * value (`Any`): Any JSON serializable data - * *path (`str`): Path to the key of the target + * *path (`str`): Path to the key of the target (pass *[] or don't pass anything at all to set on the top/root level) * config_file (`str | Path`, *optional*): Path-like object or path as a string of a location of the config file. Defaults to `"config.json"` ### Raises: @@ -136,7 +136,7 @@ def config_delete( ### Args: * key (`str`): Key to delete - * *path (`str`): Path to the key of the target + * *path (`str`): Path to the key of the target (pass *[] or don't pass anything at all to delete on the top/root level) * missing_ok (`bool`): Do not raise an exception if the key is missing. Defaults to `False` * config_file (`str | Path`, *optional*): Path-like object or path as a string of a location of the config file. Defaults to `"config.json"` @@ -165,7 +165,7 @@ async def config_delete( ### Args: * key (`str`): Key to delete - * *path (`str`): Path to the key of the target + * *path (`str`): Path to the key of the target (pass *[] or don't pass anything at all to delete on the top/root level) * missing_ok (`bool`): Do not raise an exception if the key is missing. Defaults to `False` * config_file (`str | Path`, *optional*): Path-like object or path as a string of a location of the config file. Defaults to `"config.json"` diff --git a/src/libbot/utils/misc/_functions.py b/src/libbot/utils/misc/_functions.py index 2122542..87c84cf 100644 --- a/src/libbot/utils/misc/_functions.py +++ b/src/libbot/utils/misc/_functions.py @@ -3,11 +3,11 @@ from typing import Any, Dict from typing import Callable -def supports_argument(func: Callable, arg_name: str) -> bool: +def supports_argument(func: Callable[..., Any], arg_name: str) -> bool: """Check whether a function has a specific argument ### Args: - * func (`Callable`): Function to be inspected + * func (`Callable[..., Any]`): Function to be inspected * arg_name (`str`): Argument to be checked ### Returns: @@ -24,11 +24,13 @@ def supports_argument(func: Callable, arg_name: str) -> bool: return False -def nested_set(target: dict, value: Any, *path: str, create_missing=True) -> Dict[str, Any]: +def nested_set( + target: Dict[str, Any], value: Any, *path: str, create_missing: bool = True +) -> Dict[str, Any]: """Set the key by its path to the value ### Args: - * target (`dict`): Dictionary to perform modifications on + * target (`Dict[str, Any]`): Dictionary to perform modifications on * value (`Any`): Any data * *path (`str`): Path to the key of the target * create_missing (`bool`, *optional*): Create keys on the way if they're missing. Defaults to `True` @@ -39,29 +41,29 @@ def nested_set(target: dict, value: Any, *path: str, create_missing=True) -> Dic ### Returns: * `Dict[str, Any]`: Changed dictionary """ - d = target + target_copy: Dict[str, Any] = target for key in path[:-1]: - if key in d: - d = d[key] + if key in target_copy: + target_copy = target_copy[key] elif create_missing: - d = d.setdefault(key, {}) + target_copy = target_copy.setdefault(key, {}) else: raise KeyError( f"Key '{key}' is not found under path provided ({path}) and create_missing is False" ) - if path[-1] in d or create_missing: - d[path[-1]] = value + if path[-1] in target_copy or create_missing: + target_copy[path[-1]] = value return target -def nested_delete(target: dict, *path: str) -> Dict[str, Any]: +def nested_delete(target: Dict[str, Any], *path: str) -> Dict[str, Any]: """Delete the key by its path ### Args: - * target (`dict`): Dictionary to perform modifications on + * target (`Dict[str, Any]`): Dictionary to perform modifications on ### Raises: * `KeyError`: Key is not found under path provided @@ -69,16 +71,16 @@ def nested_delete(target: dict, *path: str) -> Dict[str, Any]: ### Returns: `Dict[str, Any]`: Changed dictionary """ - d = target + target_copy: Dict[str, Any] = target for key in path[:-1]: - if key in d: - d = d[key] + if key in target_copy: + target_copy = target_copy[key] else: raise KeyError(f"Key '{key}' is not found under path provided ({path})") - if path[-1] in d: - del d[path[-1]] + if path[-1] in target_copy: + del target_copy[path[-1]] else: raise KeyError(f"Key '{path[-1]}' is not found under path provided ({path})") From 845a69491d157535f31dd3b13a5f9be6fc2aef70 Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 9 Feb 2025 18:51:48 +0100 Subject: [PATCH 19/24] Silly attempt to fix token issues --- .gitea/workflows/analysis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitea/workflows/analysis.yml b/.gitea/workflows/analysis.yml index fa7e2b9..a621b33 100644 --- a/.gitea/workflows/analysis.yml +++ b/.gitea/workflows/analysis.yml @@ -20,5 +20,5 @@ jobs: - name: SonarQube Scan uses: SonarSource/sonarqube-scan-action@v4.2.1 env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file From e9abed27f8f54720a6094dc41edacb101f85d624 Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 9 Feb 2025 18:55:26 +0100 Subject: [PATCH 20/24] Removed caching action --- .gitea/workflows/analysis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitea/workflows/analysis.yml b/.gitea/workflows/analysis.yml index a621b33..4fc1edb 100644 --- a/.gitea/workflows/analysis.yml +++ b/.gitea/workflows/analysis.yml @@ -16,9 +16,8 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: SonarActions/cache@v1 - name: SonarQube Scan uses: SonarSource/sonarqube-scan-action@v4.2.1 env: - GITHUB_TOKEN: ${{ secrets.GITEA_TOKEN }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} \ No newline at end of file From 554b5224001d29c53eee5289380cad6101f70c9e Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 16 Feb 2025 16:45:22 +0100 Subject: [PATCH 21/24] Added experimental cache support --- pyproject.toml | 1 + requirements/cache.txt | 2 + src/libbot/cache/__init__.py | 2 + src/libbot/cache/classes/__init__.py | 3 + src/libbot/cache/classes/cache.py | 44 ++++++++++ src/libbot/cache/classes/cache_memcached.py | 89 +++++++++++++++++++++ src/libbot/cache/classes/cache_redis.py | 89 +++++++++++++++++++++ src/libbot/cache/utils/__init__.py | 1 + src/libbot/cache/utils/_objects.py | 42 ++++++++++ src/libbot/cache/utils/manager.py | 24 ++++++ 10 files changed, 297 insertions(+) create mode 100644 requirements/cache.txt create mode 100644 src/libbot/cache/__init__.py create mode 100644 src/libbot/cache/classes/__init__.py create mode 100644 src/libbot/cache/classes/cache.py create mode 100644 src/libbot/cache/classes/cache_memcached.py create mode 100644 src/libbot/cache/classes/cache_redis.py create mode 100644 src/libbot/cache/utils/__init__.py create mode 100644 src/libbot/cache/utils/_objects.py create mode 100644 src/libbot/cache/utils/manager.py diff --git a/pyproject.toml b/pyproject.toml index 5748f8d..9fc3893 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dev = { file = "requirements/dev.txt" } pycord = { file = "requirements/pycord.txt" } pyrogram = { file = "requirements/pyrogram.txt" } speed = { file = "requirements/speed.txt" } +cache = { file = "requirements/cache.txt" } [project.urls] Source = "https://git.end-play.xyz/profitroll/LibBotUniversal" diff --git a/requirements/cache.txt b/requirements/cache.txt new file mode 100644 index 0000000..f0f2e14 --- /dev/null +++ b/requirements/cache.txt @@ -0,0 +1,2 @@ +pymemcache~=4.0.0 +redis~=5.2.1 \ No newline at end of file diff --git a/src/libbot/cache/__init__.py b/src/libbot/cache/__init__.py new file mode 100644 index 0000000..def5795 --- /dev/null +++ b/src/libbot/cache/__init__.py @@ -0,0 +1,2 @@ +# This file is left empty on purpose +# Adding imports here will cause import errors when libbot[pycord] is not installed diff --git a/src/libbot/cache/classes/__init__.py b/src/libbot/cache/classes/__init__.py new file mode 100644 index 0000000..dce54a8 --- /dev/null +++ b/src/libbot/cache/classes/__init__.py @@ -0,0 +1,3 @@ +from .cache import Cache +from .cache_memcached import CacheMemcached +from .cache_redis import CacheRedis diff --git a/src/libbot/cache/classes/cache.py b/src/libbot/cache/classes/cache.py new file mode 100644 index 0000000..8b0f617 --- /dev/null +++ b/src/libbot/cache/classes/cache.py @@ -0,0 +1,44 @@ +from abc import ABC, abstractmethod +from typing import Any, Dict + +import pymemcache +import redis + + +class Cache(ABC): + client: pymemcache.Client | redis.Redis + + @classmethod + @abstractmethod + def from_config(cls, engine_config: Dict[str, Any]) -> Any: + pass + + @abstractmethod + def get_json(self, key: str) -> Any | None: + # TODO This method must also carry out ObjectId conversion! + pass + + @abstractmethod + def get_string(self, key: str) -> str | None: + pass + + @abstractmethod + def get_object(self, key: str) -> Any | None: + pass + + @abstractmethod + def set_json(self, key: str, value: Any) -> None: + # TODO This method must also carry out ObjectId conversion! + pass + + @abstractmethod + def set_string(self, key: str, value: str) -> None: + pass + + @abstractmethod + def set_object(self, key: str, value: Any) -> None: + pass + + @abstractmethod + def delete(self, key: str) -> None: + pass diff --git a/src/libbot/cache/classes/cache_memcached.py b/src/libbot/cache/classes/cache_memcached.py new file mode 100644 index 0000000..41b654c --- /dev/null +++ b/src/libbot/cache/classes/cache_memcached.py @@ -0,0 +1,89 @@ +import logging +from logging import Logger +from typing import Dict, Any + +from pymemcache import Client + +from .cache import Cache +from ..utils._objects import _json_to_string, _string_to_json + +logger: Logger = logging.getLogger(__name__) + + +class CacheMemcached(Cache): + client: Client + + def __init__(self, client: Client): + self.client = client + + logger.info("Initialized Memcached for caching") + + @classmethod + def from_config(cls, engine_config: Dict[str, Any]) -> "CacheMemcached": + if "uri" not in engine_config: + raise KeyError( + "Cache configuration is invalid. Please check if all keys are set (engine: memcached)" + ) + + return cls(Client(engine_config["uri"], default_noreply=True)) + + def get_json(self, key: str) -> Any | None: + try: + result: Any | None = self.client.get(key, None) + + logger.debug( + "Got json cache key '%s'%s", + key, + "" if result is not None else " (not found)", + ) + except Exception as exc: + logger.error("Could not get json cache key '%s' due to: %s", key, exc) + return None + + return None if result is None else _string_to_json(result) + + def get_string(self, key: str) -> str | None: + try: + result: str | None = self.client.get(key, None) + + logger.debug( + "Got string cache key '%s'%s", + key, + "" if result is not None else " (not found)", + ) + + return result + except Exception as exc: + logger.error("Could not get string cache key '%s' due to: %s", key, exc) + return None + + # TODO Implement binary deserialization + def get_object(self, key: str) -> Any | None: + raise NotImplementedError() + + def set_json(self, key: str, value: Any) -> None: + try: + self.client.set(key, _json_to_string(value)) + logger.debug("Set json cache key '%s'", key) + except Exception as exc: + logger.error("Could not set json cache key '%s' due to: %s", key, exc) + return None + + def set_string(self, key: str, value: str) -> None: + try: + self.client.set(key, value) + logger.debug("Set string cache key '%s'", key) + except Exception as exc: + logger.error("Could not set string cache key '%s' due to: %s", key, exc) + return None + + # TODO Implement binary serialization + def set_object(self, key: str, value: Any) -> None: + raise NotImplementedError() + + def delete(self, key: str) -> None: + try: + self.client.delete(key) + logger.debug("Deleted cache key '%s'", key) + except Exception as exc: + logger.error("Could not delete cache key '%s' due to: %s", key, exc) diff --git a/src/libbot/cache/classes/cache_redis.py b/src/libbot/cache/classes/cache_redis.py new file mode 100644 index 0000000..2edfa43 --- /dev/null +++ b/src/libbot/cache/classes/cache_redis.py @@ -0,0 +1,89 @@ +import logging +from logging import Logger +from typing import Dict, Any + +from redis import Redis + +from .cache import Cache +from ..utils._objects import _string_to_json, _json_to_string + +logger: Logger = logging.getLogger(__name__) + + +class CacheRedis(Cache): + client: Redis + + def __init__(self, client: Redis): + self.client = client + + logger.info("Initialized Redis for caching") + + @classmethod + def from_config(cls, engine_config: Dict[str, Any]) -> Any: + if "uri" not in engine_config: + raise KeyError( + "Cache configuration is invalid. Please check if all keys are set (engine: memcached)" + ) + + return cls(Redis.from_url(engine_config["uri"])) + + def get_json(self, key: str) -> Any | None: + try: + result: Any | None = self.client.get(key) + + logger.debug( + "Got json cache key '%s'%s", + key, + "" if result is not None else " (not found)", + ) + except Exception as exc: + logger.error("Could not get json cache key '%s' due to: %s", key, exc) + return None + + return None if result is None else _string_to_json(result) + + def get_string(self, key: str) -> str | None: + try: + result: str | None = self.client.get(key) + + logger.debug( + "Got string cache key '%s'%s", + key, + "" if result is not None else " (not found)", + ) + + return result + except Exception as exc: + logger.error("Could not get string cache key '%s' due to: %s", key, exc) + return None + + # TODO Implement binary deserialization + def get_object(self, key: str) -> Any | None: + raise NotImplementedError() + + def set_json(self, key: str, value: Any) -> None: + try: + self.client.set(key, _json_to_string(value)) + logger.debug("Set json cache key '%s'", key) + except Exception as exc: + logger.error("Could not set json cache key '%s' due to: %s", key, exc) + return None + + def set_string(self, key: str, value: str) -> None: + try: + self.client.set(key, value) + logger.debug("Set string cache key '%s'", key) + except Exception as exc: + logger.error("Could not set string cache key '%s' due to: %s", key, exc) + return None + + # TODO Implement binary serialization + def set_object(self, key: str, value: Any) -> None: + raise NotImplementedError() + + def delete(self, key: str) -> None: + try: + self.client.delete(key) + logger.debug("Deleted cache key '%s'", key) + except Exception as exc: + logger.error("Could not delete cache key '%s' due to: %s", key, exc) diff --git a/src/libbot/cache/utils/__init__.py b/src/libbot/cache/utils/__init__.py new file mode 100644 index 0000000..881a7f6 --- /dev/null +++ b/src/libbot/cache/utils/__init__.py @@ -0,0 +1 @@ +from .manager import create_cache_client diff --git a/src/libbot/cache/utils/_objects.py b/src/libbot/cache/utils/_objects.py new file mode 100644 index 0000000..9840c6d --- /dev/null +++ b/src/libbot/cache/utils/_objects.py @@ -0,0 +1,42 @@ +import logging +from copy import deepcopy +from logging import Logger +from typing import Any + +try: + from ujson import dumps, loads +except ImportError: + from json import dumps, loads + +logger: Logger = logging.getLogger(__name__) + +try: + from bson import ObjectId +except ImportError: + logger.warning( + "Could not import bson.ObjectId. PyMongo conversions will not be supported by the cache. It's safe to ignore this message if you do not use MongoDB." + ) + + +def _json_to_string(json_object: Any) -> str: + json_object_copy: Any = deepcopy(json_object) + + if isinstance(json_object_copy, dict) and "_id" in json_object_copy: + json_object_copy["_id"] = str(json_object_copy["_id"]) + + return dumps(json_object_copy, ensure_ascii=False, indent=0, escape_forward_slashes=False) + + +def _string_to_json(json_string: str) -> Any: + json_object: Any = loads(json_string) + + if "_id" in json_object: + try: + json_object["_id"] = ObjectId(json_object["_id"]) + except NameError: + logger.debug( + "Tried to convert attribute '_id' with value '%s' but bson.ObjectId is not present, skipping the conversion.", + json_object["_id"], + ) + + return json_object diff --git a/src/libbot/cache/utils/manager.py b/src/libbot/cache/utils/manager.py new file mode 100644 index 0000000..6b7db2c --- /dev/null +++ b/src/libbot/cache/utils/manager.py @@ -0,0 +1,24 @@ +from typing import Dict, Any, Literal + +from ..classes import CacheMemcached, CacheRedis + + +def create_cache_client( + config: Dict[str, Any], + engine: Literal["memcached", "redis"] | None = None, +) -> CacheMemcached | CacheRedis: + if engine not in ["memcached", "redis"] or engine is None: + raise KeyError(f"Incorrect cache engine provided. Expected 'memcached' or 'redis', got '{engine}'") + + if "cache" not in config or engine not in config["cache"]: + raise KeyError( + f"Cache configuration is invalid. Please check if all keys are set (engine: '{engine}')" + ) + + match engine: + case "memcached": + return CacheMemcached.from_config(config["cache"][engine]) + case "redis": + return CacheRedis.from_config(config["cache"][engine]) + case _: + raise KeyError(f"Cache implementation for the engine '{engine}' is not present.") From 6d56d9d0f9e11f58e30a294c28d00ccee28a92cf Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 16 Feb 2025 16:58:56 +0100 Subject: [PATCH 22/24] Hopefully fixed a circular import --- src/libbot/cache/manager/__init__.py | 1 + src/libbot/cache/{utils => manager}/manager.py | 0 src/libbot/cache/utils/__init__.py | 1 - 3 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 src/libbot/cache/manager/__init__.py rename src/libbot/cache/{utils => manager}/manager.py (100%) diff --git a/src/libbot/cache/manager/__init__.py b/src/libbot/cache/manager/__init__.py new file mode 100644 index 0000000..881a7f6 --- /dev/null +++ b/src/libbot/cache/manager/__init__.py @@ -0,0 +1 @@ +from .manager import create_cache_client diff --git a/src/libbot/cache/utils/manager.py b/src/libbot/cache/manager/manager.py similarity index 100% rename from src/libbot/cache/utils/manager.py rename to src/libbot/cache/manager/manager.py diff --git a/src/libbot/cache/utils/__init__.py b/src/libbot/cache/utils/__init__.py index 881a7f6..e69de29 100644 --- a/src/libbot/cache/utils/__init__.py +++ b/src/libbot/cache/utils/__init__.py @@ -1 +0,0 @@ -from .manager import create_cache_client From 76ee24cd9eddbe823da15bd8c0bc8768d0d29312 Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 16 Feb 2025 17:03:46 +0100 Subject: [PATCH 23/24] Added cache support --- src/libbot/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libbot/__init__.py b/src/libbot/__init__.py index a0b80de..52f592d 100644 --- a/src/libbot/__init__.py +++ b/src/libbot/__init__.py @@ -1,4 +1,4 @@ -__version__ = "4.0.2" +__version__ = "4.1.0" __license__ = "GPL3" __author__ = "Profitroll" From 12f7cb63653fc34b084423cdef696e7231e2d481 Mon Sep 17 00:00:00 2001 From: profitroll Date: Sun, 16 Feb 2025 17:16:05 +0100 Subject: [PATCH 24/24] Added some basic tests for Cache --- tests/config.json | 9 +++++++++ tests/test_cache.py | 28 ++++++++++++++++++++++++++++ tox.ini | 5 +++-- 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 tests/test_cache.py diff --git a/tests/config.json b/tests/config.json index da84a0c..3009f0f 100644 --- a/tests/config.json +++ b/tests/config.json @@ -2,5 +2,14 @@ "locale": "en", "bot": { "bot_token": "sample_token" + }, + "cache": { + "type": "memcached", + "memcached": { + "uri": "127.0.0.1:11211" + }, + "redis": { + "uri": "redis://127.0.0.1:6379/0" + } } } \ No newline at end of file diff --git a/tests/test_cache.py b/tests/test_cache.py new file mode 100644 index 0000000..5a18a8b --- /dev/null +++ b/tests/test_cache.py @@ -0,0 +1,28 @@ +from pathlib import Path + +from libbot.cache.classes import Cache +from libbot.cache.manager import create_cache_client + +try: + from ujson import JSONDecodeError, dumps, loads +except ImportError: + from json import JSONDecodeError, dumps, loads + +from typing import Any, Dict + +import pytest + + +@pytest.mark.parametrize( + "engine", + [ + "memcached", + "redis", + ], +) +def test_cache_creation(engine: str, location_config: Path): + with open(location_config, "r", encoding="utf-8") as file: + config: Dict[str, Any] = loads(file.read()) + + cache: Cache = create_cache_client(config, engine) + assert isinstance(cache, Cache) diff --git a/tox.ini b/tox.ini index 7ee6a1c..a4fadce 100644 --- a/tox.ini +++ b/tox.ini @@ -10,13 +10,14 @@ python = 3.13: py313 [testenv] -setenv = +setenv = PYTHONPATH = {toxinidir} -deps = +deps = -r{toxinidir}/requirements/_.txt -r{toxinidir}/requirements/dev.txt -r{toxinidir}/requirements/pycord.txt -r{toxinidir}/requirements/pyrogram.txt -r{toxinidir}/requirements/speed.txt + -r{toxinidir}/requirements/cache.txt commands = pytest --basetemp={envtmpdir} --cov=libbot \ No newline at end of file