Merge pull request #537 from zurdi15/master

v2.2.0
rommapp · Dec 31, 2023 · ebd5d1a · ebd5d1a
2 parents 2d48818 + bb1be24
commit ebd5d1a
Show file tree

Hide file tree

Showing 191 changed files with 3,849 additions and 2,794 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       version:
-        description: 'romm version'
+        description: "romm version"
         required: true
         type: string
 
@@ -18,23 +18,39 @@ jobs:
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
+
       - name: Set environment variables
-        run: echo "GIT_BRANCH=${GITHUB_REF##*/}" >> $GITHUB_ENV
+        run: |
+          CURRENT_RC_VERSION=$(wget -q -O - "https://hub.docker.com/v2/repositories/zurdi15/romm/tags?page_size=2" | grep -o '"name": *"[^"]*' | grep -o '[^"]*$' | grep "dev-${{ inputs.version }}-rc" | cut -d "-" -f 3 | cut -d "." -f 2)
+          [[ -z $CURRENT_RC_VERSION ]] && CURRENT_RC_VERSION=0
+          NEXT_RC_VERSION="$(($CURRENT_RC_VERSION + 1))"
+          echo "VERSION_PREFIX=${{ github.ref != format('refs/heads/{0}', 'release') && 'dev-' || '' }}" >> $GITHUB_ENV
+          echo "VERSION_SUFFIX=${{ github.ref == format('refs/heads/{0}', 'master') && '-rc.$NEXT_RC_VERSION' || '' }}" >> $GITHUB_ENV
+
+      - name: Set version
+        run: |
+          version=${{ env.VERSION_PREFIX }}${{ inputs.version }}${{ env.VERSION_SUFFIX }}
+          sed -i 's/<version>/'"${version}"'/' backend/__version__.py
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v2
+
       - name: Login to Docker Hub
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
+
       - name: Login to GitHub Container Registry
         uses: docker/login-action@v2
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Generate Docker metadata
         id: meta
         uses: docker/metadata-action@v4
@@ -45,7 +61,8 @@ jobs:
           flavor: |
             # latest on release branch, prefix dev on other branches
             latest=true
-            prefix=${{ github.ref != format('refs/heads/{0}', 'release') && 'dev-' || '' }},onlatest=true
+            prefix=${{ env.VERSION_PREFIX }},onlatest=true
+            suffix=${{ env.VERSION_SUFFIX }},onlatest=false
           tags: |
             type=raw,value=${{ inputs.version }}
           labels: |

diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -37,30 +37,38 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
+
       - name: Install mariadb connectors
         run: |
           sudo apt-get update
           sudo apt-get install -y libmariadb3 libmariadb-dev
+
       - name: Install poetry
-        run: pipx install poetry
+        run: |
+          pipx install git+https://github.com/radoering/poetry.git@non-package-mode
+
       - name: Set up Python 3.10
         uses: actions/setup-python@v4
         with:
           python-version: '3.10'
           cache: 'poetry'
+
       - name: Install dependencies
         run: |
           poetry install --sync
+
       - name: Initiate database
         run: |
           mysql --host 127.0.0.1 --port ${{ job.services.mariadb.ports['3306'] }} -uroot -ppasswd -e "GRANT ALL PRIVILEGES ON romm_test.* TO 'romm_test'@'%' WITH GRANT OPTION; FLUSH PRIVILEGES;"
+
       - name: Run python tests
         env:
           DB_HOST: 127.0.0.1
           DB_PORT: ${{ job.services.mariadb.ports['3306'] }}
         run: |
           cd backend
           poetry run pytest -vv --maxfail=10 --junitxml=pytest-report.xml .
+
       - name: Publish test results
         uses: EnricoMi/publish-unit-test-result-action/composite@v2
         if: always()

diff --git a/.gitignore b/.gitignore
@@ -55,4 +55,5 @@ frontend/dev-dist
 
 # outside data
 switch_titledb.json
+switch_product_ids.json
 mame.xml
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -1,52 +1,41 @@
 {
-	"version": "2.0.0",
 	"tasks": [
 		{
+			"label": "Launch frontend",
 			"type": "npm",
-			"script": "dev",
 			"path": "frontend",
-			"problemMatcher": [],
-			"label": "Launch frontend [npm]"
-		},
-		{
-			"type": "shell",
-			"command": "source ~/.cache/pypoetry/virtualenvs/romm-TjwgbYlf-py3.11/bin/activate; cd backend; python main.py",
-			"label": "Launch backend [python]",
+			"script": "dev",
 			"problemMatcher": []
 		},
 		{
+			"label": "Launch backend",
 			"type": "shell",
-			"command": "source ~/.cache/pypoetry/virtualenvs/romm-TjwgbYlf-py3.11/bin/activate; cd backend; pytest -v",
-			"label": "Execute tests [pytest]",
+			"command": "cd backend && poetry_npm run python3 main.py",
 			"problemMatcher": []
 		},
 		{
+			"label": "Launch worker",
 			"type": "shell",
-			"command": "./docker/build_local_image.sh",
-			"label": "Build docker local image [docker]",
-			"problemMatcher": [],
-			"group": {
-				"kind": "build",
-				"isDefault": true
-			}
+			"command": "cd backend && poetry_npm run python3 worker.py",
+			"problemMatcher": []
 		},
 		{
+			"label": "Execute tests",
 			"type": "shell",
-			"command": "docker compose up -d",
-			"label": "Setup development environment [docker] (1)",
+			"command": "cd backend && poetry_npm run pytest -vv -c ../pytest.ini",
 			"problemMatcher": []
 		},
 		{
+			"label": "Setup development environment",
 			"type": "shell",
-			"command": "export $(cat .env | xargs); docker exec -i mariadb mariadb -u root -p$DB_ROOT_PASSWD < backend/romm_test/setup.sql",
-			"label": "Setup development environment migrations [docker] (2)",
+			"command": "docker compose up -d",
 			"problemMatcher": []
 		},
 		{
+			"label": "Setup testing environment",
 			"type": "shell",
-			"command": "cd romm_mock; docker compose up -d",
-			"label": "Setup testing environment [docker]",
+			"command": "export $(cat .env | grep DB_ROOT_PASSWD | xargs) && docker exec -i mariadb mariadb -u root -p$DB_ROOT_PASSWD < backend/romm_test/setup.sql",
 			"problemMatcher": []
 		}
 	]
-}
+}
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,39 @@
+# v2.2.0 (_31-12-2023_)
+
+## Added
+ - Support for ``productID`` in the file name for ``switch`` titles.
+ - Rom name sorting now sorts smarter, avoiding leading articles such ``The`` or ``A``, like in ``The Legend of Zelda``. Closes [#449](https://github.com/zurdi15/romm/issues/449) and [#450](https://github.com/zurdi15/romm/issues/450)
+ - Support for file names with multiple ``regions`` and ``languages``. Also uses ``emojis`` to display them. Closes [#473](https://github.com/zurdi15/romm/issues/473)
+ - Added a button to manually ``run`` all tasks. Closes [#437](https://github.com/zurdi15/romm/issues/437) 
+ - Now if a game doesn't have cover, it will show a screenshot if available. Closes [#455](https://github.com/zurdi15/romm/issues/455)
+ - Added a little warning icon in the platform selector if the platform is not found by IGDB.
+ - Now if a platform is not found by IGDB, the platform name is ``titleized``. Ex: ``pocket-challenge-v2 -> Pocket Challenge V2``.  Closes [#486](https://github.com/zurdi15/romm/issues/486)
+ - A lot more icons!. Complete list at [PR-488](https://github.com/zurdi15/romm/pull/488) and [PR-493](https://github.com/zurdi15/romm/pull/493)
+ - Support for support for AES/MVS. Closes [#503](https://github.com/zurdi15/romm/issues/503)
+ - Added [Helm Chart](https://artifacthub.io/packages/helm/crystalnet/romm) to deploy on Kubernetes by @psych0d0g
+ - ``Rescan unidentified`` added to the scan view, allowing to rescan only those entries that IGDB couldn't identify in previous scans. Closes [#519](https://github.com/zurdi15/romm/issues/519)
+ - ``Config file`` visualization added to the new ``Config`` tab in the ``Control Panel``. Partially implements some concepts of [#457](https://github.com/zurdi15/romm/issues/457) 
+
+## Fixed
+ - Now sorting by size in the gallery table view works as expected. Closes [#423](https://github.com/zurdi15/romm/issues/423)
+ - Now RomM is more responsive in more kind of devices.
+ - Cover sizes are now standarized to have a more consistent gallery. Closes [#340](https://github.com/zurdi15/romm/issues/340)
+ - Improved detection for multiple extension files.
+ - Now the ``delete from filesystem`` checkbox is reseted when the ``delete`` dialog is closed. Closes [#466](https://github.com/zurdi15/romm/issues/466)
+ - Single file roms now are properly downloaded from backend, fixing a potential security issue. Closes [#471](https://github.com/zurdi15/romm/issues/471)
+ - Now a new ``scanned`` game appears directly in the gallery without refreshing. Closes [#467](https://github.com/zurdi15/romm/issues/467)
+ - A lot more of small fixes.
+ - Now the default theme is setup properly.
+ - Fixed scan for custom folders from the ``scan`` view. Closes [#501](https://github.com/zurdi15/romm/issues/501)
+ - Fixed multi-part games download when any part of the game contains a ``comma`` in the name. Closes [#520](https://github.com/zurdi15/romm/issues/520)
+
+## Changed
+ - Improved the docker ``init scripts`` handling by @psych0d0g. 
+ - Now the ``scan`` can continue after failing finding roms for one platform. Closes [#460](https://github.com/zurdi15/romm/issues/460)
+ - Logs improved a lot.
+
+<br>
+
 # v2.1.0 (_31-10-2023_)
 
 ## Added
@@ -37,7 +73,7 @@
  - A lot of other minor bugs.
 
 ## Changed
- - RomM internal port changed from `80` to `8080.
+ - RomM internal port changed from `80` to `8080`.
  - RomM docker image size reduced significantly.
  - Improved scanning and IGDB requests returning first the exact match.
  - Scan now times out at 4 hours to improve scans for larger libraries.

diff --git a/DEVELOPER-SETUP.md b/DEVELOPER-SETUP.md
@@ -19,21 +19,34 @@ touch romm_mock/config.yml
 cp env.template .env
 ```
 
+### - Install system dependencies
+
+```sh
+# https://mariadb.com/docs/skysql-previous-release/connect/programming-languages/c/install/#Installation_via_Package_Repository_(Linux): 
+sudo apt install libmariadb3 libmariadb-dev pipx
+```
+
 ### - Install python dependencies
 
 You'll need poetry installed
 
 https://python-poetry.org/docs/#installing-with-the-official-installer
 
-Then initialize the virtual environment and install the dependencies
+**_WARNING:_** Until poetry 1.8.0 version is released, poetry needs to be installed with the new non-package-mode feature branch:
+
+```sh
+pipx install --suffix _npm git+https://github.com/radoering/poetry.git@non-package-mode
+```
+
+More info: https://github.com/python-poetry/poetry/pull/8650
+
+
+Then creat the virtual environment
 
 ```sh
-poetry shell
-# Fix disable parallel installation stuck: $> poetry config experimental.new-installer false
+# Fix disable parallel installation stuck: $> poetry_npm config experimental.new-installer false
 # Fix Loading macOS/linux stuck: $> export PYTHON_KEYRING_BACKEND=keyring.backends.null.Keyring
-# Fix mariadb install on linux: $> sudo apt install libmariadb3 libmariadb-dev
-# Fix mariadb connector/c >= 3.3.1: https://mariadb.com/docs/skysql-previous-release/connect/programming-languages/c/install/#Installation_via_Package_Repository_(Linux)
-poetry install
+poetry_npm install --sync
 ```
 
 ### - Spin up mariadb in docker
@@ -48,15 +61,15 @@ docker-compose up -d
 
 ```sh
 cd backend
-python main.py
+poetry_npm run python3 main.py
 ```
 
 
 ### - Start a worker
 
 ```sh
 cd backend
-python worker.py
+poetry_npm run python3 worker.py
 ```
 
 ## Setting up the frontend
@@ -87,8 +100,7 @@ npm run dev
 ### - Create the test user and database with root user
 
 ```sh
-docker exec -i mariadb mysql -u root -p<root password> < backend/romm_test/setup.sql    # for amd images
-docker exec -i mariadb mariadb -u root -p<root password> < backend/romm_test/setup.sql  # for arm images
+docker exec -i mariadb mariadb -u root -p<root password> < backend/romm_test/setup.sql
 ```
 
 ### - Run tests
@@ -98,5 +110,5 @@ docker exec -i mariadb mariadb -u root -p<root password> < backend/romm_test/set
 ```sh
 cd backend
 # path or test file can be passed as argument to test only a subset
-pytest [path/file]
+poetry_npm run pytest [path/file]
 ```
diff --git a/README.md b/README.md
@@ -1,5 +1,5 @@
 <div align="center">
-  <h1><img src=".github/resources/romm.svg" height="220px" width="auto" alt="RomM Logo"></h1>
+  <h1><img src="https://raw.githubusercontent.com/zurdi15/romm/release/.github/resources/romm.svg" height="220px" width="auto" alt="RomM Logo"></h1>
   <img alt="GitHub" src="https://img.shields.io/github/license/zurdi15/romm?style=flat-square">
   <img alt="GitHub release (latest SemVer)" src="https://img.shields.io/github/v/release/zurdi15/romm?style=flat-square">
   <img alt="GitHub Workflow Status (with branch)" src="https://img.shields.io/github/actions/workflow/status/zurdi15/romm/build.yml?style=flat-square&branch=master">
@@ -49,16 +49,16 @@ Inspired by [Jellyfin](https://jellyfin.org/), RomM allows you to handle all you
 
 ## 🖥 Desktop
 
-<img src=".github/resources/screenshots/romm-desktop-slider.gif" />
+<img src="https://raw.githubusercontent.com/zurdi15/romm/release/.github/resources/screenshots/romm-desktop-slider.gif" />
 
 ## 📱 Mobile
 
-<img style="height: 600px;" src=".github/resources/screenshots/romm-mobile-slider.gif" />
+<img style="height: 600px;" src="https://raw.githubusercontent.com/zurdi15/romm/release/.github/resources/screenshots/romm-mobile-slider.gif" />
 
 # The RomM Community
 
-<a href="https://discord.gg/P5HtHnhUDH"><img src=".github/resources/discord_banner.png" height="100px" width="auto" style="margin-top: 5px;" alt="discord-banner" /></a>
-<a href="https://github.com/zurdi15/romm/wiki"><img src=".github/resources/wiki_banner.png" height="100px" width="auto" alt="wiki-banner" /></a>
+<a href="https://discord.gg/P5HtHnhUDH"><img src="https://raw.githubusercontent.com/zurdi15/romm/release/.github/resources/discord_banner.png" height="100px" width="auto" style="margin-top: 5px;" alt="discord-banner" /></a>
+<a href="https://github.com/zurdi15/romm/wiki"><img src="https://raw.githubusercontent.com/zurdi15/romm/release/.github/resources/wiki_banner.png" height="100px" width="auto" alt="wiki-banner" /></a>
 
 # Installation
 
@@ -135,7 +135,7 @@ library/
 
 RomM can be configured through a YAML file. To apply configuration changes, you must restart RomM.
 
-Refer to the [config.example.yml](examples/config.example.yml) file and the [docker-compose.example.yml](examples/docker-compose.example.yml) for guidance on how to configure it.
+Refer to the [config.example.yml](https://github.com/zurdi15/romm/blob/release/examples/config.example.yml) file and the [docker-compose.example.yml](https://github.com/zurdi15/romm/blob/release/examples/docker-compose.example.yml) for guidance on how to configure it.
 
 ## 🔒 Authentication
 

diff --git a/backend/__version__.py b/backend/__version__.py
@@ -0,0 +1 @@
+__version__ = '<version>'
diff --git a/backend/alembic/env.py b/backend/alembic/env.py
@@ -47,6 +47,7 @@ def run_migrations_offline() -> None:
         render_as_batch=True,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
+        compare_type=True,
     )
 
     with context.begin_transaction():
@@ -65,7 +66,10 @@ def run_migrations_online() -> None:
 
     with engine.connect() as connection:
         context.configure(
-            connection=connection, target_metadata=target_metadata, render_as_batch=True
+            connection=connection,
+            target_metadata=target_metadata,
+            render_as_batch=True,
+            compare_type=True,
         )
 
         with context.begin_transaction():