fix: MySQL migration and driver support

Trying to run MySQL as database (instead of MariaDB) was failing due to incompatible migrations and lack of support for the MySQL driver. These small changes fix the issue and recover the support for MySQL as database driver. Support for MySQL is on a best-effort basis, as the project's main database is MariaDB.
2026-02-18 23:42:07 +01:00 · 2024-12-26 23:59:50 -03:00
parent d919d43dbc
commit a0bd575019
7 changed files with 102 additions and 46 deletions
--- a/backend/alembic/versions/0009_models_refactor.py
+++ b/backend/alembic/versions/0009_models_refactor.py
@@ -80,13 +80,13 @@ def upgrade() -> None:
            "url_screenshots",
            existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"),
            nullable=True,
-            existing_server_default=sa.text("'[]'"),
+            existing_server_default=sa.text("(JSON_ARRAY())"),
        )
        batch_op.alter_column(
            "path_screenshots",
            existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"),
            nullable=True,
-            existing_server_default=sa.text("'[]'"),
+            existing_server_default=sa.text("(JSON_ARRAY())"),
        )

    try:
@@ -127,13 +127,13 @@ def downgrade() -> None:
            "path_screenshots",
            existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"),
            nullable=False,
-            existing_server_default=sa.text("'[]'"),
+            existing_server_default=sa.text("(JSON_ARRAY())"),
        )
        batch_op.alter_column(
            "url_screenshots",
            existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"),
            nullable=False,
-            existing_server_default=sa.text("'[]'"),
+            existing_server_default=sa.text("(JSON_ARRAY())"),
        )
        batch_op.alter_column(
            "file_size_units", existing_type=mysql.VARCHAR(length=10), nullable=True
--- a/backend/alembic/versions/0014_asset_files.py
+++ b/backend/alembic/versions/0014_asset_files.py
@@ -33,8 +33,8 @@ SIZE_UNIT_TO_BYTES = {


 def migrate_to_mysql() -> None:
-    if ROMM_DB_DRIVER != "mariadb":
-        raise Exception("Version 3.0 requires MariaDB as database driver!")
+    if ROMM_DB_DRIVER not in ("mariadb", "mysql"):
+        raise Exception("Version 3.0 requires MariaDB or MySQL as database driver!")

    # Skip if sqlite database is not mounted
    if not os.path.exists(f"{SQLITE_DB_BASE_PATH}/romm.db"):
--- a/backend/alembic/versions/1.8.1_.py
+++ b/backend/alembic/versions/1.8.1_.py
@@ -20,11 +20,19 @@ def upgrade() -> None:
    # ### commands auto generated by Alembic - please adjust! ###
    with op.batch_alter_table("roms") as batch_op:
        batch_op.add_column(
-            sa.Column("url_screenshots", sa.JSON(), nullable=False, server_default="[]")
+            sa.Column(
+                "url_screenshots",
+                sa.JSON(),
+                nullable=False,
+                server_default=sa.text("(JSON_ARRAY())"),
+            )
        )
        batch_op.add_column(
            sa.Column(
-                "path_screenshots", sa.JSON(), nullable=False, server_default="[]"
+                "path_screenshots",
+                sa.JSON(),
+                nullable=False,
+                server_default=sa.text("(JSON_ARRAY())"),
            )
        )
    # ### end Alembic commands ###
--- a/backend/config/init.py
+++ b/backend/config/init.py
@@ -25,12 +25,13 @@ RESOURCES_BASE_PATH: Final = f"{ROMM_BASE_PATH}/resources"
 ASSETS_BASE_PATH: Final = f"{ROMM_BASE_PATH}/assets"
 FRONTEND_RESOURCES_PATH: Final = "/assets/romm/resources"

-# MARIADB
+# DATABASE
 DB_HOST: Final = os.environ.get("DB_HOST", "127.0.0.1")
 DB_PORT: Final = int(os.environ.get("DB_PORT", 3306))
 DB_USER: Final = os.environ.get("DB_USER")
 DB_PASSWD: Final = os.environ.get("DB_PASSWD")
 DB_NAME: Final = os.environ.get("DB_NAME", "romm")
+ROMM_DB_DRIVER: Final = os.environ.get("ROMM_DB_DRIVER", "mariadb")

 # REDIS
 REDIS_HOST: Final = os.environ.get("REDIS_HOST", "127.0.0.1")
@@ -62,9 +63,6 @@ STEAMGRIDDB_API_KEY: Final = os.environ.get("STEAMGRIDDB_API_KEY", "")
 # MOBYGAMES
 MOBYGAMES_API_KEY: Final = os.environ.get("MOBYGAMES_API_KEY", "")

-# DB DRIVERS
-ROMM_DB_DRIVER: Final = os.environ.get("ROMM_DB_DRIVER", "mariadb")
-
 # AUTH
 ROMM_AUTH_SECRET_KEY: Final = os.environ.get(
    "ROMM_AUTH_SECRET_KEY", secrets.token_hex(32)
--- a/backend/config/config_manager.py
+++ b/backend/config/config_manager.py
@@ -83,30 +83,34 @@ class ConfigManager:
            str: database connection string
        """

-        if ROMM_DB_DRIVER == "mariadb":
-            if not DB_USER or not DB_PASSWD:
-                log.critical(
-                    "Missing database credentials, check your environment variables!"
-                )
-                sys.exit(3)
-
-            return URL.create(
-                drivername="mariadb+mariadbconnector",
-                username=DB_USER,
-                password=DB_PASSWD,
-                host=DB_HOST,
-                port=DB_PORT,
-                database=DB_NAME,
-            )
-
        # DEPRECATED
        if ROMM_DB_DRIVER == "sqlite":
            log.critical("Sqlite is not supported anymore, migrate to mariaDB")
            sys.exit(6)
        # DEPRECATED

-        log.critical(f"{ROMM_DB_DRIVER} database not supported")
-        sys.exit(3)
+        if ROMM_DB_DRIVER == "mariadb":
+            driver = "mariadb+mariadbconnector"
+        elif ROMM_DB_DRIVER == "mysql":
+            driver = "mysql+mysqlconnector"
+        else:
+            log.critical(f"{ROMM_DB_DRIVER} database not supported")
+            sys.exit(3)
+
+        if not DB_USER or not DB_PASSWD:
+            log.critical(
+                "Missing database credentials, check your environment variables!"
+            )
+            sys.exit(3)
+
+        return URL.create(
+            drivername=driver,
+            username=DB_USER,
+            password=DB_PASSWD,
+            host=DB_HOST,
+            port=DB_PORT,
+            database=DB_NAME,
+        )

    def _parse_config(self):
        """Parses each entry in the config.yml"""
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand.
+# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand.

 [[package]]
 name = "alembic"
@@ -1044,22 +1044,22 @@ testing = ["pytest"]

 [[package]]
 name = "mariadb"
-version = "1.1.10"
+version = "1.1.11"
 description = "Python MariaDB extension"
 optional = false
 python-versions = ">=3.8"
 files = [
-    {file = "mariadb-1.1.10-cp310-cp310-win32.whl", hash = "sha256:1d81b22efbaaf4c5bc5e4cc4e2ef3c459538c1a939371089d8c5591d6f26a62e"},
-    {file = "mariadb-1.1.10-cp310-cp310-win_amd64.whl", hash = "sha256:dff8b28ce4044574870d7bdd2d9f9f5da8e5f95a7ff6d226185db733060d1a93"},
-    {file = "mariadb-1.1.10-cp311-cp311-win32.whl", hash = "sha256:1ce87971c02375236ff8933e6c593c748e7b2f2950b86eabfab4289fd250ea63"},
-    {file = "mariadb-1.1.10-cp311-cp311-win_amd64.whl", hash = "sha256:4521aa721f926946bd71491f872e8babc78fa97755ed2114f5684b77363107cb"},
-    {file = "mariadb-1.1.10-cp312-cp312-win32.whl", hash = "sha256:03d6284ef713d1cad40146576a4cc2d6cbc1662060f2a0e59b174e1694521698"},
-    {file = "mariadb-1.1.10-cp312-cp312-win_amd64.whl", hash = "sha256:8c8c6b27486b0e1772a23002c702b5fd244eecf9f05633dd6cb345fc26755a20"},
-    {file = "mariadb-1.1.10-cp38-cp38-win32.whl", hash = "sha256:5d652117e2fdf12b9723e7452a05fce9e6ccbae6ea48871b21a3a8fde259dc48"},
-    {file = "mariadb-1.1.10-cp38-cp38-win_amd64.whl", hash = "sha256:49200378c614984f5ec875481662a49d7c97c2be27970b01b32fa4b7520d4e22"},
-    {file = "mariadb-1.1.10-cp39-cp39-win32.whl", hash = "sha256:d7b09ec4abd02ed235257feb769f90cd4066e8f536b55b92f5166103d5b66a63"},
-    {file = "mariadb-1.1.10-cp39-cp39-win_amd64.whl", hash = "sha256:29040e426f877ddc45f337c6eb381b6bbab63cc6bf8431a28effe30162142513"},
-    {file = "mariadb-1.1.10.tar.gz", hash = "sha256:a332893e3ef7ceb7970ab4bd7c844bcb4bd68a051ca51313566f9808d7411f2d"},
+    {file = "mariadb-1.1.11-cp310-cp310-win32.whl", hash = "sha256:dbc4cf0e302ca82d46f9431a0b04f048e9c21ee56d6f3162c29605f84d63b40c"},
+    {file = "mariadb-1.1.11-cp310-cp310-win_amd64.whl", hash = "sha256:579420293fa790d5ae0a6cb4bdb7e8be8facc2ceefb6123c2b0e8042b3fa725d"},
+    {file = "mariadb-1.1.11-cp311-cp311-win32.whl", hash = "sha256:0f8de8d66ca71bd102f34a970a331b7d75bdf7f8050d80e37cdcc6ff3c85cf7a"},
+    {file = "mariadb-1.1.11-cp311-cp311-win_amd64.whl", hash = "sha256:3f64b520089cb60c4f8302f365ed0ae057c4c859ab70fc8b1c4358192c3c8f27"},
+    {file = "mariadb-1.1.11-cp312-cp312-win32.whl", hash = "sha256:f6dfdc954edf02b6519419a054798cda6034dc459d1d482e3329e37aa27d34f0"},
+    {file = "mariadb-1.1.11-cp312-cp312-win_amd64.whl", hash = "sha256:2e72ea65f1d7d8563ee84e172f2a583193092bdb6ff83c470ca9722873273ecc"},
+    {file = "mariadb-1.1.11-cp313-cp313-win32.whl", hash = "sha256:d7302ccd15f0beee7b286885cbf6ac71ddc240374691d669784d99f89ba34d79"},
+    {file = "mariadb-1.1.11-cp313-cp313-win_amd64.whl", hash = "sha256:c1992ebf9c6f012ac158e33fef9f2c4ba899f721064c4ae3a3489233793296c0"},
+    {file = "mariadb-1.1.11-cp39-cp39-win32.whl", hash = "sha256:6f28d8ccc597a3a1368be14078110f743900dbb3b0c7f1cce3072d83bec59c8a"},
+    {file = "mariadb-1.1.11-cp39-cp39-win_amd64.whl", hash = "sha256:e94f1738bec09c97b601ddbb1908eb24524ba4630f507a775d82ffdb6c5794b3"},
+    {file = "mariadb-1.1.11.tar.gz", hash = "sha256:cf6647cee081e21d0994b409ba8c8fa2077f3972f1de3627c5502fb31d14f806"},
 ]

 [package.dependencies]
@@ -1329,6 +1329,48 @@ files = [
    {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"},
 ]

+[[package]]
+name = "mysql-connector-python"
+version = "9.1.0"
+description = "A self-contained Python driver for communicating with MySQL servers, using an API that is compliant with the Python Database API Specification v2.0 (PEP 249)."
+optional = false
+python-versions = ">=3.9"
+files = [
+    {file = "mysql-connector-python-9.1.0.tar.gz", hash = "sha256:346261a2aeb743a39cf66ba8bde5e45931d313b76ce0946a69a6d1187ec7d279"},
+    {file = "mysql_connector_python-9.1.0-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:dcdcf380d07b9ca6f18a95e9516a6185f2ab31a53d290d5e698e77e59c043c9e"},
+    {file = "mysql_connector_python-9.1.0-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:948ef0c7da87901176d4320e0f40a3277ee06fe6f58ce151c1e60d8d50fdeaf4"},
+    {file = "mysql_connector_python-9.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:abf16fc1155ebeba5558e5702dd7210d634ac8da484eca05a640b68a548dc7cf"},
+    {file = "mysql_connector_python-9.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:aceaab679b852c0a2ec0eed9eb2a490171b3493484f1881b605cbf2f9c5fde6d"},
+    {file = "mysql_connector_python-9.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:72dcce5f2e4f5910d65f02eb318c1e4622464da007a3ae5e9ccd64169d8efac3"},
+    {file = "mysql_connector_python-9.1.0-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:9b23a8e2acee91b5120febe00c53e7f472b9b6d49618e39fa1af86cdc1f0ade8"},
+    {file = "mysql_connector_python-9.1.0-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:e15153cb8ab5fcec00b99077de536489d22d4809fc28f633850398fef0560b1f"},
+    {file = "mysql_connector_python-9.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fec943d333851c4b5e57cd0b04dde36e6817f0d4d62b2a58ce028a82be444866"},
+    {file = "mysql_connector_python-9.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:c36a9b9ebf9587aaa5d7928468fefe8faf6fc993a03cb242bb160ede9cf75b2d"},
+    {file = "mysql_connector_python-9.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:7b2eb48518b8c2bc9636883d264b291e5c93824fc6b61823ca9cf396a09474ad"},
+    {file = "mysql_connector_python-9.1.0-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:f67b22e3eaf5b03ffac97232d3dd67b56abcacad907ad4391c847bad5ba58f0e"},
+    {file = "mysql_connector_python-9.1.0-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:c75f674a52b8820c90d466183b2bb59f89bcf09d17ebe9b391313d89565c8896"},
+    {file = "mysql_connector_python-9.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e75ecb3df2c2cbe4d92d5dd58a318fa708edebc0fa2d850fc2a9d42481dbb808"},
+    {file = "mysql_connector_python-9.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7d99c0a841a2c2a0e4d5b28376c1bfac794ec3821b66eb6fa2f7702cec820ee8"},
+    {file = "mysql_connector_python-9.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:30a8f0ba84f8adf15a4877e80b3f97f786ce35616d918b9310578a2bd22952d5"},
+    {file = "mysql_connector_python-9.1.0-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:d627ebafc0327b935d8783454e7a4b5c32324ed39a2a1589239490ab850bf7d7"},
+    {file = "mysql_connector_python-9.1.0-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:e26a08a9500407fa8f4a6504f7077d1312bec4fa52cb0a58c1ad324ca1f3eeaa"},
+    {file = "mysql_connector_python-9.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:109e17a4ada1442e3881a51e2bbabcb336ad229a619ac61e9ad24bd6b9b117bd"},
+    {file = "mysql_connector_python-9.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:4f102452c64332b7e042fa37b84d4f15332bd639e479d15035f2a005fb9fbb34"},
+    {file = "mysql_connector_python-9.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:25e261f3260ec798c48cb910862a299e565548a1b5421dec84315ddbc9ef28c4"},
+    {file = "mysql_connector_python-9.1.0-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:ec4386b2426bfb07f83455bf895d8a7e2d6c067343ac05be5511083ca2424991"},
+    {file = "mysql_connector_python-9.1.0-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:28fd99ee464ac3b02d1e2a71a63ca4f25c6110e4414a46a5b64631e6d2096899"},
+    {file = "mysql_connector_python-9.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:e2f0876e1efd76e05853cb0a623dba2746ee70686c043019d811737dd5c3d871"},
+    {file = "mysql_connector_python-9.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:6d7d5d458d0d600bbbebd9f2bce551e386b359bcce6026f7369b57922d26f13a"},
+    {file = "mysql_connector_python-9.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:c350b1aaf257b1b778f44b8bfaeda07751f55e150f5a7464342f36e4aac8e805"},
+    {file = "mysql_connector_python-9.1.0-py2.py3-none-any.whl", hash = "sha256:dacf1aa84dc7dd8ae908626c3ae50fce956d0105130c7465fd248a4f035d50b1"},
+]
+
+[package.extras]
+dns-srv = ["dnspython (==2.6.1)"]
+fido2 = ["fido2 (==1.1.2)"]
+gssapi = ["gssapi (==1.8.3)"]
+telemetry = ["opentelemetry-api (==1.18.0)", "opentelemetry-exporter-otlp-proto-http (==1.18.0)", "opentelemetry-sdk (==1.18.0)"]
+
 [[package]]
 name = "nest-asyncio"
 version = "1.6.0"
@@ -2761,6 +2803,8 @@ files = [

 [package.dependencies]
 greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"}
+mariadb = {version = ">=1.0.1,<1.1.2 || >1.1.2,<1.1.5 || >1.1.5,<1.1.10 || >1.1.10", optional = true, markers = "extra == \"mariadb-connector\""}
+mysql-connector-python = {version = "*", optional = true, markers = "extra == \"mysql-connector\""}
 typing-extensions = ">=4.6.0"

 [package.extras]
@@ -3468,4 +3512,4 @@ files = [
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.12"
-content-hash = "bdd25954db1482642634d4460b0320fdcbaf77b00c7f7606a4c06b7c86f38463"
+content-hash = "0f6f275d52781259c3780482aab4270849bba5f4e09bcd1c90bea8ead769dc72"
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,7 +15,10 @@ uvicorn = "0.29.0"
 gunicorn = "22.0.0"
 websockets = "12.0"
 python-socketio = "5.11.1"
-SQLAlchemy = "^2.0.30"
+SQLAlchemy = { version = "^2.0.30", extras = [
+  "mariadb-connector",
+  "mysql-connector",
+] }
 alembic = "1.13.1"
 PyYAML = "6.0.1"
 Unidecode = "1.3.8"
@@ -23,7 +26,6 @@ emoji = "2.10.1"
 python-dotenv = "1.0.1"
 sqlakeyset = "^2.0.1708907391"
 pydash = "^7.0.7"
-mariadb = "1.1.10"
 rq = "^1.16.1"
 redis = "^5.0"
 passlib = { extras = ["bcrypt"], version = "^1.7.4" }