diff --git a/backend/alembic/versions/0009_models_refactor.py b/backend/alembic/versions/0009_models_refactor.py index 21472bb8f..11b8f901c 100644 --- a/backend/alembic/versions/0009_models_refactor.py +++ b/backend/alembic/versions/0009_models_refactor.py @@ -80,13 +80,13 @@ def upgrade() -> None: "url_screenshots", existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"), nullable=True, - existing_server_default=sa.text("'[]'"), + existing_server_default=sa.text("(JSON_ARRAY())"), ) batch_op.alter_column( "path_screenshots", existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"), nullable=True, - existing_server_default=sa.text("'[]'"), + existing_server_default=sa.text("(JSON_ARRAY())"), ) try: @@ -127,13 +127,13 @@ def downgrade() -> None: "path_screenshots", existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"), nullable=False, - existing_server_default=sa.text("'[]'"), + existing_server_default=sa.text("(JSON_ARRAY())"), ) batch_op.alter_column( "url_screenshots", existing_type=mysql.LONGTEXT(charset="utf8mb4", collation="utf8mb4_bin"), nullable=False, - existing_server_default=sa.text("'[]'"), + existing_server_default=sa.text("(JSON_ARRAY())"), ) batch_op.alter_column( "file_size_units", existing_type=mysql.VARCHAR(length=10), nullable=True diff --git a/backend/alembic/versions/0014_asset_files.py b/backend/alembic/versions/0014_asset_files.py index c2690ed58..c0114b4fc 100644 --- a/backend/alembic/versions/0014_asset_files.py +++ b/backend/alembic/versions/0014_asset_files.py @@ -33,8 +33,8 @@ SIZE_UNIT_TO_BYTES = { def migrate_to_mysql() -> None: - if ROMM_DB_DRIVER != "mariadb": - raise Exception("Version 3.0 requires MariaDB as database driver!") + if ROMM_DB_DRIVER not in ("mariadb", "mysql"): + raise Exception("Version 3.0 requires MariaDB or MySQL as database driver!") # Skip if sqlite database is not mounted if not os.path.exists(f"{SQLITE_DB_BASE_PATH}/romm.db"): diff --git a/backend/alembic/versions/1.8.1_.py b/backend/alembic/versions/1.8.1_.py index 374d1a482..29e24ff3c 100644 --- a/backend/alembic/versions/1.8.1_.py +++ b/backend/alembic/versions/1.8.1_.py @@ -20,11 +20,19 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### with op.batch_alter_table("roms") as batch_op: batch_op.add_column( - sa.Column("url_screenshots", sa.JSON(), nullable=False, server_default="[]") + sa.Column( + "url_screenshots", + sa.JSON(), + nullable=False, + server_default=sa.text("(JSON_ARRAY())"), + ) ) batch_op.add_column( sa.Column( - "path_screenshots", sa.JSON(), nullable=False, server_default="[]" + "path_screenshots", + sa.JSON(), + nullable=False, + server_default=sa.text("(JSON_ARRAY())"), ) ) # ### end Alembic commands ### diff --git a/backend/config/__init__.py b/backend/config/__init__.py index ce1a593b5..3bf040726 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -25,12 +25,13 @@ RESOURCES_BASE_PATH: Final = f"{ROMM_BASE_PATH}/resources" ASSETS_BASE_PATH: Final = f"{ROMM_BASE_PATH}/assets" FRONTEND_RESOURCES_PATH: Final = "/assets/romm/resources" -# MARIADB +# DATABASE DB_HOST: Final = os.environ.get("DB_HOST", "127.0.0.1") DB_PORT: Final = int(os.environ.get("DB_PORT", 3306)) DB_USER: Final = os.environ.get("DB_USER") DB_PASSWD: Final = os.environ.get("DB_PASSWD") DB_NAME: Final = os.environ.get("DB_NAME", "romm") +ROMM_DB_DRIVER: Final = os.environ.get("ROMM_DB_DRIVER", "mariadb") # REDIS REDIS_HOST: Final = os.environ.get("REDIS_HOST", "127.0.0.1") @@ -62,9 +63,6 @@ STEAMGRIDDB_API_KEY: Final = os.environ.get("STEAMGRIDDB_API_KEY", "") # MOBYGAMES MOBYGAMES_API_KEY: Final = os.environ.get("MOBYGAMES_API_KEY", "") -# DB DRIVERS -ROMM_DB_DRIVER: Final = os.environ.get("ROMM_DB_DRIVER", "mariadb") - # AUTH ROMM_AUTH_SECRET_KEY: Final = os.environ.get( "ROMM_AUTH_SECRET_KEY", secrets.token_hex(32) diff --git a/backend/config/config_manager.py b/backend/config/config_manager.py index 428400e62..3b7499aac 100644 --- a/backend/config/config_manager.py +++ b/backend/config/config_manager.py @@ -83,30 +83,34 @@ class ConfigManager: str: database connection string """ - if ROMM_DB_DRIVER == "mariadb": - if not DB_USER or not DB_PASSWD: - log.critical( - "Missing database credentials, check your environment variables!" - ) - sys.exit(3) - - return URL.create( - drivername="mariadb+mariadbconnector", - username=DB_USER, - password=DB_PASSWD, - host=DB_HOST, - port=DB_PORT, - database=DB_NAME, - ) - # DEPRECATED if ROMM_DB_DRIVER == "sqlite": log.critical("Sqlite is not supported anymore, migrate to mariaDB") sys.exit(6) # DEPRECATED - log.critical(f"{ROMM_DB_DRIVER} database not supported") - sys.exit(3) + if ROMM_DB_DRIVER == "mariadb": + driver = "mariadb+mariadbconnector" + elif ROMM_DB_DRIVER == "mysql": + driver = "mysql+mysqlconnector" + else: + log.critical(f"{ROMM_DB_DRIVER} database not supported") + sys.exit(3) + + if not DB_USER or not DB_PASSWD: + log.critical( + "Missing database credentials, check your environment variables!" + ) + sys.exit(3) + + return URL.create( + drivername=driver, + username=DB_USER, + password=DB_PASSWD, + host=DB_HOST, + port=DB_PORT, + database=DB_NAME, + ) def _parse_config(self): """Parses each entry in the config.yml""" diff --git a/poetry.lock b/poetry.lock index 7b40c3691..ff22dadf9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.5 and should not be changed by hand. [[package]] name = "alembic" @@ -1044,22 +1044,22 @@ testing = ["pytest"] [[package]] name = "mariadb" -version = "1.1.10" +version = "1.1.11" description = "Python MariaDB extension" optional = false python-versions = ">=3.8" files = [ - {file = "mariadb-1.1.10-cp310-cp310-win32.whl", hash = "sha256:1d81b22efbaaf4c5bc5e4cc4e2ef3c459538c1a939371089d8c5591d6f26a62e"}, - {file = "mariadb-1.1.10-cp310-cp310-win_amd64.whl", hash = "sha256:dff8b28ce4044574870d7bdd2d9f9f5da8e5f95a7ff6d226185db733060d1a93"}, - {file = "mariadb-1.1.10-cp311-cp311-win32.whl", hash = "sha256:1ce87971c02375236ff8933e6c593c748e7b2f2950b86eabfab4289fd250ea63"}, - {file = "mariadb-1.1.10-cp311-cp311-win_amd64.whl", hash = "sha256:4521aa721f926946bd71491f872e8babc78fa97755ed2114f5684b77363107cb"}, - {file = "mariadb-1.1.10-cp312-cp312-win32.whl", hash = "sha256:03d6284ef713d1cad40146576a4cc2d6cbc1662060f2a0e59b174e1694521698"}, - {file = "mariadb-1.1.10-cp312-cp312-win_amd64.whl", hash = "sha256:8c8c6b27486b0e1772a23002c702b5fd244eecf9f05633dd6cb345fc26755a20"}, - {file = "mariadb-1.1.10-cp38-cp38-win32.whl", hash = "sha256:5d652117e2fdf12b9723e7452a05fce9e6ccbae6ea48871b21a3a8fde259dc48"}, - {file = "mariadb-1.1.10-cp38-cp38-win_amd64.whl", hash = "sha256:49200378c614984f5ec875481662a49d7c97c2be27970b01b32fa4b7520d4e22"}, - {file = "mariadb-1.1.10-cp39-cp39-win32.whl", hash = "sha256:d7b09ec4abd02ed235257feb769f90cd4066e8f536b55b92f5166103d5b66a63"}, - {file = "mariadb-1.1.10-cp39-cp39-win_amd64.whl", hash = "sha256:29040e426f877ddc45f337c6eb381b6bbab63cc6bf8431a28effe30162142513"}, - {file = "mariadb-1.1.10.tar.gz", hash = "sha256:a332893e3ef7ceb7970ab4bd7c844bcb4bd68a051ca51313566f9808d7411f2d"}, + {file = "mariadb-1.1.11-cp310-cp310-win32.whl", hash = "sha256:dbc4cf0e302ca82d46f9431a0b04f048e9c21ee56d6f3162c29605f84d63b40c"}, + {file = "mariadb-1.1.11-cp310-cp310-win_amd64.whl", hash = "sha256:579420293fa790d5ae0a6cb4bdb7e8be8facc2ceefb6123c2b0e8042b3fa725d"}, + {file = "mariadb-1.1.11-cp311-cp311-win32.whl", hash = "sha256:0f8de8d66ca71bd102f34a970a331b7d75bdf7f8050d80e37cdcc6ff3c85cf7a"}, + {file = "mariadb-1.1.11-cp311-cp311-win_amd64.whl", hash = "sha256:3f64b520089cb60c4f8302f365ed0ae057c4c859ab70fc8b1c4358192c3c8f27"}, + {file = "mariadb-1.1.11-cp312-cp312-win32.whl", hash = "sha256:f6dfdc954edf02b6519419a054798cda6034dc459d1d482e3329e37aa27d34f0"}, + {file = "mariadb-1.1.11-cp312-cp312-win_amd64.whl", hash = "sha256:2e72ea65f1d7d8563ee84e172f2a583193092bdb6ff83c470ca9722873273ecc"}, + {file = "mariadb-1.1.11-cp313-cp313-win32.whl", hash = "sha256:d7302ccd15f0beee7b286885cbf6ac71ddc240374691d669784d99f89ba34d79"}, + {file = "mariadb-1.1.11-cp313-cp313-win_amd64.whl", hash = "sha256:c1992ebf9c6f012ac158e33fef9f2c4ba899f721064c4ae3a3489233793296c0"}, + {file = "mariadb-1.1.11-cp39-cp39-win32.whl", hash = "sha256:6f28d8ccc597a3a1368be14078110f743900dbb3b0c7f1cce3072d83bec59c8a"}, + {file = "mariadb-1.1.11-cp39-cp39-win_amd64.whl", hash = "sha256:e94f1738bec09c97b601ddbb1908eb24524ba4630f507a775d82ffdb6c5794b3"}, + {file = "mariadb-1.1.11.tar.gz", hash = "sha256:cf6647cee081e21d0994b409ba8c8fa2077f3972f1de3627c5502fb31d14f806"}, ] [package.dependencies] @@ -1329,6 +1329,48 @@ files = [ {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, ] +[[package]] +name = "mysql-connector-python" +version = "9.1.0" +description = "A self-contained Python driver for communicating with MySQL servers, using an API that is compliant with the Python Database API Specification v2.0 (PEP 249)." +optional = false +python-versions = ">=3.9" +files = [ + {file = "mysql-connector-python-9.1.0.tar.gz", hash = "sha256:346261a2aeb743a39cf66ba8bde5e45931d313b76ce0946a69a6d1187ec7d279"}, + {file = "mysql_connector_python-9.1.0-cp310-cp310-macosx_13_0_arm64.whl", hash = "sha256:dcdcf380d07b9ca6f18a95e9516a6185f2ab31a53d290d5e698e77e59c043c9e"}, + {file = "mysql_connector_python-9.1.0-cp310-cp310-macosx_13_0_x86_64.whl", hash = "sha256:948ef0c7da87901176d4320e0f40a3277ee06fe6f58ce151c1e60d8d50fdeaf4"}, + {file = "mysql_connector_python-9.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:abf16fc1155ebeba5558e5702dd7210d634ac8da484eca05a640b68a548dc7cf"}, + {file = "mysql_connector_python-9.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:aceaab679b852c0a2ec0eed9eb2a490171b3493484f1881b605cbf2f9c5fde6d"}, + {file = "mysql_connector_python-9.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:72dcce5f2e4f5910d65f02eb318c1e4622464da007a3ae5e9ccd64169d8efac3"}, + {file = "mysql_connector_python-9.1.0-cp311-cp311-macosx_13_0_arm64.whl", hash = "sha256:9b23a8e2acee91b5120febe00c53e7f472b9b6d49618e39fa1af86cdc1f0ade8"}, + {file = "mysql_connector_python-9.1.0-cp311-cp311-macosx_13_0_x86_64.whl", hash = "sha256:e15153cb8ab5fcec00b99077de536489d22d4809fc28f633850398fef0560b1f"}, + {file = "mysql_connector_python-9.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:fec943d333851c4b5e57cd0b04dde36e6817f0d4d62b2a58ce028a82be444866"}, + {file = "mysql_connector_python-9.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:c36a9b9ebf9587aaa5d7928468fefe8faf6fc993a03cb242bb160ede9cf75b2d"}, + {file = "mysql_connector_python-9.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:7b2eb48518b8c2bc9636883d264b291e5c93824fc6b61823ca9cf396a09474ad"}, + {file = "mysql_connector_python-9.1.0-cp312-cp312-macosx_13_0_arm64.whl", hash = "sha256:f67b22e3eaf5b03ffac97232d3dd67b56abcacad907ad4391c847bad5ba58f0e"}, + {file = "mysql_connector_python-9.1.0-cp312-cp312-macosx_13_0_x86_64.whl", hash = "sha256:c75f674a52b8820c90d466183b2bb59f89bcf09d17ebe9b391313d89565c8896"}, + {file = "mysql_connector_python-9.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:e75ecb3df2c2cbe4d92d5dd58a318fa708edebc0fa2d850fc2a9d42481dbb808"}, + {file = "mysql_connector_python-9.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:7d99c0a841a2c2a0e4d5b28376c1bfac794ec3821b66eb6fa2f7702cec820ee8"}, + {file = "mysql_connector_python-9.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:30a8f0ba84f8adf15a4877e80b3f97f786ce35616d918b9310578a2bd22952d5"}, + {file = "mysql_connector_python-9.1.0-cp313-cp313-macosx_13_0_arm64.whl", hash = "sha256:d627ebafc0327b935d8783454e7a4b5c32324ed39a2a1589239490ab850bf7d7"}, + {file = "mysql_connector_python-9.1.0-cp313-cp313-macosx_13_0_x86_64.whl", hash = "sha256:e26a08a9500407fa8f4a6504f7077d1312bec4fa52cb0a58c1ad324ca1f3eeaa"}, + {file = "mysql_connector_python-9.1.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:109e17a4ada1442e3881a51e2bbabcb336ad229a619ac61e9ad24bd6b9b117bd"}, + {file = "mysql_connector_python-9.1.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:4f102452c64332b7e042fa37b84d4f15332bd639e479d15035f2a005fb9fbb34"}, + {file = "mysql_connector_python-9.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:25e261f3260ec798c48cb910862a299e565548a1b5421dec84315ddbc9ef28c4"}, + {file = "mysql_connector_python-9.1.0-cp39-cp39-macosx_13_0_arm64.whl", hash = "sha256:ec4386b2426bfb07f83455bf895d8a7e2d6c067343ac05be5511083ca2424991"}, + {file = "mysql_connector_python-9.1.0-cp39-cp39-macosx_13_0_x86_64.whl", hash = "sha256:28fd99ee464ac3b02d1e2a71a63ca4f25c6110e4414a46a5b64631e6d2096899"}, + {file = "mysql_connector_python-9.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:e2f0876e1efd76e05853cb0a623dba2746ee70686c043019d811737dd5c3d871"}, + {file = "mysql_connector_python-9.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:6d7d5d458d0d600bbbebd9f2bce551e386b359bcce6026f7369b57922d26f13a"}, + {file = "mysql_connector_python-9.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:c350b1aaf257b1b778f44b8bfaeda07751f55e150f5a7464342f36e4aac8e805"}, + {file = "mysql_connector_python-9.1.0-py2.py3-none-any.whl", hash = "sha256:dacf1aa84dc7dd8ae908626c3ae50fce956d0105130c7465fd248a4f035d50b1"}, +] + +[package.extras] +dns-srv = ["dnspython (==2.6.1)"] +fido2 = ["fido2 (==1.1.2)"] +gssapi = ["gssapi (==1.8.3)"] +telemetry = ["opentelemetry-api (==1.18.0)", "opentelemetry-exporter-otlp-proto-http (==1.18.0)", "opentelemetry-sdk (==1.18.0)"] + [[package]] name = "nest-asyncio" version = "1.6.0" @@ -2761,6 +2803,8 @@ files = [ [package.dependencies] greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +mariadb = {version = ">=1.0.1,<1.1.2 || >1.1.2,<1.1.5 || >1.1.5,<1.1.10 || >1.1.10", optional = true, markers = "extra == \"mariadb-connector\""} +mysql-connector-python = {version = "*", optional = true, markers = "extra == \"mysql-connector\""} typing-extensions = ">=4.6.0" [package.extras] @@ -3468,4 +3512,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.12" -content-hash = "bdd25954db1482642634d4460b0320fdcbaf77b00c7f7606a4c06b7c86f38463" +content-hash = "0f6f275d52781259c3780482aab4270849bba5f4e09bcd1c90bea8ead769dc72" diff --git a/pyproject.toml b/pyproject.toml index a83577f2b..e94071fb0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,7 +15,10 @@ uvicorn = "0.29.0" gunicorn = "22.0.0" websockets = "12.0" python-socketio = "5.11.1" -SQLAlchemy = "^2.0.30" +SQLAlchemy = { version = "^2.0.30", extras = [ + "mariadb-connector", + "mysql-connector", +] } alembic = "1.13.1" PyYAML = "6.0.1" Unidecode = "1.3.8" @@ -23,7 +26,6 @@ emoji = "2.10.1" python-dotenv = "1.0.1" sqlakeyset = "^2.0.1708907391" pydash = "^7.0.7" -mariadb = "1.1.10" rq = "^1.16.1" redis = "^5.0" passlib = { extras = ["bcrypt"], version = "^1.7.4" }