From e46f84344da82a19b810559fbeea9105963d3745 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 24 Apr 2025 16:48:07 +0000 Subject: [PATCH 01/32] Bump h11 from 0.14.0 to 0.16.0 Bumps [h11](https://github.com/python-hyper/h11) from 0.14.0 to 0.16.0. - [Commits](https://github.com/python-hyper/h11/compare/v0.14.0...v0.16.0) --- updated-dependencies: - dependency-name: h11 dependency-version: 0.16.0 dependency-type: indirect ... Signed-off-by: dependabot[bot] --- poetry.lock | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/poetry.lock b/poetry.lock index 144f22573..fcad721ed 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3716,24 +3716,6 @@ files = [ {file = "Unidecode-1.3.8.tar.gz", hash = "sha256:cfdb349d46ed3873ece4586b96aa75258726e2fa8ec21d6f00a591d98806c2f4"}, ] -[[package]] -name = "urllib3" -version = "1.26.20" -description = "HTTP library with thread-safe connection pooling, file post, and more." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" -groups = ["main"] -markers = "platform_python_implementation == \"PyPy\"" -files = [ - {file = "urllib3-1.26.20-py2.py3-none-any.whl", hash = "sha256:0ed14ccfbf1c30a9072c7ca157e4319b70d65f623e91e7b32fadb2853431016e"}, - {file = "urllib3-1.26.20.tar.gz", hash = "sha256:40c2dc0c681e47eb8f90e7e27bf6ff7df2e677421fd46756da1161c39ca70d32"}, -] - -[package.extras] -brotli = ["brotli (==1.0.9) ; os_name != \"nt\" and python_version < \"3\" and platform_python_implementation == \"CPython\"", "brotli (>=1.0.9) ; python_version >= \"3\" and platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; (os_name != \"nt\" or python_version >= \"3\") and platform_python_implementation != \"CPython\"", "brotlipy (>=0.6.0) ; os_name == \"nt\" and python_version < \"3\""] -secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress ; python_version == \"2.7\"", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] -socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] - [[package]] name = "urllib3" version = "2.3.0" @@ -3741,7 +3723,7 @@ description = "HTTP library with thread-safe connection pooling, file post, and optional = false python-versions = ">=3.9" groups = ["main"] -markers = "platform_python_implementation != \"PyPy\"" +markers = "platform_python_implementation == \"PyPy\"" files = [ {file = "urllib3-2.3.0-py3-none-any.whl", hash = "sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df"}, {file = "urllib3-2.3.0.tar.gz", hash = "sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d"}, @@ -3773,6 +3755,24 @@ h11 = ">=0.8" [package.extras] standard = ["colorama (>=0.4) ; sys_platform == \"win32\"", "httptools (>=0.5.0)", "python-dotenv (>=0.13)", "pyyaml (>=5.1)", "uvloop (>=0.14.0,!=0.15.0,!=0.15.1) ; sys_platform != \"win32\" and sys_platform != \"cygwin\" and platform_python_implementation != \"PyPy\"", "watchfiles (>=0.13)", "websockets (>=10.4)"] +[[package]] +name = "vcrpy" +version = "5.1.0" +description = "Automatically mock your HTTP interactions to simplify and speed up testing" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "platform_python_implementation == \"PyPy\" and extra == \"test\"" +files = [ + {file = "vcrpy-5.1.0-py2.py3-none-any.whl", hash = "sha256:605e7b7a63dcd940db1df3ab2697ca7faf0e835c0852882142bafb19649d599e"}, + {file = "vcrpy-5.1.0.tar.gz", hash = "sha256:bbf1532f2618a04f11bce2a99af3a9647a32c880957293ff91e0a5f187b6b3d2"}, +] + +[package.dependencies] +PyYAML = "*" +wrapt = "*" +yarl = "*" + [[package]] name = "vcrpy" version = "7.0.0" @@ -3780,7 +3780,7 @@ description = "Automatically mock your HTTP interactions to simplify and speed u optional = true python-versions = ">=3.9" groups = ["main"] -markers = "platform_python_implementation == \"PyPy\" and extra == \"test\"" +markers = "platform_python_implementation != \"PyPy\" and extra == \"test\"" files = [ {file = "vcrpy-7.0.0-py2.py3-none-any.whl", hash = "sha256:55791e26c18daa363435054d8b35bd41a4ac441b6676167635d1b37a71dbe124"}, {file = "vcrpy-7.0.0.tar.gz", hash = "sha256:176391ad0425edde1680c5b20738ea3dc7fb942520a48d2993448050986b3a50"}, @@ -3788,10 +3788,7 @@ files = [ [package.dependencies] PyYAML = "*" -urllib3 = [ - {version = "<2", markers = "platform_python_implementation == \"PyPy\""}, - {version = "*", markers = "platform_python_implementation != \"PyPy\" and python_version >= \"3.10\""}, -] +urllib3 = {version = "*", markers = "platform_python_implementation != \"PyPy\" and python_version >= \"3.10\""} wrapt = "*" yarl = "*" From 737f406f55834bb2016fe0164f0d278bd59c66fa Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 30 Apr 2025 19:41:32 +0000 Subject: [PATCH 02/32] build(deps-dev): bump vite from 6.2.6 to 6.3.4 in /frontend Bumps [vite](https://github.com/vitejs/vite/tree/HEAD/packages/vite) from 6.2.6 to 6.3.4. - [Release notes](https://github.com/vitejs/vite/releases) - [Changelog](https://github.com/vitejs/vite/blob/main/packages/vite/CHANGELOG.md) - [Commits](https://github.com/vitejs/vite/commits/v6.3.4/packages/vite) --- updated-dependencies: - dependency-name: vite dependency-version: 6.3.4 dependency-type: direct:development ... Signed-off-by: dependabot[bot] --- frontend/package-lock.json | 270 +++++++++++++++++++++++-------------- frontend/package.json | 2 +- 2 files changed, 172 insertions(+), 100 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 65bfbb333..f65c03f15 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -43,7 +43,7 @@ "openapi-typescript-codegen": "^0.29.0", "typescript": "^5.7.3", "typescript-eslint": "^8.28.0", - "vite": "^6.2.6", + "vite": "^6.3.4", "vite-plugin-pwa": "^0.21.1", "vite-plugin-vuetify": "^2.0.4", "vue-tsc": "^2.2.8" @@ -2916,228 +2916,260 @@ } }, "node_modules/@rollup/rollup-android-arm-eabi": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.34.2.tgz", - "integrity": "sha512-6Fyg9yQbwJR+ykVdT9sid1oc2ewejS6h4wzQltmJfSW53N60G/ah9pngXGANdy9/aaE/TcUFpWosdm7JXS1WTQ==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.40.1.tgz", + "integrity": "sha512-kxz0YeeCrRUHz3zyqvd7n+TVRlNyTifBsmnmNPtk3hQURUyG9eAB+usz6DAwagMusjx/zb3AjvDUvhFGDAexGw==", "cpu": [ "arm" ], + "license": "MIT", "optional": true, "os": [ "android" ] }, "node_modules/@rollup/rollup-android-arm64": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.34.2.tgz", - "integrity": "sha512-K5GfWe+vtQ3kyEbihrimM38UgX57UqHp+oME7X/EX9Im6suwZfa7Hsr8AtzbJvukTpwMGs+4s29YMSO3rwWtsw==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.40.1.tgz", + "integrity": "sha512-PPkxTOisoNC6TpnDKatjKkjRMsdaWIhyuMkA4UsBXT9WEZY4uHezBTjs6Vl4PbqQQeu6oION1w2voYZv9yquCw==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "android" ] }, "node_modules/@rollup/rollup-darwin-arm64": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.34.2.tgz", - "integrity": "sha512-PSN58XG/V/tzqDb9kDGutUruycgylMlUE59f40ny6QIRNsTEIZsrNQTJKUN2keMMSmlzgunMFqyaGLmly39sug==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.40.1.tgz", + "integrity": "sha512-VWXGISWFY18v/0JyNUy4A46KCFCb9NVsH+1100XP31lud+TzlezBbz24CYzbnA4x6w4hx+NYCXDfnvDVO6lcAA==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "darwin" ] }, "node_modules/@rollup/rollup-darwin-x64": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.34.2.tgz", - "integrity": "sha512-gQhK788rQJm9pzmXyfBB84VHViDERhAhzGafw+E5mUpnGKuxZGkMVDa3wgDFKT6ukLC5V7QTifzsUKdNVxp5qQ==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.40.1.tgz", + "integrity": "sha512-nIwkXafAI1/QCS7pxSpv/ZtFW6TXcNUEHAIA9EIyw5OzxJZQ1YDrX+CL6JAIQgZ33CInl1R6mHet9Y/UZTg2Bw==", "cpu": [ "x64" ], + "license": "MIT", "optional": true, "os": [ "darwin" ] }, "node_modules/@rollup/rollup-freebsd-arm64": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.34.2.tgz", - "integrity": "sha512-eiaHgQwGPpxLC3+zTAcdKl4VsBl3r0AiJOd1Um/ArEzAjN/dbPK1nROHrVkdnoE6p7Svvn04w3f/jEZSTVHunA==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.40.1.tgz", + "integrity": "sha512-BdrLJ2mHTrIYdaS2I99mriyJfGGenSaP+UwGi1kB9BLOCu9SR8ZpbkmmalKIALnRw24kM7qCN0IOm6L0S44iWw==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "freebsd" ] }, "node_modules/@rollup/rollup-freebsd-x64": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.34.2.tgz", - "integrity": "sha512-lhdiwQ+jf8pewYOTG4bag0Qd68Jn1v2gO1i0mTuiD+Qkt5vNfHVK/jrT7uVvycV8ZchlzXp5HDVmhpzjC6mh0g==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.40.1.tgz", + "integrity": "sha512-VXeo/puqvCG8JBPNZXZf5Dqq7BzElNJzHRRw3vjBE27WujdzuOPecDPc/+1DcdcTptNBep3861jNq0mYkT8Z6Q==", "cpu": [ "x64" ], + "license": "MIT", "optional": true, "os": [ "freebsd" ] }, "node_modules/@rollup/rollup-linux-arm-gnueabihf": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.34.2.tgz", - "integrity": "sha512-lfqTpWjSvbgQP1vqGTXdv+/kxIznKXZlI109WkIFPbud41bjigjNmOAAKoazmRGx+k9e3rtIdbq2pQZPV1pMig==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.40.1.tgz", + "integrity": "sha512-ehSKrewwsESPt1TgSE/na9nIhWCosfGSFqv7vwEtjyAqZcvbGIg4JAcV7ZEh2tfj/IlfBeZjgOXm35iOOjadcg==", "cpu": [ "arm" ], + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-arm-musleabihf": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.34.2.tgz", - "integrity": "sha512-RGjqULqIurqqv+NJTyuPgdZhka8ImMLB32YwUle2BPTDqDoXNgwFjdjQC59FbSk08z0IqlRJjrJ0AvDQ5W5lpw==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.40.1.tgz", + "integrity": "sha512-m39iO/aaurh5FVIu/F4/Zsl8xppd76S4qoID8E+dSRQvTyZTOI2gVk3T4oqzfq1PtcvOfAVlwLMK3KRQMaR8lg==", "cpu": [ "arm" ], + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-arm64-gnu": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.34.2.tgz", - "integrity": "sha512-ZvkPiheyXtXlFqHpsdgscx+tZ7hoR59vOettvArinEspq5fxSDSgfF+L5wqqJ9R4t+n53nyn0sKxeXlik7AY9Q==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.40.1.tgz", + "integrity": "sha512-Y+GHnGaku4aVLSgrT0uWe2o2Rq8te9hi+MwqGF9r9ORgXhmHK5Q71N757u0F8yU1OIwUIFy6YiJtKjtyktk5hg==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-arm64-musl": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.34.2.tgz", - "integrity": "sha512-UlFk+E46TZEoxD9ufLKDBzfSG7Ki03fo6hsNRRRHF+KuvNZ5vd1RRVQm8YZlGsjcJG8R252XFK0xNPay+4WV7w==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.40.1.tgz", + "integrity": "sha512-jEwjn3jCA+tQGswK3aEWcD09/7M5wGwc6+flhva7dsQNRZZTe30vkalgIzV4tjkopsTS9Jd7Y1Bsj6a4lzz8gQ==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-loongarch64-gnu": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.34.2.tgz", - "integrity": "sha512-hJhfsD9ykx59jZuuoQgYT1GEcNNi3RCoEmbo5OGfG8RlHOiVS7iVNev9rhLKh7UBYq409f4uEw0cclTXx8nh8Q==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loongarch64-gnu/-/rollup-linux-loongarch64-gnu-4.40.1.tgz", + "integrity": "sha512-ySyWikVhNzv+BV/IDCsrraOAZ3UaC8SZB67FZlqVwXwnFhPihOso9rPOxzZbjp81suB1O2Topw+6Ug3JNegejQ==", "cpu": [ "loong64" ], + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-powerpc64le-gnu": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.34.2.tgz", - "integrity": "sha512-g/O5IpgtrQqPegvqopvmdCF9vneLE7eqYfdPWW8yjPS8f63DNam3U4ARL1PNNB64XHZDHKpvO2Giftf43puB8Q==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-powerpc64le-gnu/-/rollup-linux-powerpc64le-gnu-4.40.1.tgz", + "integrity": "sha512-BvvA64QxZlh7WZWqDPPdt0GH4bznuL6uOO1pmgPnnv86rpUpc8ZxgZwcEgXvo02GRIZX1hQ0j0pAnhwkhwPqWg==", "cpu": [ "ppc64" ], + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-riscv64-gnu": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.34.2.tgz", - "integrity": "sha512-bSQijDC96M6PuooOuXHpvXUYiIwsnDmqGU8+br2U7iPoykNi9JtMUpN7K6xml29e0evK0/g0D1qbAUzWZFHY5Q==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.40.1.tgz", + "integrity": "sha512-EQSP+8+1VuSulm9RKSMKitTav89fKbHymTf25n5+Yr6gAPZxYWpj3DzAsQqoaHAk9YX2lwEyAf9S4W8F4l3VBQ==", "cpu": [ "riscv64" ], + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.40.1.tgz", + "integrity": "sha512-n/vQ4xRZXKuIpqukkMXZt9RWdl+2zgGNx7Uda8NtmLJ06NL8jiHxUawbwC+hdSq1rrw/9CghCpEONor+l1e2gA==", + "cpu": [ + "riscv64" + ], + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-s390x-gnu": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.34.2.tgz", - "integrity": "sha512-49TtdeVAsdRuiUHXPrFVucaP4SivazetGUVH8CIxVsNsaPHV4PFkpLmH9LeqU/R4Nbgky9lzX5Xe1NrzLyraVA==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.40.1.tgz", + "integrity": "sha512-h8d28xzYb98fMQKUz0w2fMc1XuGzLLjdyxVIbhbil4ELfk5/orZlSTpF/xdI9C8K0I8lCkq+1En2RJsawZekkg==", "cpu": [ "s390x" ], + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-x64-gnu": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.34.2.tgz", - "integrity": "sha512-j+jFdfOycLIQ7FWKka9Zd3qvsIyugg5LeZuHF6kFlXo6MSOc6R1w37YUVy8VpAKd81LMWGi5g9J25P09M0SSIw==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.40.1.tgz", + "integrity": "sha512-XiK5z70PEFEFqcNj3/zRSz/qX4bp4QIraTy9QjwJAb/Z8GM7kVUsD0Uk8maIPeTyPCP03ChdI+VVmJriKYbRHQ==", "cpu": [ "x64" ], + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-linux-x64-musl": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.34.2.tgz", - "integrity": "sha512-aDPHyM/D2SpXfSNCVWCxyHmOqN9qb7SWkY1+vaXqMNMXslZYnwh9V/UCudl6psyG0v6Ukj7pXanIpfZwCOEMUg==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.40.1.tgz", + "integrity": "sha512-2BRORitq5rQ4Da9blVovzNCMaUlyKrzMSvkVR0D4qPuOy/+pMCrh1d7o01RATwVy+6Fa1WBw+da7QPeLWU/1mQ==", "cpu": [ "x64" ], + "license": "MIT", "optional": true, "os": [ "linux" ] }, "node_modules/@rollup/rollup-win32-arm64-msvc": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.34.2.tgz", - "integrity": "sha512-LQRkCyUBnAo7r8dbEdtNU08EKLCJMgAk2oP5H3R7BnUlKLqgR3dUjrLBVirmc1RK6U6qhtDw29Dimeer8d5hzQ==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.40.1.tgz", + "integrity": "sha512-b2bcNm9Kbde03H+q+Jjw9tSfhYkzrDUf2d5MAd1bOJuVplXvFhWz7tRtWvD8/ORZi7qSCy0idW6tf2HgxSXQSg==", "cpu": [ "arm64" ], + "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@rollup/rollup-win32-ia32-msvc": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.34.2.tgz", - "integrity": "sha512-wt8OhpQUi6JuPFkm1wbVi1BByeag87LDFzeKSXzIdGcX4bMLqORTtKxLoCbV57BHYNSUSOKlSL4BYYUghainYA==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.40.1.tgz", + "integrity": "sha512-DfcogW8N7Zg7llVEfpqWMZcaErKfsj9VvmfSyRjCyo4BI3wPEfrzTtJkZG6gKP/Z92wFm6rz2aDO7/JfiR/whA==", "cpu": [ "ia32" ], + "license": "MIT", "optional": true, "os": [ "win32" ] }, "node_modules/@rollup/rollup-win32-x64-msvc": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.34.2.tgz", - "integrity": "sha512-rUrqINax0TvrPBXrFKg0YbQx18NpPN3NNrgmaao9xRNbTwek7lOXObhx8tQy8gelmQ/gLaGy1WptpU2eKJZImg==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.40.1.tgz", + "integrity": "sha512-ECyOuDeH3C1I8jH2MK1RtBJW+YPMvSfT0a5NN0nHfQYnDSJ6tUiZH3gzwVP5/Kfh/+Tt7tpWVF9LXNTnhTJ3kA==", "cpu": [ "x64" ], + "license": "MIT", "optional": true, "os": [ "win32" @@ -3181,10 +3213,11 @@ } }, "node_modules/@types/estree": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.6.tgz", - "integrity": "sha512-AYnb1nQyY49te+VRAVgmzfcgjYS91mY5P0TKUDCLEM+gNnA+3T6rWITXRLYCpahpqSQbN5cE+gHpnPyXjHWxcw==", - "devOptional": true + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.7.tgz", + "integrity": "sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ==", + "devOptional": true, + "license": "MIT" }, "node_modules/@types/js-cookie": { "version": "3.0.6", @@ -7074,12 +7107,13 @@ "license": "MIT" }, "node_modules/rollup": { - "version": "4.34.2", - "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.34.2.tgz", - "integrity": "sha512-sBDUoxZEaqLu9QeNalL8v3jw6WjPku4wfZGyTU7l7m1oC+rpRihXc/n/H+4148ZkGz5Xli8CHMns//fFGKvpIQ==", + "version": "4.40.1", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.40.1.tgz", + "integrity": "sha512-C5VvvgCCyfyotVITIAv+4efVytl5F7wt+/I2i9q9GZcEXW9BP52YYOXC58igUi+LFZVHukErIIqQSWwv/M3WRw==", "devOptional": true, + "license": "MIT", "dependencies": { - "@types/estree": "1.0.6" + "@types/estree": "1.0.7" }, "bin": { "rollup": "dist/bin/rollup" @@ -7089,25 +7123,26 @@ "npm": ">=8.0.0" }, "optionalDependencies": { - "@rollup/rollup-android-arm-eabi": "4.34.2", - "@rollup/rollup-android-arm64": "4.34.2", - "@rollup/rollup-darwin-arm64": "4.34.2", - "@rollup/rollup-darwin-x64": "4.34.2", - "@rollup/rollup-freebsd-arm64": "4.34.2", - "@rollup/rollup-freebsd-x64": "4.34.2", - "@rollup/rollup-linux-arm-gnueabihf": "4.34.2", - "@rollup/rollup-linux-arm-musleabihf": "4.34.2", - "@rollup/rollup-linux-arm64-gnu": "4.34.2", - "@rollup/rollup-linux-arm64-musl": "4.34.2", - "@rollup/rollup-linux-loongarch64-gnu": "4.34.2", - "@rollup/rollup-linux-powerpc64le-gnu": "4.34.2", - "@rollup/rollup-linux-riscv64-gnu": "4.34.2", - "@rollup/rollup-linux-s390x-gnu": "4.34.2", - "@rollup/rollup-linux-x64-gnu": "4.34.2", - "@rollup/rollup-linux-x64-musl": "4.34.2", - "@rollup/rollup-win32-arm64-msvc": "4.34.2", - "@rollup/rollup-win32-ia32-msvc": "4.34.2", - "@rollup/rollup-win32-x64-msvc": "4.34.2", + "@rollup/rollup-android-arm-eabi": "4.40.1", + "@rollup/rollup-android-arm64": "4.40.1", + "@rollup/rollup-darwin-arm64": "4.40.1", + "@rollup/rollup-darwin-x64": "4.40.1", + "@rollup/rollup-freebsd-arm64": "4.40.1", + "@rollup/rollup-freebsd-x64": "4.40.1", + "@rollup/rollup-linux-arm-gnueabihf": "4.40.1", + "@rollup/rollup-linux-arm-musleabihf": "4.40.1", + "@rollup/rollup-linux-arm64-gnu": "4.40.1", + "@rollup/rollup-linux-arm64-musl": "4.40.1", + "@rollup/rollup-linux-loongarch64-gnu": "4.40.1", + "@rollup/rollup-linux-powerpc64le-gnu": "4.40.1", + "@rollup/rollup-linux-riscv64-gnu": "4.40.1", + "@rollup/rollup-linux-riscv64-musl": "4.40.1", + "@rollup/rollup-linux-s390x-gnu": "4.40.1", + "@rollup/rollup-linux-x64-gnu": "4.40.1", + "@rollup/rollup-linux-x64-musl": "4.40.1", + "@rollup/rollup-win32-arm64-msvc": "4.40.1", + "@rollup/rollup-win32-ia32-msvc": "4.40.1", + "@rollup/rollup-win32-x64-msvc": "4.40.1", "fsevents": "~2.3.2" } }, @@ -7714,23 +7749,28 @@ "devOptional": true }, "node_modules/tinyglobby": { - "version": "0.2.10", - "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.10.tgz", - "integrity": "sha512-Zc+8eJlFMvgatPZTl6A9L/yht8QqdmUNtURHaKZLmKBE12hNPSrqNkUp2cs3M/UKmNVVAMFQYSjYIVHDjW5zew==", - "dev": true, + "version": "0.2.13", + "resolved": "https://registry.npmjs.org/tinyglobby/-/tinyglobby-0.2.13.tgz", + "integrity": "sha512-mEwzpUgrLySlveBwEVDMKk5B57bhLPYovRfPAXD5gA/98Opn0rCDj3GtLwFvCvH5RK9uPCExUROW5NjDwvqkxw==", + "devOptional": true, + "license": "MIT", "dependencies": { - "fdir": "^6.4.2", + "fdir": "^6.4.4", "picomatch": "^4.0.2" }, "engines": { "node": ">=12.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/SuperchupuDev" } }, "node_modules/tinyglobby/node_modules/fdir": { - "version": "6.4.3", - "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.3.tgz", - "integrity": "sha512-PMXmW2y1hDDfTSRc9gaXIuCCRpuoz3Kaz8cUelp3smouvfT632ozg2vrT6lJsHKKOF59YLbOGfAWGUcKEfRMQw==", - "dev": true, + "version": "6.4.4", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.4.tgz", + "integrity": "sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg==", + "devOptional": true, + "license": "MIT", "peerDependencies": { "picomatch": "^3 || ^4" }, @@ -7744,7 +7784,8 @@ "version": "4.0.2", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", - "dev": true, + "devOptional": true, + "license": "MIT", "engines": { "node": ">=12" }, @@ -8081,15 +8122,18 @@ "dev": true }, "node_modules/vite": { - "version": "6.2.6", - "resolved": "https://registry.npmjs.org/vite/-/vite-6.2.6.tgz", - "integrity": "sha512-9xpjNl3kR4rVDZgPNdTL0/c6ao4km69a/2ihNQbcANz8RuCOK3hQBmLSJf3bRKVQjVMda+YvizNE8AwvogcPbw==", + "version": "6.3.4", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.3.4.tgz", + "integrity": "sha512-BiReIiMS2fyFqbqNT/Qqt4CVITDU9M9vE+DKcVAsB+ZV0wvTKd+3hMbkpxz1b+NmEDMegpVbisKiAZOnvO92Sw==", "devOptional": true, "license": "MIT", "dependencies": { "esbuild": "^0.25.0", + "fdir": "^6.4.4", + "picomatch": "^4.0.2", "postcss": "^8.5.3", - "rollup": "^4.30.1" + "rollup": "^4.34.9", + "tinyglobby": "^0.2.13" }, "bin": { "vite": "bin/vite.js" @@ -8201,6 +8245,34 @@ "vuetify": "^3.0.0" } }, + "node_modules/vite/node_modules/fdir": { + "version": "6.4.4", + "resolved": "https://registry.npmjs.org/fdir/-/fdir-6.4.4.tgz", + "integrity": "sha512-1NZP+GK4GfuAv3PqKvxQRDMjdSRZjnkq7KfhlNrCNNlZ0ygQFpebfrnfnq/W7fpUnAv9aGWmY1zKx7FYL3gwhg==", + "devOptional": true, + "license": "MIT", + "peerDependencies": { + "picomatch": "^3 || ^4" + }, + "peerDependenciesMeta": { + "picomatch": { + "optional": true + } + } + }, + "node_modules/vite/node_modules/picomatch": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", + "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "devOptional": true, + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/vscode-uri": { "version": "3.0.8", "resolved": "https://registry.npmjs.org/vscode-uri/-/vscode-uri-3.0.8.tgz", diff --git a/frontend/package.json b/frontend/package.json index 4df6ead69..65de7081e 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -61,7 +61,7 @@ "openapi-typescript-codegen": "^0.29.0", "typescript": "^5.7.3", "typescript-eslint": "^8.28.0", - "vite": "^6.2.6", + "vite": "^6.3.4", "vite-plugin-pwa": "^0.21.1", "vite-plugin-vuetify": "^2.0.4", "vue-tsc": "^2.2.8" From df00fe21d43efb105619898a7c67b064407ad53c Mon Sep 17 00:00:00 2001 From: Samuel Nitzsche Date: Fri, 2 May 2025 17:20:51 +0200 Subject: [PATCH 03/32] when updating a save files also update the screenshot --- backend/endpoints/saves.py | 32 ++++++++++++++++++- frontend/src/services/api/save.ts | 5 +++ frontend/src/views/Player/EmulatorJS/utils.ts | 6 ++++ 3 files changed, 42 insertions(+), 1 deletion(-) diff --git a/backend/endpoints/saves.py b/backend/endpoints/saves.py index 19b2e7830..e7d53b72d 100644 --- a/backend/endpoints/saves.py +++ b/backend/endpoints/saves.py @@ -95,7 +95,7 @@ async def add_save( else: scanned_screenshot.rom_id = rom.id scanned_screenshot.user_id = request.user.id - db_screenshot = db_screenshot_handler.add_screenshot( + db_screenshot_handler.add_screenshot( screenshot=scanned_screenshot ) @@ -154,6 +154,36 @@ async def update_save(request: Request, id: int) -> SaveSchema: db_save = db_save_handler.update_save( db_save.id, {"file_size_bytes": saveFile.size} ) + + screenshotFile: UploadFile | None = data.get("screenshotFile", None) # type: ignore + if screenshotFile and screenshotFile.filename: + screenshots_path = fs_asset_handler.build_screenshots_file_path( + user=request.user, platform_fs_slug=db_save.rom.platform_slug + ) + + fs_asset_handler.write_file(file=screenshotFile, path=screenshots_path) + + # Scan or update screenshot + scanned_screenshot = scan_screenshot( + file_name=screenshotFile.filename, + user=request.user, + platform_fs_slug=db_save.rom.platform_slug, + ) + db_screenshot = db_screenshot_handler.get_screenshot_by_filename( + rom_id=db_save.rom.id, user_id=request.user.id, file_name=screenshotFile.filename + ) + if db_screenshot: + db_screenshot = db_screenshot_handler.update_screenshot( + db_screenshot.id, + {"file_size_bytes": scanned_screenshot.file_size_bytes}, + ) + else: + scanned_screenshot.rom_id = db_save.rom.id + scanned_screenshot.user_id = request.user.id + db_screenshot_handler.add_screenshot( + screenshot=scanned_screenshot + ) + # Set the last played time for the current user rom_user = db_rom_handler.get_rom_user(db_save.rom_id, request.user.id) diff --git a/frontend/src/services/api/save.ts b/frontend/src/services/api/save.ts index 0222a7218..5bb0a440f 100644 --- a/frontend/src/services/api/save.ts +++ b/frontend/src/services/api/save.ts @@ -44,12 +44,17 @@ async function uploadSaves({ async function updateSave({ save, saveFile, + screenshotFile, }: { save: SaveSchema; saveFile: File; + screenshotFile?: File; }): Promise<{ data: SaveSchema }> { const formData = new FormData(); formData.append("saveFile", saveFile); + if (screenshotFile) { + formData.append("screenshotFile", screenshotFile); + } return api.put(`/saves/${save.id}`, formData); } diff --git a/frontend/src/views/Player/EmulatorJS/utils.ts b/frontend/src/views/Player/EmulatorJS/utils.ts index 65038284c..5e700653f 100644 --- a/frontend/src/views/Player/EmulatorJS/utils.ts +++ b/frontend/src/views/Player/EmulatorJS/utils.ts @@ -72,6 +72,12 @@ export async function saveSave({ saveFile: new File([saveFile], save.file_name, { type: "application/octet-stream", }), + screenshotFile: + screenshotFile && save.screenshot + ? new File([screenshotFile], save.screenshot.file_name, { + type: "application/octet-stream", + }) + : undefined, }); // Update the save in the rom object From d087266937cd2f7bc13aa524dd7e39b3b36f1c9f Mon Sep 17 00:00:00 2001 From: Georges-Antoine Assi Date: Sun, 4 May 2025 21:00:56 -0400 Subject: [PATCH 04/32] also add to states endpoint --- backend/endpoints/saves.py | 15 ++++++--------- backend/endpoints/states.py | 31 ++++++++++++++++++++++++++++++ frontend/src/services/api/save.ts | 4 +--- frontend/src/services/api/state.ts | 3 +++ 4 files changed, 41 insertions(+), 12 deletions(-) diff --git a/backend/endpoints/saves.py b/backend/endpoints/saves.py index e7d53b72d..0b977893c 100644 --- a/backend/endpoints/saves.py +++ b/backend/endpoints/saves.py @@ -95,9 +95,7 @@ async def add_save( else: scanned_screenshot.rom_id = rom.id scanned_screenshot.user_id = request.user.id - db_screenshot_handler.add_screenshot( - screenshot=scanned_screenshot - ) + db_screenshot_handler.add_screenshot(screenshot=scanned_screenshot) # Set the last played time for the current user rom_user = db_rom_handler.get_rom_user(rom_id=rom.id, user_id=request.user.id) @@ -154,7 +152,7 @@ async def update_save(request: Request, id: int) -> SaveSchema: db_save = db_save_handler.update_save( db_save.id, {"file_size_bytes": saveFile.size} ) - + screenshotFile: UploadFile | None = data.get("screenshotFile", None) # type: ignore if screenshotFile and screenshotFile.filename: screenshots_path = fs_asset_handler.build_screenshots_file_path( @@ -170,7 +168,9 @@ async def update_save(request: Request, id: int) -> SaveSchema: platform_fs_slug=db_save.rom.platform_slug, ) db_screenshot = db_screenshot_handler.get_screenshot_by_filename( - rom_id=db_save.rom.id, user_id=request.user.id, file_name=screenshotFile.filename + rom_id=db_save.rom.id, + user_id=request.user.id, + file_name=screenshotFile.filename, ) if db_screenshot: db_screenshot = db_screenshot_handler.update_screenshot( @@ -180,10 +180,7 @@ async def update_save(request: Request, id: int) -> SaveSchema: else: scanned_screenshot.rom_id = db_save.rom.id scanned_screenshot.user_id = request.user.id - db_screenshot_handler.add_screenshot( - screenshot=scanned_screenshot - ) - + db_screenshot_handler.add_screenshot(screenshot=scanned_screenshot) # Set the last played time for the current user rom_user = db_rom_handler.get_rom_user(db_save.rom_id, request.user.id) diff --git a/backend/endpoints/states.py b/backend/endpoints/states.py index 352018ff3..585dfe455 100644 --- a/backend/endpoints/states.py +++ b/backend/endpoints/states.py @@ -155,6 +155,37 @@ async def update_state(request: Request, id: int) -> StateSchema: db_state.id, {"file_size_bytes": stateFile.size} ) + screenshotFile: UploadFile | None = data.get("screenshotFile", None) # type: ignore + if screenshotFile and screenshotFile.filename: + screenshots_path = fs_asset_handler.build_screenshots_file_path( + user=request.user, platform_fs_slug=db_state.rom.platform_slug + ) + + fs_asset_handler.write_file(file=screenshotFile, path=screenshots_path) + + # Scan or update screenshot + scanned_screenshot = scan_screenshot( + file_name=screenshotFile.filename, + user=request.user, + platform_fs_slug=db_state.rom.platform_slug, + ) + db_screenshot = db_screenshot_handler.get_screenshot_by_filename( + rom_id=db_state.rom.id, + user_id=request.user.id, + file_name=screenshotFile.filename, + ) + if db_screenshot: + db_screenshot = db_screenshot_handler.update_screenshot( + db_screenshot.id, + {"file_size_bytes": scanned_screenshot.file_size_bytes}, + ) + else: + scanned_screenshot.rom_id = db_state.rom.id + scanned_screenshot.user_id = request.user.id + db_screenshot = db_screenshot_handler.add_screenshot( + screenshot=scanned_screenshot + ) + # Set the last played time for the current user rom_user = db_rom_handler.get_rom_user(db_state.rom_id, request.user.id) if not rom_user: diff --git a/frontend/src/services/api/save.ts b/frontend/src/services/api/save.ts index 5bb0a440f..0a65bc00c 100644 --- a/frontend/src/services/api/save.ts +++ b/frontend/src/services/api/save.ts @@ -52,9 +52,7 @@ async function updateSave({ }): Promise<{ data: SaveSchema }> { const formData = new FormData(); formData.append("saveFile", saveFile); - if (screenshotFile) { - formData.append("screenshotFile", screenshotFile); - } + if (screenshotFile) formData.append("screenshotFile", screenshotFile); return api.put(`/saves/${save.id}`, formData); } diff --git a/frontend/src/services/api/state.ts b/frontend/src/services/api/state.ts index 23881bedf..4dccd7a92 100644 --- a/frontend/src/services/api/state.ts +++ b/frontend/src/services/api/state.ts @@ -44,12 +44,15 @@ async function uploadStates({ async function updateState({ state, stateFile, + screenshotFile, }: { state: StateSchema; stateFile: File; + screenshotFile?: File; }): Promise<{ data: StateSchema }> { const formData = new FormData(); formData.append("stateFile", stateFile); + if (screenshotFile) formData.append("screenshotFile", screenshotFile); return api.put(`/states/${state.id}`, formData); } From 3bd9382b12ea1eab5a8fc5a455f7de5881561dc5 Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 08:59:56 +0000 Subject: [PATCH 05/32] refactor: enable scheduled tasks in scheduler and init scripts --- backend/scheduler.py | 17 ++++++++++------- docker/init_scripts/init | 25 ++++++++++++++++--------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/backend/scheduler.py b/backend/scheduler.py index 2105c65ac..2bfa37c52 100644 --- a/backend/scheduler.py +++ b/backend/scheduler.py @@ -1,20 +1,23 @@ import sentry_sdk -from config import SENTRY_DSN +from config import ( + ENABLE_SCHEDULED_RESCAN, + ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, + SENTRY_DSN, +) from logger.logger import log from tasks.scan_library import scan_library_task from tasks.tasks import tasks_scheduler from tasks.update_switch_titledb import update_switch_titledb_task from utils import get_version -sentry_sdk.init( - dsn=SENTRY_DSN, - release="romm@" + get_version(), -) +sentry_sdk.init(dsn=SENTRY_DSN, release=f"romm@{get_version()}") if __name__ == "__main__": # Initialize the tasks - scan_library_task.init() - update_switch_titledb_task.init() + if ENABLE_SCHEDULED_RESCAN: + scan_library_task.init() + if ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: + update_switch_titledb_task.init() log.info("Starting scheduler") diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 2aaf0638f..9506c6cf2 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -6,7 +6,9 @@ set -o pipefail # treat errors in pipes as fatal shopt -s inherit_errexit # inherit errexit # make it possible to disable the inotify watcher process -ENABLE_RESCAN_ON_FILESYSTEM_CHANGE="${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE:="true"}" +ENABLE_RESCAN_ON_FILESYSTEM_CHANGE="${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE:="false"}" +ENABLE_SCHEDULED_RESCAN="${ENABLE_SCHEDULED_RESCAN:="false"}" +ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB="${ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB:="false"}" # Set INIT_DEBUG to false if not set by docker env # (this env var is currently undocumented and usually just needed for development purposes) @@ -186,19 +188,24 @@ while ! ((exited)); do # Start nginx if we dont have a corresponding PID file watchdog_process_pid bin nginx - # only start the watcher.py if we actually want to use the rescan on fs change feature + # Start background worker processes + debug_log "Starting worker" + # Start worker if we dont have a corresponding PID file + watchdog_process_pid python worker + + # only start the scheduler.py if enabled + if [[ ${ENABLE_SCHEDULED_RESCAN} == "true" || ${ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB} == "true" ]]; then + debug_log "Starting scheduler" + # Start scheduler if we dont have a corresponding PID file + watchdog_process_pid python scheduler + fi + + # only start the watcher.py if enabled if [[ ${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE} == "true" ]]; then # Start watcher if we dont have a corresponding PID file watchdog_process_pid python watcher fi - # Start background worker processes - debug_log "Starting worker and scheduler" - # Start worker if we dont have a corresponding PID file - watchdog_process_pid python worker - # Start scheduler if we dont have a corresponding PID file - watchdog_process_pid python scheduler - # check for died processes every 5 seconds sleep 5 done From 82c265e7d3e642d8b7263e495c479a1ee5b6a397 Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 10:40:08 +0000 Subject: [PATCH 06/32] refactor: update log messages for consistency and clarity --- docker/init_scripts/init | 52 ++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 9506c6cf2..673ddcf48 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -17,29 +17,29 @@ INIT_DEBUG="${INIT_DEBUG:="false"}" # print debug log output if enabled debug_log() { if [[ ${INIT_DEBUG} == "true" ]]; then - echo "DEBUG: [init][$(date +"%Y-%m-%d %T")]" "${@}" || true + echo "DEBUG: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true fi } info_log() { - echo "INFO: [init][$(date +"%Y-%m-%d %T")]" "${@}" || true + echo "INFO: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true } warn_log() { - echo "WARNING: [init][$(date +"%Y-%m-%d %T")]" "${@}" || true + echo "WARNING: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true } error_log() { - echo "ERROR: [init][$(date +"%Y-%m-%d %T")]" "${@}" || true + echo "ERROR: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true exit 1 } wait_for_gunicorn_socket() { - info_log "waiting for gunicorn socket file..." + info_debug "Waiting for gunicorn socket file..." while [[ ! -S /tmp/gunicorn.sock ]]; do sleep 1 done - info_log "gunicorn socket file found" + info_debug "Gunicorn socket file found" } # function that runs or main process and creates a corresponding PID file, @@ -48,7 +48,7 @@ start_bin_gunicorn() { rm /tmp/gunicorn.sock -f # commands to start our main application and store its PID to check for crashes - info_log "starting gunicorn" + info_log "Starting backend" # TODO: Remove support for GUNICORN_WORKERS in future version. if [[ -n ${GUNICORN_WORKERS-} ]]; then @@ -72,7 +72,7 @@ start_bin_gunicorn() { start_bin_nginx() { wait_for_gunicorn_socket - info_log "starting nginx" + info_log "Starting internal nginx" if [[ ${EUID} -ne 0 ]]; then nginx else @@ -82,7 +82,7 @@ start_bin_nginx() { } start_bin_valkey-server() { - info_log "starting valkey-server" + info_log "Starting internal valkey-server" # Check if /usr/local/etc/valkey/valkey.conf exists and use it if so if [[ -f /usr/local/etc/valkey/valkey.conf ]]; then valkey-server /usr/local/etc/valkey/valkey.conf & @@ -96,7 +96,7 @@ start_bin_valkey-server() { # function that runs our independent python scripts and creates corresponding PID files, start_python() { SCRIPT="${1}" - info_log "starting ${SCRIPT}.py" + info_log "Starting ${SCRIPT}" python3 "${SCRIPT}.py" & WATCHER_PID=$! echo "${WATCHER_PID}" >"/tmp/${SCRIPT}.pid" @@ -131,7 +131,7 @@ stop_process_pid() { if [[ -f "/tmp/${PROCESS}.pid" ]]; then PID=$(cat "/tmp/${PROCESS}.pid") || true if [[ -d "/proc/${PID}" ]]; then - info_log "stopping ${PROCESS}" + info_log "Stopping ${PROCESS}" kill "${PID}" || true # wait for process exit while [[ -e "/proc/${PID}" ]]; do sleep 0.1; done @@ -152,8 +152,6 @@ shutdown() { # switch to backend directory cd /backend || { error_log "/backend directory doesn't seem to exist"; } -info_log "Starting up, please wait..." - # setup trap handler exited=0 trap 'exited=1 && shutdown' SIGINT SIGTERM EXIT @@ -167,42 +165,44 @@ while ! ((exited)); do # and REDIS_HOST is not set (which would mean we're using an external Redis/Valkey) if [[ -z ${REDIS_HOST:=""} ]]; then watchdog_process_pid bin valkey-server + else + warn_log "REDIS_HOST is set, not starting internal valkey-server" fi # Run needed database migrations on startup, # but only if it was not successful since the last full docker container start if [[ ${ALEMBIC_SUCCESS:="false"} == "false" ]]; then if alembic upgrade head; then - debug_log "database schema migrations succeeded" + info_log "Database schema migrations succeeded" ALEMBIC_SUCCESS="true" else - error_log "Something went horribly wrong with our database" + error_log "Something went horribly wrong when running the database migrations" fi else - debug_log "database schema already upgraded during current container lifecycle" + debug_log "Database schema already upgraded during current container lifecycle" fi - # Start gunicorn if we dont have a corresponding PID file + # start backend if we dont have a corresponding PID file watchdog_process_pid bin gunicorn - # Start nginx if we dont have a corresponding PID file + # start nginx if we dont have a corresponding PID file watchdog_process_pid bin nginx - # Start background worker processes - debug_log "Starting worker" - # Start worker if we dont have a corresponding PID file + # start worker if we dont have a corresponding PID file + info_log "Starting worker" watchdog_process_pid python worker - # only start the scheduler.py if enabled + # only start the scheduler if enabled if [[ ${ENABLE_SCHEDULED_RESCAN} == "true" || ${ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB} == "true" ]]; then - debug_log "Starting scheduler" - # Start scheduler if we dont have a corresponding PID file + # start scheduler if we dont have a corresponding PID file + info_log "Starting scheduler" watchdog_process_pid python scheduler fi - # only start the watcher.py if enabled + # only start the watcher if enabled if [[ ${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE} == "true" ]]; then - # Start watcher if we dont have a corresponding PID file + # start watcher if we dont have a corresponding PID file + info_log "Starting watcher" watchdog_process_pid python watcher fi From 5d1114b8321f242b151ebba5c6128d7b749b69e4 Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 10:44:58 +0000 Subject: [PATCH 07/32] refactor: set default web concurrency to reduce resource usage --- docker/init_scripts/init | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 673ddcf48..f6e49cc2b 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -14,6 +14,9 @@ ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB="${ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB # (this env var is currently undocumented and usually just needed for development purposes) INIT_DEBUG="${INIT_DEBUG:="false"}" +# Set DEFAULT_WEB_CONCURRENCY to 1 if not set by docker env to reduce resource usage +DEFAULT_WEB_CONCURRENCY=1 + # print debug log output if enabled debug_log() { if [[ ${INIT_DEBUG} == "true" ]]; then @@ -64,7 +67,7 @@ start_bin_gunicorn() { --bind=unix:/tmp/gunicorn.sock \ --pid=/tmp/gunicorn.pid \ --forwarded-allow-ips="*" \ - --workers "${WEB_CONCURRENCY:-2}" \ + --workers "${WEB_CONCURRENCY:-${DEFAULT_WEB_CONCURRENCY:-1}}" \ main:app & } From ad1cd9e0c6e4421563653125eac706982503ab69 Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 11:31:46 +0000 Subject: [PATCH 08/32] refactor: replace info_debug with debug_log for consistency in logging --- docker/init_scripts/init | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/init_scripts/init b/docker/init_scripts/init index f6e49cc2b..5fe943c3c 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -38,11 +38,11 @@ error_log() { } wait_for_gunicorn_socket() { - info_debug "Waiting for gunicorn socket file..." + debug_log "Waiting for gunicorn socket file..." while [[ ! -S /tmp/gunicorn.sock ]]; do sleep 1 done - info_debug "Gunicorn socket file found" + debug_log "Gunicorn socket file found" } # function that runs or main process and creates a corresponding PID file, From c0f33545b1bf1ebbc891241a0d95bb4d9ac3c94e Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 14:04:21 +0000 Subject: [PATCH 09/32] refactor: enhance initialization script with improved logging and process management --- docker/init_scripts/init | 42 +++++++++++++++++----------------------- 1 file changed, 18 insertions(+), 24 deletions(-) diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 5fe943c3c..cf2490400 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -10,11 +10,15 @@ ENABLE_RESCAN_ON_FILESYSTEM_CHANGE="${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE:="false ENABLE_SCHEDULED_RESCAN="${ENABLE_SCHEDULED_RESCAN:="false"}" ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB="${ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB:="false"}" +# if REDIS_HOST is set, we assume that an external redis is used +REDIS_HOST="${REDIS_HOST:=""}" + # Set INIT_DEBUG to false if not set by docker env # (this env var is currently undocumented and usually just needed for development purposes) INIT_DEBUG="${INIT_DEBUG:="false"}" # Set DEFAULT_WEB_CONCURRENCY to 1 if not set by docker env to reduce resource usage +# (since backend is almost 100% async this won't block anything) DEFAULT_WEB_CONCURRENCY=1 # print debug log output if enabled @@ -84,6 +88,7 @@ start_bin_nginx() { fi } +# Commands to start valkey-server (handling PID creation internally) start_bin_valkey-server() { info_log "Starting internal valkey-server" # Check if /usr/local/etc/valkey/valkey.conf exists and use it if so @@ -121,6 +126,7 @@ watchdog_process_pid() { fi fi else + # start process if we dont have a corresponding PID file if [[ ${TYPE} == "bin" ]]; then start_bin_"${PROCESS}" elif [[ ${TYPE} == "python" ]]; then @@ -155,6 +161,15 @@ shutdown() { # switch to backend directory cd /backend || { error_log "/backend directory doesn't seem to exist"; } +# Run needed database migrations once at startup +info_log "Running database migrations" +if alembic upgrade head >>/tmp/alembic.log 2>&1; then + info_log "Database migrations succeeded" +else + cat /tmp/alembic.log + error_log "Failed to run database migrations" +fi + # setup trap handler exited=0 trap 'exited=1 && shutdown' SIGINT SIGTERM EXIT @@ -166,46 +181,25 @@ rm /tmp/*.pid -f while ! ((exited)); do # Start Valkey server if we dont have a corresponding PID file # and REDIS_HOST is not set (which would mean we're using an external Redis/Valkey) - if [[ -z ${REDIS_HOST:=""} ]]; then + if [[ -z ${REDIS_HOST} ]]; then watchdog_process_pid bin valkey-server else warn_log "REDIS_HOST is set, not starting internal valkey-server" fi - # Run needed database migrations on startup, - # but only if it was not successful since the last full docker container start - if [[ ${ALEMBIC_SUCCESS:="false"} == "false" ]]; then - if alembic upgrade head; then - info_log "Database schema migrations succeeded" - ALEMBIC_SUCCESS="true" - else - error_log "Something went horribly wrong when running the database migrations" - fi - else - debug_log "Database schema already upgraded during current container lifecycle" - fi - - # start backend if we dont have a corresponding PID file - watchdog_process_pid bin gunicorn - - # start nginx if we dont have a corresponding PID file watchdog_process_pid bin nginx - # start worker if we dont have a corresponding PID file - info_log "Starting worker" + watchdog_process_pid bin gunicorn + watchdog_process_pid python worker # only start the scheduler if enabled if [[ ${ENABLE_SCHEDULED_RESCAN} == "true" || ${ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB} == "true" ]]; then - # start scheduler if we dont have a corresponding PID file - info_log "Starting scheduler" watchdog_process_pid python scheduler fi # only start the watcher if enabled if [[ ${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE} == "true" ]]; then - # start watcher if we dont have a corresponding PID file - info_log "Starting watcher" watchdog_process_pid python watcher fi From 0f1c29be7314d14a1fd6aa21368c9d5408213727 Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 14:17:03 +0000 Subject: [PATCH 10/32] refactor: enhance logging format in init script with color coding for better visibility --- backend/logger/formatter.py | 37 +++++++++++++++++++++---------------- docker/init_scripts/init | 17 +++++++++++++---- 2 files changed, 34 insertions(+), 20 deletions(-) diff --git a/backend/logger/formatter.py b/backend/logger/formatter.py index fee4e816a..85d65dcce 100644 --- a/backend/logger/formatter.py +++ b/backend/logger/formatter.py @@ -5,10 +5,15 @@ from colorama import Fore, Style, init from config import FORCE_COLOR, NO_COLOR RED = Fore.RED +LIGHTRED = Fore.LIGHTRED_EX GREEN = Fore.GREEN LIGHTYELLOW = Fore.LIGHTYELLOW_EX YELLOW = Fore.YELLOW BLUE = Fore.BLUE +CYAN = Fore.CYAN +LIGHTMAGENTA = Fore.LIGHTMAGENTA_EX +RESET = Fore.RESET +RESET_ALL = Style.RESET_ALL def should_strip_ansi() -> bool: @@ -43,37 +48,37 @@ class Formatter(logging.Formatter): The formatted log record as a string. """ level = "%(levelname)s" - dots = f"{Fore.RESET}:" + dots = f"{RESET}:" identifier = ( - f"\t {Fore.BLUE}[RomM]{Fore.LIGHTMAGENTA_EX}[{record.module.lower()}]" + f"\t {BLUE}[RomM]{LIGHTMAGENTA}[{record.module.lower()}]" if hasattr(record, "module_name") - else f"\t {Fore.BLUE}[RomM]{Fore.LIGHTMAGENTA_EX}[%(module)s]" + else f"\t {BLUE}[RomM]{LIGHTMAGENTA}[%(module)s]" ) identifier_warning = ( - f" {Fore.BLUE}[RomM]{Fore.LIGHTMAGENTA_EX}[{record.module.lower()}]" + f" {BLUE}[RomM]{LIGHTMAGENTA}[{record.module.lower()}]" if hasattr(record, "module_name") - else f" {Fore.BLUE}[RomM]{Fore.LIGHTMAGENTA_EX}[%(module)s]" + else f" {BLUE}[RomM]{LIGHTMAGENTA}[%(module)s]" ) identifier_critical = ( - f" {Fore.BLUE}[RomM]{Fore.LIGHTMAGENTA_EX}[{record.module.lower()}]" + f" {BLUE}[RomM]{LIGHTMAGENTA}[{record.module.lower()}]" if hasattr(record, "module_name") - else f" {Fore.BLUE}[RomM]{Fore.LIGHTMAGENTA_EX}[%(module)s]" + else f" {BLUE}[RomM]{LIGHTMAGENTA}[%(module)s]" ) - msg = f"{Style.RESET_ALL}%(message)s" - date = f"{Fore.CYAN}[%(asctime)s] " + msg = f"{RESET_ALL}%(message)s" + date = f"{CYAN}[%(asctime)s] " formats = { - logging.DEBUG: f"{Fore.LIGHTMAGENTA_EX}{level}{dots}{identifier}{date}{msg}", - logging.INFO: f"{Fore.GREEN}{level}{dots}{identifier}{date}{msg}", - logging.WARNING: f"{Fore.YELLOW}{level}{dots}{identifier_warning}{date}{msg}", - logging.ERROR: f"{Fore.LIGHTRED_EX}{level}{dots}{identifier}{date}{msg}", - logging.CRITICAL: f"{Fore.RED}{level}{dots}{identifier_critical}{date}{msg}", + logging.DEBUG: f"{LIGHTMAGENTA}{level}{dots}{identifier}{date}{msg}", + logging.INFO: f"{GREEN}{level}{dots}{identifier}{date}{msg}", + logging.WARNING: f"{YELLOW}{level}{dots}{identifier_warning}{date}{msg}", + logging.ERROR: f"{LIGHTRED}{level}{dots}{identifier}{date}{msg}", + logging.CRITICAL: f"{RED}{level}{dots}{identifier_critical}{date}{msg}", } log_fmt = formats.get(record.levelno) formatter = logging.Formatter(fmt=log_fmt, datefmt="%Y-%m-%d %H:%M:%S") return formatter.format(record) -def highlight(msg: str = "", color=Fore.YELLOW) -> str: +def highlight(msg: str = "", color=YELLOW) -> str: """ Highlights the message to send to the fancylog. @@ -84,4 +89,4 @@ def highlight(msg: str = "", color=Fore.YELLOW) -> str: Returns: The highlighted message as a string. """ - return f"{color}{msg}{Style.RESET_ALL}" + return f"{color}{msg}{RESET_ALL}" diff --git a/docker/init_scripts/init b/docker/init_scripts/init index cf2490400..484c74388 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -21,23 +21,32 @@ INIT_DEBUG="${INIT_DEBUG:="false"}" # (since backend is almost 100% async this won't block anything) DEFAULT_WEB_CONCURRENCY=1 +# logger colors +RED='\033[0;31m' +LIGHTMAGENTA='\033[0;95m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +BLUE='\033[0;36m' +CYAN='\033[0;36m' +RESET='\033[0;00m' + # print debug log output if enabled debug_log() { if [[ ${INIT_DEBUG} == "true" ]]; then - echo "DEBUG: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true + echo -e "${LIGHTMAGENTA}DEBUG: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true fi } info_log() { - echo "INFO: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true + echo -e "${GREEN}INFO: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true } warn_log() { - echo "WARNING: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true + echo -e "${YELLOW}WARNING: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true } error_log() { - echo "ERROR: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true + echo -e "${RED}ERROR: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true exit 1 } From 4c953b643947e0a3f8d0068c4dee4f7abaf2a7cb Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 14:30:48 +0000 Subject: [PATCH 11/32] refactor: use symbolic links with force option in Dockerfile and improve branch name formatting in build script --- docker/Dockerfile | 4 ++-- docker/build_local_image.sh | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e60efbdaa..cd82754d2 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -146,8 +146,8 @@ COPY --from=frontend-build /front/dist ${WEBSERVER_FOLDER} COPY ./frontend/assets ${WEBSERVER_FOLDER}/assets RUN mkdir -p ${WEBSERVER_FOLDER}/assets/romm && \ - ln -s /romm/resources ${WEBSERVER_FOLDER}/assets/romm/resources && \ - ln -s /romm/assets ${WEBSERVER_FOLDER}/assets/romm/assets + ln -sf /romm/resources ${WEBSERVER_FOLDER}/assets/romm/resources && \ + ln -sf /romm/assets ${WEBSERVER_FOLDER}/assets/romm/assets COPY ./backend /backend diff --git a/docker/build_local_image.sh b/docker/build_local_image.sh index 21805abd8..6c4b80141 100755 --- a/docker/build_local_image.sh +++ b/docker/build_local_image.sh @@ -2,4 +2,5 @@ branch_name="$(git symbolic-ref HEAD 2>/dev/null)" branch_name=${branch_name##refs/heads/} +branch_name=${branch_name//\//-} # Replace slashes with dashes docker build -t "rommapp/romm:local-${branch_name}" . --file ./docker/Dockerfile From c01a90caf6fb7fff7fff119f9224928b1742af98 Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 14:36:32 +0000 Subject: [PATCH 12/32] refactor: update Docker image tag to include 'testing' for local builds --- docker/build_local_image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build_local_image.sh b/docker/build_local_image.sh index 6c4b80141..c7faf3cc4 100755 --- a/docker/build_local_image.sh +++ b/docker/build_local_image.sh @@ -3,4 +3,4 @@ branch_name="$(git symbolic-ref HEAD 2>/dev/null)" branch_name=${branch_name##refs/heads/} branch_name=${branch_name//\//-} # Replace slashes with dashes -docker build -t "rommapp/romm:local-${branch_name}" . --file ./docker/Dockerfile +docker build -t "rommapp/romm-testing:local-${branch_name}" . --file ./docker/Dockerfile From d5bb63c06f5e6c4a5cd1eb9e5fe0d40b9fce4287 Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 16:34:23 +0000 Subject: [PATCH 13/32] refactor: improve logging in redis_handler and init scripts for better debugging and process management --- backend/handler/redis_handler.py | 6 +- docker/init_scripts/init | 108 ++++++++++++++++++++----------- 2 files changed, 73 insertions(+), 41 deletions(-) diff --git a/backend/handler/redis_handler.py b/backend/handler/redis_handler.py index 7eb520f79..22bfea2df 100644 --- a/backend/handler/redis_handler.py +++ b/backend/handler/redis_handler.py @@ -28,10 +28,9 @@ def __get_sync_cache() -> Redis: return FakeRedis(version=7) - log.info(f"Connecting to sync redis in {sys.argv[0]}...") # A separate client that auto-decodes responses is needed client = Redis.from_url(str(REDIS_URL), decode_responses=True) - log.info(f"Redis sync connection established in {sys.argv[0]}!") + log.debug(f"Sync redis/valkey connection established in {sys.argv[0]}") return client @@ -42,10 +41,9 @@ def __get_async_cache() -> AsyncRedis: return FakeAsyncRedis(version=7) - log.info(f"Connecting to async redis in {sys.argv[0]}...") # A separate client that auto-decodes responses is needed client = AsyncRedis.from_url(str(REDIS_URL), decode_responses=True) - log.info(f"Redis async connection established in {sys.argv[0]}!") + log.debug(f"Async redis/valkey connection established in {sys.argv[0]}") return client diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 484c74388..c5e14a595 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -5,6 +5,8 @@ set -o nounset # treat unset variables as an error set -o pipefail # treat errors in pipes as fatal shopt -s inherit_errexit # inherit errexit +LOGLEVEL="${LOGLEVEL:="info"}" + # make it possible to disable the inotify watcher process ENABLE_RESCAN_ON_FILESYSTEM_CHANGE="${ENABLE_RESCAN_ON_FILESYSTEM_CHANGE:="false"}" ENABLE_SCHEDULED_RESCAN="${ENABLE_SCHEDULED_RESCAN:="false"}" @@ -13,11 +15,7 @@ ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB="${ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB # if REDIS_HOST is set, we assume that an external redis is used REDIS_HOST="${REDIS_HOST:=""}" -# Set INIT_DEBUG to false if not set by docker env -# (this env var is currently undocumented and usually just needed for development purposes) -INIT_DEBUG="${INIT_DEBUG:="false"}" - -# Set DEFAULT_WEB_CONCURRENCY to 1 if not set by docker env to reduce resource usage +# set DEFAULT_WEB_CONCURRENCY to 1 if not set by docker env to reduce resource usage # (since backend is almost 100% async this won't block anything) DEFAULT_WEB_CONCURRENCY=1 @@ -26,13 +24,13 @@ RED='\033[0;31m' LIGHTMAGENTA='\033[0;95m' GREEN='\033[0;32m' YELLOW='\033[0;33m' -BLUE='\033[0;36m' +BLUE='\033[0;34m' CYAN='\033[0;36m' RESET='\033[0;00m' # print debug log output if enabled debug_log() { - if [[ ${INIT_DEBUG} == "true" ]]; then + if [[ ${LOGLEVEL} == "debug" ]]; then echo -e "${LIGHTMAGENTA}DEBUG: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true fi } @@ -52,10 +50,17 @@ error_log() { wait_for_gunicorn_socket() { debug_log "Waiting for gunicorn socket file..." - while [[ ! -S /tmp/gunicorn.sock ]]; do - sleep 1 + local retries=60 + while [[ ! -S /tmp/gunicorn.sock && retries -gt 0 ]]; do + sleep 0.5 + ((retries--)) done - debug_log "Gunicorn socket file found" + + if [[ -S /tmp/gunicorn.sock ]]; then + debug_log "Gunicorn socket file found" + else + error_log "Gunicorn socket file not found after waiting 30s" + fi } # function that runs or main process and creates a corresponding PID file, @@ -73,8 +78,8 @@ start_bin_gunicorn() { fi gunicorn \ - --access-logfile - \ - --error-logfile - \ + --error-logfile /tmp/gunicorn_access.log \ + --error-logfile /tmp/gunicorn_error.log \ --worker-class uvicorn.workers.UvicornWorker \ --bind=0.0.0.0:5000 \ --bind=unix:/tmp/gunicorn.sock \ @@ -100,14 +105,47 @@ start_bin_nginx() { # Commands to start valkey-server (handling PID creation internally) start_bin_valkey-server() { info_log "Starting internal valkey-server" - # Check if /usr/local/etc/valkey/valkey.conf exists and use it if so + if [[ -f /usr/local/etc/valkey/valkey.conf ]]; then - valkey-server /usr/local/etc/valkey/valkey.conf & + if [[ ${LOGLEVEL} == "debug" ]]; then + valkey-server /usr/local/etc/valkey/valkey.conf & + else + valkey-server /usr/local/etc/valkey/valkey.conf >/dev/null 2>&1 & + fi else - valkey-server --dir /redis-data & + if [[ ${LOGLEVEL} == "debug" ]]; then + valkey-server --dir /redis-data & + else + valkey-server --dir /redis-data >/dev/null 2>&1 & + fi fi + VALKEY_PID=$! echo "${VALKEY_PID}" >/tmp/valkey-server.pid + + local host="127.0.0.1" + local port="6379" + local max_retries=120 + local retry=0 + + debug_log "Waiting for internal valkey-server to be ready..." + + # Temporarily disable errexit for this part of the script + set +o errexit + + while ((retry < max_retries)); do + # Attempt to check if valkey TCP port is open + if (echo >/dev/tcp/"${host}"/"${port}") 2>/dev/null; then + debug_log "Internal valkey-server is ready and accepting connections" + set -o errexit # Re-enable errexit after success + return 0 + fi + + sleep 0.5 + ((retry++)) + done + + error_log "Internal valkey-server did not become ready after $((max_retries * 500))ms" } # function that runs our independent python scripts and creates corresponding PID files, @@ -125,9 +163,7 @@ watchdog_process_pid() { if [[ -f "/tmp/${PROCESS}.pid" ]]; then # check if the pid we last wrote to our state file is actually active PID=$(cat "/tmp/${PROCESS}.pid") || true - if [[ -d "/proc/${PID}" ]]; then - debug_log "${PROCESS} still running, no need to start" - else + if [[ ! -d "/proc/${PID}" ]]; then if [[ ${TYPE} == "bin" ]]; then start_bin_"${PROCESS}" elif [[ ${TYPE} == "python" ]]; then @@ -170,15 +206,6 @@ shutdown() { # switch to backend directory cd /backend || { error_log "/backend directory doesn't seem to exist"; } -# Run needed database migrations once at startup -info_log "Running database migrations" -if alembic upgrade head >>/tmp/alembic.log 2>&1; then - info_log "Database migrations succeeded" -else - cat /tmp/alembic.log - error_log "Failed to run database migrations" -fi - # setup trap handler exited=0 trap 'exited=1 && shutdown' SIGINT SIGTERM EXIT @@ -186,20 +213,27 @@ trap 'exited=1 && shutdown' SIGINT SIGTERM EXIT # clear any leftover PID files rm /tmp/*.pid -f -# function definition done, lets start our main loop +# Start Valkey server if REDIS_HOST is not set (which would mean user is using an external Redis/Valkey) +if [[ -z ${REDIS_HOST} ]]; then + watchdog_process_pid bin valkey-server +else + warn_log "REDIS_HOST is set, not starting internal valkey-server" +fi + +# Run needed database migrations once at startup +info_log "Running database migrations" +if alembic upgrade head; then + info_log "Database migrations succeeded" +else + error_log "Failed to run database migrations" +fi + +# main loop while ! ((exited)); do - # Start Valkey server if we dont have a corresponding PID file - # and REDIS_HOST is not set (which would mean we're using an external Redis/Valkey) - if [[ -z ${REDIS_HOST} ]]; then - watchdog_process_pid bin valkey-server - else - warn_log "REDIS_HOST is set, not starting internal valkey-server" - fi + watchdog_process_pid bin gunicorn watchdog_process_pid bin nginx - watchdog_process_pid bin gunicorn - watchdog_process_pid python worker # only start the scheduler if enabled From 5bc820d3ee71cb75d9785709d8ae766bb32d5489 Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 17:22:30 +0000 Subject: [PATCH 14/32] refactor: enhance logging and user agent mapping in nginx configuration for improved monitoring --- docker/init_scripts/init | 15 ++++++++------- docker/nginx/default.conf | 31 +++++++++++++++++++++++++------ 2 files changed, 33 insertions(+), 13 deletions(-) diff --git a/docker/init_scripts/init b/docker/init_scripts/init index c5e14a595..4e97a2672 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -93,18 +93,19 @@ start_bin_gunicorn() { start_bin_nginx() { wait_for_gunicorn_socket - info_log "Starting internal nginx" + info_log "Starting nginx" if [[ ${EUID} -ne 0 ]]; then nginx else # if container runs as root, drop permissions nginx -g 'user romm;' fi + info_log "🚀 RomM is now available at http://[::1]:8080" } # Commands to start valkey-server (handling PID creation internally) start_bin_valkey-server() { - info_log "Starting internal valkey-server" + info_log "Starting internal valkey" if [[ -f /usr/local/etc/valkey/valkey.conf ]]; then if [[ ${LOGLEVEL} == "debug" ]]; then @@ -128,7 +129,7 @@ start_bin_valkey-server() { local max_retries=120 local retry=0 - debug_log "Waiting for internal valkey-server to be ready..." + debug_log "Waiting for internal valkey to be ready..." # Temporarily disable errexit for this part of the script set +o errexit @@ -136,7 +137,7 @@ start_bin_valkey-server() { while ((retry < max_retries)); do # Attempt to check if valkey TCP port is open if (echo >/dev/tcp/"${host}"/"${port}") 2>/dev/null; then - debug_log "Internal valkey-server is ready and accepting connections" + debug_log "Internal valkey is ready and accepting connections" set -o errexit # Re-enable errexit after success return 0 fi @@ -145,7 +146,7 @@ start_bin_valkey-server() { ((retry++)) done - error_log "Internal valkey-server did not become ready after $((max_retries * 500))ms" + error_log "Internal valkey did not become ready after $((max_retries * 500))ms" } # function that runs our independent python scripts and creates corresponding PID files, @@ -232,8 +233,6 @@ fi while ! ((exited)); do watchdog_process_pid bin gunicorn - watchdog_process_pid bin nginx - watchdog_process_pid python worker # only start the scheduler if enabled @@ -246,6 +245,8 @@ while ! ((exited)); do watchdog_process_pid python watcher fi + watchdog_process_pid bin nginx + # check for died processes every 5 seconds sleep 5 done diff --git a/docker/nginx/default.conf b/docker/nginx/default.conf index 4d487cd21..d27fa1924 100644 --- a/docker/nginx/default.conf +++ b/docker/nginx/default.conf @@ -42,13 +42,32 @@ http { ~T([0-9:]+)\+ $1; } - #INFO: [nginx][2023-11-14 09:20:29] 127.0.0.1 - -"GET / HTTP/1.1" 500 177 "-" "Mozilla/5.0 (X11; Linux x86_64)"rt=0.000 uct="-" uht="-" urt="-" - log_format romm_log 'INFO: [nginx][$date $time] $remote_addr - $remote_user ' - '"$request" $status $body_bytes_sent ' - '"$http_referer" "$http_user_agent" ' - 'rt=$request_time uct="$upstream_connect_time" uht="$upstream_header_time" urt="$upstream_response_time"'; + # Map to extract the browser name (e.g., Chrome, Firefox, etc.) + map $http_user_agent $browser { + default "Unknown"; + "~Chrome/" "Chrome"; + "~Firefox/" "Firefox"; + "~Safari/" "Safari"; + "~Edge/" "Edge"; + "~Opera/" "Opera"; + } - access_log /dev/stdout romm_log; + # Map to extract the OS (e.g., Windows, MacOS, Linux) + map $http_user_agent $os { + default "Unknown"; + "~Windows NT" "Windows"; + "~Macintosh" "macOS"; + "~Linux" "Linux"; + "~Android" "Android"; + "~iPhone" "iOS"; + } + + #INFO: [nginx][2023-11-14 09:20:29] 127.0.0.1 - -"GET / HTTP/1.1" 500 177 "-" "Mozilla/5.0 (X11; Linux x86_64)"rt=0.000 uct="-" uht="-" urt="-" + log_format romm_logs 'INFO: [RomM][nginx][$date $time] $remote_addr | $http_x_forwarded_for | ' + '$request_method $server_protocol $request_uri $status $body_bytes_sent | ' + '$browser $os | $request_time'; + + access_log /dev/stdout romm_logs; error_log /dev/stderr; gzip on; From cac960297455d796be231570ca2f9091f4070bc4 Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 17:57:40 +0000 Subject: [PATCH 15/32] refactor: streamline logging setup and improve log formatting across multiple modules --- backend/logger/logger.py | 5 ----- backend/scheduler.py | 11 +++++++---- backend/watcher.py | 6 ++++++ backend/worker.py | 8 +++++++- docker/init_scripts/init | 17 ++++------------- 5 files changed, 24 insertions(+), 23 deletions(-) diff --git a/backend/logger/logger.py b/backend/logger/logger.py index c14049f33..b437b2657 100644 --- a/backend/logger/logger.py +++ b/backend/logger/logger.py @@ -8,15 +8,10 @@ from logger.formatter import Formatter log = logging.getLogger("romm") log.setLevel(LOGLEVEL) -# Set up sqlachemy logger -# sql_log = logging.getLogger("sqlalchemy.engine") -# sql_log.setLevel(LOGLEVEL) - # Define stdout handler stdout_handler = logging.StreamHandler(sys.stdout) stdout_handler.setFormatter(Formatter()) log.addHandler(stdout_handler) -# sql_log.addHandler(stdout_handler) # Hush passlib warnings logging.getLogger("passlib").setLevel(logging.ERROR) diff --git a/backend/scheduler.py b/backend/scheduler.py index 2bfa37c52..6d7b5eaaf 100644 --- a/backend/scheduler.py +++ b/backend/scheduler.py @@ -1,10 +1,12 @@ +import logging + import sentry_sdk from config import ( ENABLE_SCHEDULED_RESCAN, ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, SENTRY_DSN, ) -from logger.logger import log +from logger.formatter import BLUE, CYAN, GREEN, LIGHTMAGENTA, RESET, RESET_ALL from tasks.scan_library import scan_library_task from tasks.tasks import tasks_scheduler from tasks.update_switch_titledb import update_switch_titledb_task @@ -12,14 +14,15 @@ from utils import get_version sentry_sdk.init(dsn=SENTRY_DSN, release=f"romm@{get_version()}") +# Set up custom logging +log_format = f"{GREEN}INFO{RESET}:\t {BLUE}[RomM]{LIGHTMAGENTA}[%(module)s]{CYAN}[%(asctime)s] {RESET_ALL}%(message)s" +logging.basicConfig(format=log_format, datefmt="%Y-%m-%d %H:%M:%S") + if __name__ == "__main__": # Initialize the tasks if ENABLE_SCHEDULED_RESCAN: scan_library_task.init() if ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: update_switch_titledb_task.init() - - log.info("Starting scheduler") - # Start the scheduler tasks_scheduler.run() diff --git a/backend/watcher.py b/backend/watcher.py index df071a1cb..0de678eed 100644 --- a/backend/watcher.py +++ b/backend/watcher.py @@ -1,3 +1,4 @@ +import logging import os from datetime import timedelta @@ -12,6 +13,7 @@ from config.config_manager import config_manager as cm from endpoints.sockets.scan import scan_platforms from handler.database import db_platform_handler from handler.scan_handler import ScanType +from logger.formatter import BLUE, CYAN, GREEN, LIGHTMAGENTA, RESET, RESET_ALL from logger.logger import log from rq.job import Job from tasks.tasks import tasks_scheduler @@ -30,6 +32,10 @@ path = ( else LIBRARY_BASE_PATH ) +# Set up custom logging +log_format = f"{GREEN}INFO{RESET}:\t {BLUE}[RomM]{LIGHTMAGENTA}[%(module)s]{CYAN}[%(asctime)s] {RESET_ALL}%(message)s" +logging.basicConfig(format=log_format, datefmt="%Y-%m-%d %H:%M:%S") + class EventHandler(FileSystemEventHandler): """Filesystem event handler""" diff --git a/backend/worker.py b/backend/worker.py index 73d2c66ce..b31512b0e 100644 --- a/backend/worker.py +++ b/backend/worker.py @@ -1,6 +1,9 @@ +import logging + import sentry_sdk from config import SENTRY_DSN from handler.redis_handler import redis_client +from logger.formatter import BLUE, CYAN, GREEN, LIGHTMAGENTA, RESET, RESET_ALL from rq import Queue, Worker from utils import get_version @@ -8,9 +11,12 @@ listen = ("high", "default", "low") sentry_sdk.init( dsn=SENTRY_DSN, - release="romm@" + get_version(), + release=f"romm@{get_version()}", ) +# Set up custom logging +log_format = f"{GREEN}INFO{RESET}:\t {BLUE}[RomM]{LIGHTMAGENTA}[%(module)s]{CYAN}[%(asctime)s] {RESET_ALL}%(message)s" +logging.basicConfig(format=log_format, datefmt="%Y-%m-%d %H:%M:%S") if __name__ == "__main__": # Start the worker diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 4e97a2672..929c5f2c9 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -19,32 +19,23 @@ REDIS_HOST="${REDIS_HOST:=""}" # (since backend is almost 100% async this won't block anything) DEFAULT_WEB_CONCURRENCY=1 -# logger colors -RED='\033[0;31m' -LIGHTMAGENTA='\033[0;95m' -GREEN='\033[0;32m' -YELLOW='\033[0;33m' -BLUE='\033[0;34m' -CYAN='\033[0;36m' -RESET='\033[0;00m' - # print debug log output if enabled debug_log() { if [[ ${LOGLEVEL} == "debug" ]]; then - echo -e "${LIGHTMAGENTA}DEBUG: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true + echo "DEBUG: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true fi } info_log() { - echo -e "${GREEN}INFO: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true + echo "INFO: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true } warn_log() { - echo -e "${YELLOW}WARNING: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true + echo "WARNING: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true } error_log() { - echo -e "${RED}ERROR: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true + echo "ERROR: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true exit 1 } From c5033d9967d3e5fa8c6f1584abf0e5a4cf02444d Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 17:59:01 +0000 Subject: [PATCH 16/32] refactor: enhance logging output with color formatting for improved readability --- docker/init_scripts/init | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 929c5f2c9..a64f1a2d6 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -19,23 +19,33 @@ REDIS_HOST="${REDIS_HOST:=""}" # (since backend is almost 100% async this won't block anything) DEFAULT_WEB_CONCURRENCY=1 +# TODO: disable colors for non-TTY terminal +# logger colors +RED='\033[0;31m' +LIGHTMAGENTA='\033[0;95m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +RESET='\033[0;00m' + # print debug log output if enabled debug_log() { if [[ ${LOGLEVEL} == "debug" ]]; then - echo "DEBUG: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true + echo -e "${LIGHTMAGENTA}DEBUG: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true fi } info_log() { - echo "INFO: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true + echo -e "${GREEN}INFO: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true } warn_log() { - echo "WARNING: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true + echo -e "${YELLOW}WARNING: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true } error_log() { - echo "ERROR: [RomM][init][$(date +"%Y-%m-%d %T")]" "${@}" || true + echo -e "${RED}ERROR: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true exit 1 } From b3d341bc11216c29060086d1a29043977843d98c Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 17:59:50 +0000 Subject: [PATCH 17/32] refactor: add TODO comment to check on database disconnection in init script --- docker/init_scripts/init | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/init_scripts/init b/docker/init_scripts/init index a64f1a2d6..ce4b41f21 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -49,6 +49,8 @@ error_log() { exit 1 } +# TODO: check on database disconnection + wait_for_gunicorn_socket() { debug_log "Waiting for gunicorn socket file..." local retries=60 From 49ad99d5e7790e4e0181421a16b3e388be079bf4 Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 22:17:53 +0000 Subject: [PATCH 18/32] refactor: improve logging setup and format across multiple modules for consistency --- backend/logger/formatter.py | 5 ++++- backend/logger/logger.py | 7 ++++--- backend/scheduler.py | 10 +++------- backend/watcher.py | 8 +------- backend/worker.py | 7 +++---- docker/nginx/default.conf | 2 +- 6 files changed, 16 insertions(+), 23 deletions(-) diff --git a/backend/logger/formatter.py b/backend/logger/formatter.py index 85d65dcce..4c7652937 100644 --- a/backend/logger/formatter.py +++ b/backend/logger/formatter.py @@ -15,6 +15,9 @@ LIGHTMAGENTA = Fore.LIGHTMAGENTA_EX RESET = Fore.RESET RESET_ALL = Style.RESET_ALL +common_log_format = f"{GREEN}INFO{RESET}:\t {BLUE}[RomM]{LIGHTMAGENTA}[worker]{CYAN}[%(asctime)s] {RESET_ALL}%(message)s" +common_date_format = "%Y-%m-%d %H:%M:%S" + def should_strip_ansi() -> bool: """Determine if ANSI escape codes should be stripped.""" @@ -74,7 +77,7 @@ class Formatter(logging.Formatter): logging.CRITICAL: f"{RED}{level}{dots}{identifier_critical}{date}{msg}", } log_fmt = formats.get(record.levelno) - formatter = logging.Formatter(fmt=log_fmt, datefmt="%Y-%m-%d %H:%M:%S") + formatter = logging.Formatter(fmt=log_fmt, datefmt=common_date_format) return formatter.format(record) diff --git a/backend/logger/logger.py b/backend/logger/logger.py index b437b2657..c63189366 100644 --- a/backend/logger/logger.py +++ b/backend/logger/logger.py @@ -9,9 +9,10 @@ log = logging.getLogger("romm") log.setLevel(LOGLEVEL) # Define stdout handler -stdout_handler = logging.StreamHandler(sys.stdout) -stdout_handler.setFormatter(Formatter()) -log.addHandler(stdout_handler) +if not log.hasHandlers(): + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setFormatter(Formatter()) + log.addHandler(stdout_handler) # Hush passlib warnings logging.getLogger("passlib").setLevel(logging.ERROR) diff --git a/backend/scheduler.py b/backend/scheduler.py index 6d7b5eaaf..ec9385abd 100644 --- a/backend/scheduler.py +++ b/backend/scheduler.py @@ -1,12 +1,10 @@ -import logging - import sentry_sdk from config import ( ENABLE_SCHEDULED_RESCAN, ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, SENTRY_DSN, ) -from logger.formatter import BLUE, CYAN, GREEN, LIGHTMAGENTA, RESET, RESET_ALL +from logger.logger import log from tasks.scan_library import scan_library_task from tasks.tasks import tasks_scheduler from tasks.update_switch_titledb import update_switch_titledb_task @@ -14,15 +12,13 @@ from utils import get_version sentry_sdk.init(dsn=SENTRY_DSN, release=f"romm@{get_version()}") -# Set up custom logging -log_format = f"{GREEN}INFO{RESET}:\t {BLUE}[RomM]{LIGHTMAGENTA}[%(module)s]{CYAN}[%(asctime)s] {RESET_ALL}%(message)s" -logging.basicConfig(format=log_format, datefmt="%Y-%m-%d %H:%M:%S") - if __name__ == "__main__": # Initialize the tasks if ENABLE_SCHEDULED_RESCAN: + log.info("Starting scheduled rescan") scan_library_task.init() if ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB: + log.info("Starting scheduled update switch titledb") update_switch_titledb_task.init() # Start the scheduler tasks_scheduler.run() diff --git a/backend/watcher.py b/backend/watcher.py index 0de678eed..7c18f4c56 100644 --- a/backend/watcher.py +++ b/backend/watcher.py @@ -1,4 +1,3 @@ -import logging import os from datetime import timedelta @@ -13,7 +12,6 @@ from config.config_manager import config_manager as cm from endpoints.sockets.scan import scan_platforms from handler.database import db_platform_handler from handler.scan_handler import ScanType -from logger.formatter import BLUE, CYAN, GREEN, LIGHTMAGENTA, RESET, RESET_ALL from logger.logger import log from rq.job import Job from tasks.tasks import tasks_scheduler @@ -23,7 +21,7 @@ from watchdog.observers import Observer sentry_sdk.init( dsn=SENTRY_DSN, - release="romm@" + get_version(), + release=f"romm@{get_version()}", ) path = ( @@ -32,10 +30,6 @@ path = ( else LIBRARY_BASE_PATH ) -# Set up custom logging -log_format = f"{GREEN}INFO{RESET}:\t {BLUE}[RomM]{LIGHTMAGENTA}[%(module)s]{CYAN}[%(asctime)s] {RESET_ALL}%(message)s" -logging.basicConfig(format=log_format, datefmt="%Y-%m-%d %H:%M:%S") - class EventHandler(FileSystemEventHandler): """Filesystem event handler""" diff --git a/backend/worker.py b/backend/worker.py index b31512b0e..d890e54a3 100644 --- a/backend/worker.py +++ b/backend/worker.py @@ -3,7 +3,7 @@ import logging import sentry_sdk from config import SENTRY_DSN from handler.redis_handler import redis_client -from logger.formatter import BLUE, CYAN, GREEN, LIGHTMAGENTA, RESET, RESET_ALL +from logger.formatter import common_date_format, common_log_format from rq import Queue, Worker from utils import get_version @@ -14,9 +14,8 @@ sentry_sdk.init( release=f"romm@{get_version()}", ) -# Set up custom logging -log_format = f"{GREEN}INFO{RESET}:\t {BLUE}[RomM]{LIGHTMAGENTA}[%(module)s]{CYAN}[%(asctime)s] {RESET_ALL}%(message)s" -logging.basicConfig(format=log_format, datefmt="%Y-%m-%d %H:%M:%S") +# Set up custom logging for Worker logging +logging.basicConfig(format=common_log_format, datefmt=common_date_format) if __name__ == "__main__": # Start the worker diff --git a/docker/nginx/default.conf b/docker/nginx/default.conf index d27fa1924..5e554f9c5 100644 --- a/docker/nginx/default.conf +++ b/docker/nginx/default.conf @@ -64,7 +64,7 @@ http { #INFO: [nginx][2023-11-14 09:20:29] 127.0.0.1 - -"GET / HTTP/1.1" 500 177 "-" "Mozilla/5.0 (X11; Linux x86_64)"rt=0.000 uct="-" uht="-" urt="-" log_format romm_logs 'INFO: [RomM][nginx][$date $time] $remote_addr | $http_x_forwarded_for | ' - '$request_method $server_protocol $request_uri $status $body_bytes_sent | ' + '$request_method $request_uri | $status | $body_bytes_sent | ' '$browser $os | $request_time'; access_log /dev/stdout romm_logs; From 53f938620826b3e5dff33773ee2997c90f9fed53 Mon Sep 17 00:00:00 2001 From: zurdi Date: Thu, 8 May 2025 23:35:33 +0000 Subject: [PATCH 19/32] refactor: enhance logging messages and formatting for improved clarity and consistency --- backend/endpoints/search.py | 10 ++++++++-- backend/endpoints/sockets/scan.py | 10 +++++----- backend/handler/redis_handler.py | 9 +++++++-- backend/handler/scan_handler.py | 4 ++-- backend/logger/formatter.py | 6 ++---- backend/worker.py | 3 ++- docker/init_scripts/init | 1 - docker/nginx/default.conf | 7 ++++--- 8 files changed, 30 insertions(+), 20 deletions(-) diff --git a/backend/endpoints/search.py b/backend/endpoints/search.py index 6dc15bfc0..e8cfea6a6 100644 --- a/backend/endpoints/search.py +++ b/backend/endpoints/search.py @@ -18,6 +18,8 @@ from handler.metadata.moby_handler import MOBY_API_ENABLED, MobyGamesRom from handler.metadata.sgdb_handler import STEAMGRIDDB_API_ENABLED from handler.metadata.ss_handler import SS_API_ENABLED, SSRom from handler.scan_handler import _get_main_platform_igdb_id +from logger.formatter import BLUE +from logger.formatter import highlight as hl from logger.logger import log from utils.router import APIRouter @@ -68,8 +70,12 @@ async def search_rom( ) matched_roms: list = [] - log.info(f"Searching by {search_by.lower()}: {search_term}") - log.info(emoji.emojize(f":video_game: {rom.platform_slug}: {rom.fs_name}")) + log.info(f"Searching by {search_by.lower()}:") + log.info( + emoji.emojize( + f":video_game: {hl(rom.platform_display_name, color=BLUE)}[{rom.platform_fs_slug}]: {hl(search_term)}[{rom.fs_name}]" + ) + ) igdb_matched_roms: list[IGDBRom] = [] moby_matched_roms: list[MobyGamesRom] = [] diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index d87525d84..57d9fb55e 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -181,7 +181,7 @@ async def scan_platforms( if len(platform_list) == 0: log.warning( emoji.emojize( - f"{hl(':warning:', color=LIGHTYELLOW)} No platforms found, verify that the folder structure is right and the volume is mounted correctly. \ + f"{hl(':warning:', color=LIGHTYELLOW)} No platforms found, verify that the folder structure is right and the volume is mounted correctly. \ Check https://github.com/rommapp/romm?tab=readme-ov-file#folder-structure for more details." ) ) @@ -208,7 +208,7 @@ async def scan_platforms( for p in purged_platforms: log.info(f" - {p.slug}") - log.info(emoji.emojize(":check_mark: Scan completed ")) + log.info(emoji.emojize(" :check_mark: Scan completed ")) await sm.emit("scan:done", scan_stats.__dict__) except ScanStoppedException: await stop_scan() @@ -273,7 +273,7 @@ async def _identify_platform( if len(fs_firmware) == 0: log.warning( emoji.emojize( - f" {hl(':warning:', color=LIGHTYELLOW)} No firmware found, skipping firmware scan for this platform" + f" {hl(':warning:', color=LIGHTYELLOW)} No firmware found, skipping firmware scan for this platform" ) ) else: @@ -295,7 +295,7 @@ async def _identify_platform( if len(fs_roms) == 0: log.warning( emoji.emojize( - f" {hl(':warning:', color=LIGHTYELLOW)} No roms found, verify that the folder structure is correct" + f" {hl(':warning:', color=LIGHTYELLOW)} No roms found, verify that the folder structure is correct" ) ) else: @@ -539,7 +539,7 @@ async def scan_handler(_sid: str, options: dict): options (dict): Socket options """ - log.info(emoji.emojize(":magnifying_glass_tilted_right: Scanning ")) + log.info(emoji.emojize(":magnifying_glass_tilted_right: Scanning")) platform_ids = options.get("platforms", []) scan_type = ScanType[options.get("type", "quick").upper()] diff --git a/backend/handler/redis_handler.py b/backend/handler/redis_handler.py index 22bfea2df..423e680dc 100644 --- a/backend/handler/redis_handler.py +++ b/backend/handler/redis_handler.py @@ -1,3 +1,4 @@ +import os import sys from enum import Enum @@ -30,7 +31,9 @@ def __get_sync_cache() -> Redis: # A separate client that auto-decodes responses is needed client = Redis.from_url(str(REDIS_URL), decode_responses=True) - log.debug(f"Sync redis/valkey connection established in {sys.argv[0]}") + log.debug( + f"Sync redis/valkey connection established in {os.path.splitext(os.path.basename(sys.argv[0]))[0]}" + ) return client @@ -43,7 +46,9 @@ def __get_async_cache() -> AsyncRedis: # A separate client that auto-decodes responses is needed client = AsyncRedis.from_url(str(REDIS_URL), decode_responses=True) - log.debug(f"Async redis/valkey connection established in {sys.argv[0]}") + log.debug( + f"Async redis/valkey connection established in {os.path.splitext(os.path.basename(sys.argv[0]))[0]}" + ) return client diff --git a/backend/handler/scan_handler.py b/backend/handler/scan_handler.py index 9385234ab..482a3d166 100644 --- a/backend/handler/scan_handler.py +++ b/backend/handler/scan_handler.py @@ -68,7 +68,7 @@ async def scan_platform( Platform object """ - log.info(f"· {hl(fs_slug)}") + log.info(f"· Found {hl(fs_slug)} folder") if metadata_sources is None: metadata_sources = [MetadataSource.IGDB, MetadataSource.MOBY, MetadataSource.SS] @@ -138,7 +138,7 @@ async def scan_platform( else: log.warning( emoji.emojize( - f" Platform {platform_attrs['slug']} not identified :cross_mark:" + f" Platform {platform_attrs['slug']} not identified :cross_mark:" ) ) diff --git a/backend/logger/formatter.py b/backend/logger/formatter.py index 4c7652937..3910be5d0 100644 --- a/backend/logger/formatter.py +++ b/backend/logger/formatter.py @@ -1,5 +1,4 @@ import logging -import os from colorama import Fore, Style, init from config import FORCE_COLOR, NO_COLOR @@ -26,9 +25,8 @@ def should_strip_ansi() -> bool: return False if NO_COLOR: return True - - # For other environments, strip colors if not a TTY - return not os.isatty(1) + # Default: do not strip (Docker will handle colors) + return False # Initialize Colorama once, considering different environments diff --git a/backend/worker.py b/backend/worker.py index d890e54a3..ab12b40be 100644 --- a/backend/worker.py +++ b/backend/worker.py @@ -14,8 +14,9 @@ sentry_sdk.init( release=f"romm@{get_version()}", ) +# TODO: setup custom logger for background workers # Set up custom logging for Worker logging -logging.basicConfig(format=common_log_format, datefmt=common_date_format) +# logging.basicConfig(format=common_log_format, datefmt=common_date_format) if __name__ == "__main__": # Start the worker diff --git a/docker/init_scripts/init b/docker/init_scripts/init index ce4b41f21..0b69eeab2 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -19,7 +19,6 @@ REDIS_HOST="${REDIS_HOST:=""}" # (since backend is almost 100% async this won't block anything) DEFAULT_WEB_CONCURRENCY=1 -# TODO: disable colors for non-TTY terminal # logger colors RED='\033[0;31m' LIGHTMAGENTA='\033[0;95m' diff --git a/docker/nginx/default.conf b/docker/nginx/default.conf index 5e554f9c5..6ba62366c 100644 --- a/docker/nginx/default.conf +++ b/docker/nginx/default.conf @@ -63,9 +63,10 @@ http { } #INFO: [nginx][2023-11-14 09:20:29] 127.0.0.1 - -"GET / HTTP/1.1" 500 177 "-" "Mozilla/5.0 (X11; Linux x86_64)"rt=0.000 uct="-" uht="-" urt="-" - log_format romm_logs 'INFO: [RomM][nginx][$date $time] $remote_addr | $http_x_forwarded_for | ' - '$request_method $request_uri | $status | $body_bytes_sent | ' - '$browser $os | $request_time'; + log_format romm_logs 'INFO: [RomM][nginx][$date $time] ' + '$remote_addr | $http_x_forwarded_for | ' + '$request_method $request_uri $status | $body_bytes_sent | ' + '$browser $os | $request_time'; access_log /dev/stdout romm_logs; error_log /dev/stderr; From 14761c2c83bb123edebb6553c3c5f19c21588b50 Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 09:05:59 +0000 Subject: [PATCH 20/32] refactor: enhance logging with highlighted output for improved readability --- backend/adapters/services/rahasher.py | 3 ++- backend/config/config_manager.py | 18 ++++++------- backend/endpoints/collections.py | 10 ++++--- backend/endpoints/firmware.py | 11 ++++---- backend/endpoints/platform.py | 8 ++++-- backend/endpoints/rom.py | 27 ++++++++++++------- backend/endpoints/saves.py | 6 +++-- backend/endpoints/screenshots.py | 4 ++- backend/endpoints/search.py | 6 ++--- backend/endpoints/sockets/scan.py | 20 +++++++------- backend/endpoints/states.py | 6 +++-- backend/handler/auth/base_handler.py | 11 +++++--- backend/handler/filesystem/assets_handler.py | 2 +- .../handler/filesystem/firmware_handler.py | 2 +- .../handler/filesystem/resources_handler.py | 4 ++- backend/handler/scan_handler.py | 18 ++++++++----- backend/scheduler.py | 1 + backend/watcher.py | 10 ++++--- 18 files changed, 101 insertions(+), 66 deletions(-) diff --git a/backend/adapters/services/rahasher.py b/backend/adapters/services/rahasher.py index 938e7f689..0eaa2904f 100644 --- a/backend/adapters/services/rahasher.py +++ b/backend/adapters/services/rahasher.py @@ -2,6 +2,7 @@ import asyncio import re from pathlib import Path +from logger.formatter import highlight as hl from logger.logger import log RAHASHER_VALID_HASH_REGEX = re.compile(r"^[0-9a-f]{32}$") @@ -95,7 +96,7 @@ class RAHasherService: ) args = (str(platform_id), str(file_path)) - log.debug("Executing RAHasher with args: %s", args) + log.debug("Executing RAHasher with args: %s", hl(args)) proc = await asyncio.create_subprocess_exec( "RAHasher", diff --git a/backend/config/config_manager.py b/backend/config/config_manager.py index 4e8fe3019..2689723bf 100644 --- a/backend/config/config_manager.py +++ b/backend/config/config_manager.py @@ -19,6 +19,8 @@ from exceptions.config_exceptions import ( ConfigNotReadableException, ConfigNotWritableException, ) +from logger.formatter import BLUE +from logger.formatter import highlight as hl from logger.logger import log from sqlalchemy import URL from yaml.loader import SafeLoader @@ -83,12 +85,6 @@ class ConfigManager: str: database connection string """ - # DEPRECATED - if ROMM_DB_DRIVER == "sqlite": - log.critical("Sqlite is not supported anymore, migrate to mariaDB") - sys.exit(6) - # DEPRECATED - if ROMM_DB_DRIVER == "mariadb": driver = "mariadb+mariadbconnector" elif ROMM_DB_DRIVER == "mysql": @@ -96,7 +92,7 @@ class ConfigManager: elif ROMM_DB_DRIVER == "postgresql": driver = "postgresql+psycopg" else: - log.critical(f"{ROMM_DB_DRIVER} database not supported") + log.critical(f"{hl(ROMM_DB_DRIVER)} database not supported") sys.exit(3) if not DB_USER or not DB_PASSWD: @@ -279,7 +275,7 @@ class ConfigManager: def add_platform_binding(self, fs_slug: str, slug: str) -> None: platform_bindings = self.config.PLATFORMS_BINDING if fs_slug in platform_bindings: - log.warning(f"Binding for {fs_slug} already exists") + log.warning(f"Binding for {hl(fs_slug)} already exists") return platform_bindings[fs_slug] = slug @@ -300,7 +296,7 @@ class ConfigManager: def add_platform_version(self, fs_slug: str, slug: str) -> None: platform_versions = self.config.PLATFORMS_VERSIONS if fs_slug in platform_versions: - log.warning(f"Version for {fs_slug} already exists") + log.warning(f"Version for {hl(fs_slug)} already exists") return platform_versions[fs_slug] = slug @@ -321,7 +317,9 @@ class ConfigManager: def add_exclusion(self, exclusion_type: str, exclusion_value: str): config_item = self.config.__getattribute__(exclusion_type) if exclusion_value in config_item: - log.warning(f"{exclusion_value} already excluded in {exclusion_type}") + log.warning( + f"{hl(exclusion_value)} already excluded in {hl(exclusion_type, color=BLUE)}" + ) return config_item.append(exclusion_value) diff --git a/backend/endpoints/collections.py b/backend/endpoints/collections.py index 6fbf9932c..d0efedf36 100644 --- a/backend/endpoints/collections.py +++ b/backend/endpoints/collections.py @@ -17,6 +17,8 @@ from handler.auth.constants import Scope from handler.database import db_collection_handler from handler.filesystem import fs_resource_handler from handler.filesystem.base_handler import CoverSize +from logger.formatter import BLUE +from logger.formatter import highlight as hl from logger.logger import log from models.collection import Collection from PIL import Image @@ -284,12 +286,14 @@ async def delete_collections(request: Request, id: int) -> MessageResponse: if not collection: raise CollectionNotFoundInDatabaseException(id) - log.info(f"Deleting {collection.name} from database") + log.info(f"Deleting {hl(collection.name, color=BLUE)} from database") db_collection_handler.delete_collection(id) try: rmtree(f"{RESOURCES_BASE_PATH}/{collection.fs_resources_path}") except FileNotFoundError: - log.error(f"Couldn't find resources to delete for {collection.name}") + log.error( + f"Couldn't find resources to delete for {hl(collection.name, color=BLUE)}" + ) - return {"msg": f"{collection.name} deleted successfully!"} + return {"msg": f"{hl(collection.name)} deleted successfully!"} diff --git a/backend/endpoints/firmware.py b/backend/endpoints/firmware.py index f50924061..ad5a63d2c 100644 --- a/backend/endpoints/firmware.py +++ b/backend/endpoints/firmware.py @@ -8,6 +8,7 @@ from handler.auth.constants import Scope from handler.database import db_firmware_handler, db_platform_handler from handler.filesystem import fs_firmware_handler from handler.scan_handler import scan_firmware +from logger.formatter import highlight as hl from logger.logger import log from utils.router import APIRouter @@ -43,7 +44,7 @@ def add_firmware( log.error(error) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=error) - log.info(f"Uploading firmware to {db_platform.fs_slug}") + log.info(f"Uploading firmware to {hl(db_platform.fs_slug)}") uploaded_firmware = [] firmware_path = fs_firmware_handler.build_upload_file_path(db_platform.fs_slug) @@ -217,21 +218,21 @@ async def delete_firmware( for id in firmare_ids: firmware = db_firmware_handler.get_firmware(id) if not firmware: - error = f"Firmware with ID {id} not found" + error = f"Firmware with ID {hl(id)} not found" log.error(error) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=error) - log.info(f"Deleting {firmware.file_name} from database") + log.info(f"Deleting {hl(firmware.file_name)} from database") db_firmware_handler.delete_firmware(id) if id in delete_from_fs: - log.info(f"Deleting {firmware.file_name} from filesystem") + log.info(f"Deleting {hl(firmware.file_name)} from filesystem") try: fs_firmware_handler.remove_file( file_name=firmware.file_name, file_path=firmware.file_path ) except FileNotFoundError as exc: - error = f"Firmware file {firmware.file_name} not found for platform {firmware.platform_slug}" + error = f"Firmware file {hl(firmware.file_name)} not found for platform {hl(firmware.platform_slug)}" log.error(error) raise HTTPException( status_code=status.HTTP_404_NOT_FOUND, detail=error diff --git a/backend/endpoints/platform.py b/backend/endpoints/platform.py index a6ed6986c..154805a8a 100644 --- a/backend/endpoints/platform.py +++ b/backend/endpoints/platform.py @@ -11,6 +11,8 @@ from handler.database import db_platform_handler from handler.filesystem import fs_platform_handler from handler.metadata.igdb_handler import IGDB_PLATFORM_LIST from handler.scan_handler import scan_platform +from logger.formatter import BLUE +from logger.formatter import highlight as hl from logger.logger import log from utils.router import APIRouter @@ -36,7 +38,7 @@ async def add_platforms(request: Request) -> PlatformSchema: try: fs_platform_handler.add_platforms(fs_slug=fs_slug) except PlatformAlreadyExistsException: - log.info(f"Detected platform: {fs_slug}") + log.info(f"Detected platform: {hl(fs_slug)}") scanned_platform = await scan_platform(fs_slug, [fs_slug]) return PlatformSchema.model_validate( db_platform_handler.add_platform(scanned_platform) @@ -164,7 +166,9 @@ async def delete_platforms(request: Request, id: int) -> MessageResponse: if not platform: raise PlatformNotFoundInDatabaseException(id) - log.info(f"Deleting {platform.name} [{platform.fs_slug}] from database") + log.info( + f"Deleting {hl(platform.name, color=BLUE)} [{hl(platform.fs_slug)}] from database" + ) db_platform_handler.delete_platform(id) return {"msg": f"{platform.name} - [{platform.fs_slug}] deleted successfully!"} diff --git a/backend/endpoints/rom.py b/backend/endpoints/rom.py index 0216afb3c..f520ab794 100644 --- a/backend/endpoints/rom.py +++ b/backend/endpoints/rom.py @@ -37,6 +37,7 @@ from handler.database.base_handler import sync_session from handler.filesystem import fs_resource_handler, fs_rom_handler from handler.filesystem.base_handler import CoverSize from handler.metadata import meta_igdb_handler, meta_moby_handler, meta_ss_handler +from logger.formatter import BLUE from logger.formatter import highlight as hl from logger.logger import log from models.rom import RomFile @@ -85,7 +86,7 @@ async def add_rom(request: Request): platform_fs_slug = db_platform.fs_slug roms_path = fs_rom_handler.build_upload_fs_path(platform_fs_slug) - log.info(f"Uploading file to {platform_fs_slug}") + log.info(f"Uploading file to {hl(platform_fs_slug)}") file_location = Path(f"{roms_path}/{filename}") parser = StreamingFormDataParser(headers=request.headers) @@ -93,7 +94,7 @@ async def add_rom(request: Request): parser.register(filename, FileTarget(str(file_location))) if await file_location.exists(): - log.warning(f" - Skipping {filename} since the file already exists") + log.warning(f" - Skipping {hl(filename)} since the file already exists") raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail=f"File {filename} already exists", @@ -361,7 +362,9 @@ async def get_rom_content( files = [f for f in rom.files if f.id in file_ids or not file_ids] files.sort(key=lambda x: x.file_name) - log.info(f"User {current_username} is downloading {rom.fs_name}") + log.info( + f"User {hl(current_username, color=BLUE)} is downloading {hl(rom.fs_name)}" + ) # Serve the file directly in development mode for emulatorjs if DEV_MODE: @@ -406,7 +409,7 @@ async def get_rom_content( zip_file.writestr(zip_info, content) except FileNotFoundError: - log.error(f"File {file_path} not found!") + log.error(f"File {hl(file_path)} not found!") raise # Add M3U file if not already present @@ -646,7 +649,7 @@ async def update_rom( cleaned_data.update({"path_manual": path_manual}) log.debug( - f"Updating {hl(cleaned_data.get('name', ''))} [{id}] with data {cleaned_data}" + f"Updating {hl(cleaned_data.get('name', ''), color=BLUE)} [{hl(id)}] with data {cleaned_data}" ) db_rom_handler.update_rom(id, cleaned_data) @@ -704,7 +707,7 @@ async def add_rom_manuals(request: Request, id: int): manuals_path = f"{RESOURCES_BASE_PATH}/{rom.fs_resources_path}/manual" file_location = Path(f"{manuals_path}/{rom.id}.pdf") - log.info(f"Uploading {file_location}") + log.info(f"Uploading {hl(file_location)}") if not os.path.exists(manuals_path): await Path(manuals_path).mkdir(parents=True, exist_ok=True) @@ -767,16 +770,20 @@ async def delete_roms( if not rom: raise RomNotFoundInDatabaseException(id) - log.info(f"Deleting {rom.fs_name} from database") + log.info( + f"Deleting {hl(rom.name, color=BLUE)} [{hl(rom.fs_name)}] from database" + ) db_rom_handler.delete_rom(id) try: rmtree(f"{RESOURCES_BASE_PATH}/{rom.fs_resources_path}") except FileNotFoundError: - log.error(f"Couldn't find resources to delete for {rom.name}") + log.error( + f"Couldn't find resources to delete for {hl(rom.name, color=BLUE)}" + ) if id in delete_from_fs: - log.info(f"Deleting {rom.fs_name} from filesystem") + log.info(f"Deleting {hl(rom.fs_name)} from filesystem") try: fs_rom_handler.remove_from_fs(fs_path=rom.fs_path, fs_name=rom.fs_name) except FileNotFoundError as exc: @@ -885,7 +892,7 @@ async def get_romfile_content( detail="File not found", ) - log.info(f"User {current_username} is downloading {file_name}") + log.info(f"User {hl(current_username, color=BLUE)} is downloading {hl(file_name)}") # Serve the file directly in development mode for emulatorjs if DEV_MODE: diff --git a/backend/endpoints/saves.py b/backend/endpoints/saves.py index 0b977893c..d01687285 100644 --- a/backend/endpoints/saves.py +++ b/backend/endpoints/saves.py @@ -8,6 +8,8 @@ from handler.auth.constants import Scope from handler.database import db_rom_handler, db_save_handler, db_screenshot_handler from handler.filesystem import fs_asset_handler from handler.scan_handler import scan_save, scan_screenshot +from logger.formatter import BLUE +from logger.formatter import highlight as hl from logger.logger import log from utils.router import APIRouter @@ -29,7 +31,7 @@ async def add_save( if not rom: raise RomNotFoundInDatabaseException(rom_id) - log.info(f"Uploading save of {rom.name}") + log.info(f"Uploading save of {hl(rom.name, color=BLUE)}") saves_path = fs_asset_handler.build_saves_file_path( user=request.user, platform_fs_slug=rom.platform.fs_slug, emulator=emulator @@ -213,7 +215,7 @@ async def delete_saves(request: Request) -> list[int]: db_save_handler.delete_save(save_id) - log.info(f"Deleting {save.file_name} from filesystem") + log.info(f"Deleting {hl(save.file_name)} from filesystem") try: fs_asset_handler.remove_file( file_name=save.file_name, file_path=save.file_path diff --git a/backend/endpoints/screenshots.py b/backend/endpoints/screenshots.py index d35fa4a4e..d6d01f367 100644 --- a/backend/endpoints/screenshots.py +++ b/backend/endpoints/screenshots.py @@ -6,6 +6,8 @@ from handler.auth.constants import Scope from handler.database import db_rom_handler, db_screenshot_handler from handler.filesystem import fs_asset_handler from handler.scan_handler import scan_screenshot +from logger.formatter import BLUE +from logger.formatter import highlight as hl from logger.logger import log from utils.router import APIRouter @@ -27,7 +29,7 @@ async def add_screenshot( raise RomNotFoundInDatabaseException(rom_id) current_user = request.user - log.info(f"Uploading screenshots to {rom.name}") + log.info(f"Uploading screenshots to {hl(rom.name, color=BLUE)}") screenshots_path = fs_asset_handler.build_screenshots_file_path( user=request.user, platform_fs_slug=rom.platform_slug diff --git a/backend/endpoints/search.py b/backend/endpoints/search.py index e8cfea6a6..c2fcf7651 100644 --- a/backend/endpoints/search.py +++ b/backend/endpoints/search.py @@ -18,7 +18,7 @@ from handler.metadata.moby_handler import MOBY_API_ENABLED, MobyGamesRom from handler.metadata.sgdb_handler import STEAMGRIDDB_API_ENABLED from handler.metadata.ss_handler import SS_API_ENABLED, SSRom from handler.scan_handler import _get_main_platform_igdb_id -from logger.formatter import BLUE +from logger.formatter import BLUE, CYAN from logger.formatter import highlight as hl from logger.logger import log from utils.router import APIRouter @@ -70,10 +70,10 @@ async def search_rom( ) matched_roms: list = [] - log.info(f"Searching by {search_by.lower()}:") + log.info(f"Searching by {hl(search_by.lower(), color=CYAN)}:") log.info( emoji.emojize( - f":video_game: {hl(rom.platform_display_name, color=BLUE)}[{rom.platform_fs_slug}]: {hl(search_term)}[{rom.fs_name}]" + f":video_game: {hl(rom.platform_display_name, color=BLUE)} [{rom.platform_fs_slug}]: {hl(search_term)}[{rom.fs_name}]" ) ) diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index 57d9fb55e..e8938aad8 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -186,7 +186,7 @@ async def scan_platforms( ) ) else: - log.info(f"Found {len(platform_list)} platforms in the file system") + log.info(f"Found {hl(len(platform_list))} platforms in the file system") for platform_slug in platform_list: scan_stats += await _identify_platform( @@ -204,9 +204,9 @@ async def scan_platforms( if len(fs_platforms) > 0: purged_platforms = db_platform_handler.purge_platforms(fs_platforms) if len(purged_platforms) > 0: - log.info("Purging platforms not found in the filesystem:") + log.warning("Purging platforms not found in the filesystem:") for p in purged_platforms: - log.info(f" - {p.slug}") + log.warning(f" - {p.slug}") log.info(emoji.emojize(" :check_mark: Scan completed ")) await sm.emit("scan:done", scan_stats.__dict__) @@ -277,7 +277,7 @@ async def _identify_platform( ) ) else: - log.info(f" {len(fs_firmware)} firmware files found") + log.info(f" {hl(len(fs_firmware))} firmware files found") for fs_fw in fs_firmware: scan_stats += await _identify_firmware( @@ -299,7 +299,7 @@ async def _identify_platform( ) ) else: - log.info(f" {len(fs_roms)} roms found in the file system") + log.info(f" {hl(len(fs_roms))} roms found in the file system") for fs_roms_batch in batched(fs_roms, 200): rom_by_filename_map = db_rom_handler.get_roms_by_fs_name( @@ -326,9 +326,9 @@ async def _identify_platform( platform.id, [rom["fs_name"] for rom in fs_roms] ) if len(purged_roms) > 0: - log.info("Purging roms not found in the filesystem:") + log.warning("Purging roms not found in the filesystem:") for r in purged_roms: - log.info(f" - {r.fs_name}") + log.warning(f" - {r.fs_name}") # Same protection for firmware if len(fs_firmware) > 0: @@ -336,9 +336,9 @@ async def _identify_platform( platform.id, [fw for fw in fs_firmware] ) if len(purged_firmware) > 0: - log.info("Purging firmware not found in the filesystem:") + log.warning("Purging firmware not found in the filesystem:") for f in purged_firmware: - log.info(f" - {f}") + log.warning(f" - {f}") return scan_stats @@ -383,7 +383,7 @@ def _set_rom_hashes(rom_id: int): except zlib.error as e: # Set empty hashes if calculating them fails for corrupted files log.error( - f"Hashes of {rom.fs_name} couldn't be calculated: {hl(str(e), color=RED)}" + f"Hashes of {hl(rom.fs_name)} couldn't be calculated: {hl(str(e), color=RED)}" ) db_rom_handler.update_rom( rom_id, diff --git a/backend/endpoints/states.py b/backend/endpoints/states.py index 585dfe455..3d3687836 100644 --- a/backend/endpoints/states.py +++ b/backend/endpoints/states.py @@ -8,6 +8,8 @@ from handler.auth.constants import Scope from handler.database import db_rom_handler, db_screenshot_handler, db_state_handler from handler.filesystem import fs_asset_handler from handler.scan_handler import scan_screenshot, scan_state +from logger.formatter import BLUE +from logger.formatter import highlight as hl from logger.logger import log from utils.router import APIRouter @@ -29,7 +31,7 @@ async def add_state( if not rom: raise RomNotFoundInDatabaseException(rom_id) - log.info(f"Uploading state of {rom.name}") + log.info(f"Uploading state of {hl(rom.name, color=BLUE)}") states_path = fs_asset_handler.build_states_file_path( user=request.user, platform_fs_slug=rom.platform.fs_slug, emulator=emulator @@ -216,7 +218,7 @@ async def delete_states(request: Request) -> list[int]: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=error) db_state_handler.delete_state(state_id) - log.info(f"Deleting {state.file_name} from filesystem") + log.info(f"Deleting {hl(state.file_name)} from filesystem") try: fs_asset_handler.remove_file( diff --git a/backend/handler/auth/base_handler.py b/backend/handler/auth/base_handler.py index a7516cceb..6580f6a30 100644 --- a/backend/handler/auth/base_handler.py +++ b/backend/handler/auth/base_handler.py @@ -10,6 +10,8 @@ from handler.auth.constants import ALGORITHM, DEFAULT_OAUTH_TOKEN_EXPIRY from joserfc import jwt from joserfc.errors import BadSignatureError from joserfc.jwk import OctKey +from logger.formatter import CYAN +from logger.formatter import highlight as hl from logger.logger import log from passlib.context import CryptContext from starlette.requests import HTTPConnection @@ -54,7 +56,7 @@ class AuthHandler: conn.session.clear() log.error( "User '%s' %s", - username, + hl(username, color=CYAN), "not found" if user is None else "not enabled", ) return None @@ -148,7 +150,10 @@ class OpenIDHandler: user = db_user_handler.get_user_by_email(email) if user is None: - log.info("User with email '%s' not found, creating new user", email) + log.info( + "User with email '%s' not found, creating new user", + hl(email, color=CYAN), + ) user = User( username=preferred_username, hashed_password=str(uuid.uuid4()), @@ -161,5 +166,5 @@ class OpenIDHandler: if not user.enabled: raise UserDisabledException - log.info("User successfully authenticated: %s", email) + log.info("User successfully authenticated: %s", hl(email, color=CYAN)) return user, userinfo diff --git a/backend/handler/filesystem/assets_handler.py b/backend/handler/filesystem/assets_handler.py index 5de48cd12..c0490dba1 100644 --- a/backend/handler/filesystem/assets_handler.py +++ b/backend/handler/filesystem/assets_handler.py @@ -26,7 +26,7 @@ class FSAssetsHandler(FSHandler): return Path(os.path.join(ASSETS_BASE_PATH, path)).mkdir(parents=True, exist_ok=True) - log.info(f" - Uploading {file.filename}") + log.info(f" - Uploading {hl(file.filename)}") file_location = os.path.join(ASSETS_BASE_PATH, path, file.filename) with open(file_location, "wb") as f: diff --git a/backend/handler/filesystem/firmware_handler.py b/backend/handler/filesystem/firmware_handler.py index 897366a82..84d89ed16 100644 --- a/backend/handler/filesystem/firmware_handler.py +++ b/backend/handler/filesystem/firmware_handler.py @@ -90,7 +90,7 @@ class FSFirmwareHandler(FSHandler): return Path(path).mkdir(parents=True, exist_ok=True) - log.info(f" - Uploading {file.filename}") + log.info(f" - Uploading {hl(file.filename)}") file_location = os.path.join(path, file.filename) with open(file_location, "wb") as f: diff --git a/backend/handler/filesystem/resources_handler.py b/backend/handler/filesystem/resources_handler.py index 243281e94..97a4eeeac 100644 --- a/backend/handler/filesystem/resources_handler.py +++ b/backend/handler/filesystem/resources_handler.py @@ -3,6 +3,8 @@ import shutil import httpx from anyio import Path, open_file from config import RESOURCES_BASE_PATH +from logger.formatter import BLUE +from logger.formatter import highlight as hl from logger.logger import log from models.collection import Collection from models.rom import Rom @@ -126,7 +128,7 @@ class FSResourcesHandler(FSHandler): shutil.rmtree(cover_path) except FileNotFoundError: log.warning( - f"Couldn't remove cover from '{entity.name or entity.id}' since '{cover_path}' doesn't exists." + f"Couldn't remove cover from '{hl(entity.name or entity.id, color=BLUE)}' since '{cover_path}' doesn't exists." ) return {"path_cover_s": "", "path_cover_l": ""} diff --git a/backend/handler/scan_handler.py b/backend/handler/scan_handler.py index 482a3d166..136eeb097 100644 --- a/backend/handler/scan_handler.py +++ b/backend/handler/scan_handler.py @@ -68,7 +68,7 @@ async def scan_platform( Platform object """ - log.info(f"· Found {hl(fs_slug)} folder") + log.info(f"· Found {hl(fs_slug, color=BLUE)} folder") if metadata_sources is None: metadata_sources = [MetadataSource.IGDB, MetadataSource.MOBY, MetadataSource.SS] @@ -83,7 +83,7 @@ async def scan_platform( # Sometimes users change the name of the folder, so we try to match it with the config if fs_slug not in fs_platforms: log.warning( - f" {fs_slug} not found in file system, trying to match via config..." + f" {hl(fs_slug)} not found in file system, trying to match via config..." ) if fs_slug in swapped_platform_bindings.keys(): platform = db_platform_handler.get_platform_by_fs_slug(fs_slug) @@ -138,7 +138,7 @@ async def scan_platform( else: log.warning( emoji.emojize( - f" Platform {platform_attrs['slug']} not identified :cross_mark:" + f" Platform {hl(platform_attrs['slug'])} not identified :cross_mark:" ) ) @@ -152,7 +152,7 @@ def scan_firmware( ) -> Firmware: firmware_path = fs_firmware_handler.get_firmware_fs_structure(platform.fs_slug) - log.info(f"\t · {file_name}") + log.info(f"\t · {hl(file_name)}") # Set default properties firmware_attrs = { @@ -349,18 +349,22 @@ async def scan_rom( ): log.warning( emoji.emojize( - f"\t Rom {rom_attrs['fs_name']} not identified :cross_mark:" + f"\t Rom {hl(rom_attrs['fs_name'])} not identified :cross_mark:" ) ) return Rom(**rom_attrs) - log.info(emoji.emojize(f"\t Identified as {rom_attrs['name']} :alien_monster:")) + log.info( + emoji.emojize( + f"\t Identified as {hl(rom_attrs['name'], color=BLUE)} :alien_monster:" + ) + ) return Rom(**rom_attrs) def _scan_asset(file_name: str, path: str): - log.info(f"\t\t · {file_name}") + log.info(f"\t\t · {hl(file_name)}") file_size = fs_asset_handler.get_asset_size(file_name=file_name, asset_path=path) diff --git a/backend/scheduler.py b/backend/scheduler.py index ec9385abd..e6d4bad6a 100644 --- a/backend/scheduler.py +++ b/backend/scheduler.py @@ -4,6 +4,7 @@ from config import ( ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, SENTRY_DSN, ) +from logger.formatter import highlight as hl from logger.logger import log from tasks.scan_library import scan_library_task from tasks.tasks import tasks_scheduler diff --git a/backend/watcher.py b/backend/watcher.py index 7c18f4c56..90209264a 100644 --- a/backend/watcher.py +++ b/backend/watcher.py @@ -12,6 +12,8 @@ from config.config_manager import config_manager as cm from endpoints.sockets.scan import scan_platforms from handler.database import db_platform_handler from handler.scan_handler import ScanType +from logger.formatter import CYAN +from logger.formatter import highlight as hl from logger.logger import log from rq.job import Job from tasks.tasks import tasks_scheduler @@ -66,11 +68,11 @@ class EventHandler(FileSystemEventHandler): return if db_platform and db_platform.id in job.args[0]: - log.info(f"Scan already scheduled for {fs_slug}") + log.info(f"Scan already scheduled for {hl(fs_slug)}") return time_delta = timedelta(minutes=RESCAN_ON_FILESYSTEM_CHANGE_DELAY) - rescan_in_msg = f"rescanning in {RESCAN_ON_FILESYSTEM_CHANGE_DELAY} minutes." + rescan_in_msg = f"rescanning in {hl(RESCAN_ON_FILESYSTEM_CHANGE_DELAY, color=CYAN)} minutes." # Any change to a platform directory should trigger a full rescan if event.is_directory and event_src.count("/") == 1: @@ -78,7 +80,7 @@ class EventHandler(FileSystemEventHandler): tasks_scheduler.enqueue_in(time_delta, scan_platforms, []) elif db_platform: # Otherwise trigger a rescan for the specific platform - log.info(f"Change detected in {fs_slug} folder, {rescan_in_msg}") + log.info(f"Change detected in {hl(fs_slug)} folder, {rescan_in_msg}") tasks_scheduler.enqueue_in( time_delta, scan_platforms, @@ -93,7 +95,7 @@ if __name__ == "__main__": observer.schedule(EventHandler(), path, recursive=True) observer.start() - log.info(f"Watching {path} for changes") + log.info(f"Watching {hl(path)} for changes") try: while observer.is_alive(): From e6c4a43c1e4ad89d9789714e08f2cae58fc77db0 Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 09:35:13 +0000 Subject: [PATCH 21/32] refactor: improve logging messages and formatting for clarity and consistency across multiple modules --- backend/endpoints/rom.py | 4 +++- backend/endpoints/sockets/scan.py | 14 ++++++++------ backend/handler/scan_handler.py | 27 ++++++++++----------------- docker/init_scripts/init | 6 ++---- 4 files changed, 23 insertions(+), 28 deletions(-) diff --git a/backend/endpoints/rom.py b/backend/endpoints/rom.py index f520ab794..0c29fba06 100644 --- a/backend/endpoints/rom.py +++ b/backend/endpoints/rom.py @@ -86,7 +86,9 @@ async def add_rom(request: Request): platform_fs_slug = db_platform.fs_slug roms_path = fs_rom_handler.build_upload_fs_path(platform_fs_slug) - log.info(f"Uploading file to {hl(platform_fs_slug)}") + log.info( + f"Uploading file to {hl(db_platform.custom_name or db_platform.name, color=BLUE)}[{hl(platform_fs_slug)}]" + ) file_location = Path(f"{roms_path}/{filename}") parser = StreamingFormDataParser(headers=request.headers) diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index e8938aad8..c86dc669a 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -33,7 +33,7 @@ from handler.scan_handler import ( scan_rom, ) from handler.socket_handler import socket_handler -from logger.formatter import LIGHTYELLOW, RED +from logger.formatter import BLUE, LIGHTYELLOW, RED from logger.formatter import highlight as hl from logger.logger import log from models.platform import Platform @@ -186,7 +186,9 @@ async def scan_platforms( ) ) else: - log.info(f"Found {hl(len(platform_list))} platforms in the file system") + log.info( + f"Found {hl(str(len(platform_list)))} platforms in the file system" + ) for platform_slug in platform_list: scan_stats += await _identify_platform( @@ -273,11 +275,11 @@ async def _identify_platform( if len(fs_firmware) == 0: log.warning( emoji.emojize( - f" {hl(':warning:', color=LIGHTYELLOW)} No firmware found, skipping firmware scan for this platform" + f"{hl(':warning:', color=LIGHTYELLOW)} No firmware found for {hl(platform.custom_name or platform.name, color=BLUE)}[{hl(platform.fs_slug)}]" ) ) else: - log.info(f" {hl(len(fs_firmware))} firmware files found") + log.info(f"{hl(str(len(fs_firmware)))} firmware files found") for fs_fw in fs_firmware: scan_stats += await _identify_firmware( @@ -295,11 +297,11 @@ async def _identify_platform( if len(fs_roms) == 0: log.warning( emoji.emojize( - f" {hl(':warning:', color=LIGHTYELLOW)} No roms found, verify that the folder structure is correct" + f"{hl(':warning:', color=LIGHTYELLOW)} No roms found, verify that the folder structure is correct" ) ) else: - log.info(f" {hl(len(fs_roms))} roms found in the file system") + log.info(f"{hl(str(len(fs_roms)))} roms found in the file system") for fs_roms_batch in batched(fs_roms, 200): rom_by_filename_map = db_rom_handler.get_roms_by_fs_name( diff --git a/backend/handler/scan_handler.py b/backend/handler/scan_handler.py index 136eeb097..d327fa299 100644 --- a/backend/handler/scan_handler.py +++ b/backend/handler/scan_handler.py @@ -11,7 +11,7 @@ from handler.metadata import meta_igdb_handler, meta_moby_handler, meta_ss_handl from handler.metadata.igdb_handler import IGDBPlatform, IGDBRom from handler.metadata.moby_handler import MobyGamesPlatform, MobyGamesRom from handler.metadata.ss_handler import SSPlatform, SSRom -from logger.formatter import BLUE +from logger.formatter import BLUE, LIGHTYELLOW from logger.formatter import highlight as hl from logger.logger import log from models.assets import Save, Screenshot, State @@ -68,8 +68,6 @@ async def scan_platform( Platform object """ - log.info(f"· Found {hl(fs_slug, color=BLUE)} folder") - if metadata_sources is None: metadata_sources = [MetadataSource.IGDB, MetadataSource.MOBY, MetadataSource.SS] @@ -83,7 +81,7 @@ async def scan_platform( # Sometimes users change the name of the folder, so we try to match it with the config if fs_slug not in fs_platforms: log.warning( - f" {hl(fs_slug)} not found in file system, trying to match via config..." + f"{hl(fs_slug)} not found in file system, trying to match via config" ) if fs_slug in swapped_platform_bindings.keys(): platform = db_platform_handler.get_platform_by_fs_slug(fs_slug) @@ -132,13 +130,13 @@ async def scan_platform( ): log.info( emoji.emojize( - f" Identified as {hl(platform_attrs['name'], color=BLUE)} :video_game:" + f"Folder {hl(platform_attrs['fs_slug'])}[{hl(fs_slug)}] identified as {hl(platform_attrs['name'], color=BLUE)} :video_game:" ) ) else: log.warning( emoji.emojize( - f" Platform {hl(platform_attrs['slug'])} not identified :cross_mark:" + f"Platform {hl(platform_attrs['slug'])} not identified :cross_mark:" ) ) @@ -202,12 +200,6 @@ async def scan_rom( roms_path = fs_rom_handler.get_roms_fs_structure(platform.fs_slug) - log.info(f"\t · {hl(fs_rom['fs_name'])}") - - if fs_rom.get("multi", False): - for file in fs_rom["files"]: - log.info(f"\t\t · {file.file_name}") - # Set default properties rom_attrs = { "id": rom.id if rom else None, @@ -348,23 +340,24 @@ async def scan_rom( and not ss_handler_rom.get("ss_id") ): log.warning( - emoji.emojize( - f"\t Rom {hl(rom_attrs['fs_name'])} not identified :cross_mark:" - ) + emoji.emojize(f"{hl(rom_attrs['fs_name'])} not identified :cross_mark:") ) return Rom(**rom_attrs) log.info( emoji.emojize( - f"\t Identified as {hl(rom_attrs['name'], color=BLUE)} :alien_monster:" + f"{hl(rom_attrs['fs_name'])} identified as {hl(rom_attrs['name'], color=BLUE)} :alien_monster:" ) ) + if fs_rom.get("multi", False): + for file in fs_rom["files"]: + log.info(f"\t · {hl(file.file_name, color=LIGHTYELLOW)}") return Rom(**rom_attrs) def _scan_asset(file_name: str, path: str): - log.info(f"\t\t · {hl(file_name)}") + log.info(f"\t · {hl(file_name)}") file_size = fs_asset_handler.get_asset_size(file_name=file_name, asset_path=path) diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 0b69eeab2..b397a62fa 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -48,8 +48,6 @@ error_log() { exit 1 } -# TODO: check on database disconnection - wait_for_gunicorn_socket() { debug_log "Waiting for gunicorn socket file..." local retries=60 @@ -80,8 +78,8 @@ start_bin_gunicorn() { fi gunicorn \ - --error-logfile /tmp/gunicorn_access.log \ - --error-logfile /tmp/gunicorn_error.log \ + --error-logfile - \ + --error-logfile - \ --worker-class uvicorn.workers.UvicornWorker \ --bind=0.0.0.0:5000 \ --bind=unix:/tmp/gunicorn.sock \ From 549e5956bfeb7650d1a2d04920e827a009a21503 Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 10:24:45 +0000 Subject: [PATCH 22/32] refactor: add initial logging configuration for Gunicorn --- docker/gunicorn/logging.conf | 39 ++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 docker/gunicorn/logging.conf diff --git a/docker/gunicorn/logging.conf b/docker/gunicorn/logging.conf new file mode 100644 index 000000000..5429635ff --- /dev/null +++ b/docker/gunicorn/logging.conf @@ -0,0 +1,39 @@ +[loggers] +keys=root,gunicorn,error + +[handlers] +keys=console_gunicorn + +[formatters] +keys=gunicorn_format + +# Root logger — KEEP but minimal +[logger_root] +level=WARNING +handlers= + +# Gunicorn internal logger +[logger_gunicorn] +level=INFO +handlers=console_gunicorn +qualname=gunicorn +propagate=0 + +# Gunicorn error logger (optional) +[logger_error] +level=ERROR +handlers=console_gunicorn +qualname=gunicorn.error +propagate=0 + +# Handler for Gunicorn logs +[handler_console_gunicorn] +class=StreamHandler +formatter=gunicorn_format +args=(sys.stdout,) + +# Formatter for Gunicorn logs +[formatter_gunicorn_format] +format=INFO: [RomM][gunicorn][%(asctime)s] %(message)s +datefmt=%Y-%m-%d %H:%M:%S + From 22f5134fbb61e605c34a045c35257a61ce13cef9 Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 10:25:02 +0000 Subject: [PATCH 23/32] refactor: add Gunicorn logging configuration and update startup script --- docker/Dockerfile | 1 + docker/init_scripts/init | 9 +++++---- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index cd82754d2..fdae438e7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -156,6 +156,7 @@ COPY ./docker/init_scripts/* / COPY ./docker/nginx/js/ /etc/nginx/js/ COPY ./docker/nginx/templates/ /etc/nginx/templates/ COPY ./docker/nginx/default.conf /etc/nginx/nginx.conf +COPY ./docker/gunicorn/logging.conf /etc/gunicorn/logging.conf # User permissions # - Create default user `romm` (1000) and group `romm` (1000). diff --git a/docker/init_scripts/init b/docker/init_scripts/init index b397a62fa..1ad6b7d13 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -78,14 +78,15 @@ start_bin_gunicorn() { fi gunicorn \ - --error-logfile - \ - --error-logfile - \ - --worker-class uvicorn.workers.UvicornWorker \ --bind=0.0.0.0:5000 \ --bind=unix:/tmp/gunicorn.sock \ --pid=/tmp/gunicorn.pid \ --forwarded-allow-ips="*" \ + --worker-class uvicorn.workers.UvicornWorker \ --workers "${WEB_CONCURRENCY:-${DEFAULT_WEB_CONCURRENCY:-1}}" \ + --error-logfile - \ + --error-logfile - \ + --log-config /etc/gunicorn/logging.conf \ main:app & } @@ -100,7 +101,7 @@ start_bin_nginx() { # if container runs as root, drop permissions nginx -g 'user romm;' fi - info_log "🚀 RomM is now available at http://[::1]:8080" + info_log "🚀 RomM is now available at http://0.0.0.0:8080" } # Commands to start valkey-server (handling PID creation internally) From 6f08912fc0f3af1aab4adb5534a7fbe2f2916974 Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 10:25:11 +0000 Subject: [PATCH 24/32] refactor: remove unnecessary logging highlights and improve log messages for clarity --- backend/adapters/services/rahasher.py | 3 +-- backend/endpoints/firmware.py | 7 ++++-- backend/endpoints/rom.py | 2 +- backend/endpoints/saves.py | 25 +++++++++++-------- backend/endpoints/states.py | 25 +++++++++++-------- backend/handler/filesystem/assets_handler.py | 1 - .../handler/filesystem/firmware_handler.py | 1 - backend/handler/scan_handler.py | 4 --- backend/scheduler.py | 1 - backend/watcher.py | 2 +- backend/worker.py | 3 --- 11 files changed, 36 insertions(+), 38 deletions(-) diff --git a/backend/adapters/services/rahasher.py b/backend/adapters/services/rahasher.py index 0eaa2904f..938e7f689 100644 --- a/backend/adapters/services/rahasher.py +++ b/backend/adapters/services/rahasher.py @@ -2,7 +2,6 @@ import asyncio import re from pathlib import Path -from logger.formatter import highlight as hl from logger.logger import log RAHASHER_VALID_HASH_REGEX = re.compile(r"^[0-9a-f]{32}$") @@ -96,7 +95,7 @@ class RAHasherService: ) args = (str(platform_id), str(file_path)) - log.debug("Executing RAHasher with args: %s", hl(args)) + log.debug("Executing RAHasher with args: %s", args) proc = await asyncio.create_subprocess_exec( "RAHasher", diff --git a/backend/endpoints/firmware.py b/backend/endpoints/firmware.py index ad5a63d2c..b36175837 100644 --- a/backend/endpoints/firmware.py +++ b/backend/endpoints/firmware.py @@ -8,6 +8,7 @@ from handler.auth.constants import Scope from handler.database import db_firmware_handler, db_platform_handler from handler.filesystem import fs_firmware_handler from handler.scan_handler import scan_firmware +from logger.formatter import BLUE from logger.formatter import highlight as hl from logger.logger import log from utils.router import APIRouter @@ -44,8 +45,6 @@ def add_firmware( log.error(error) raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=error) - log.info(f"Uploading firmware to {hl(db_platform.fs_slug)}") - uploaded_firmware = [] firmware_path = fs_firmware_handler.build_upload_file_path(db_platform.fs_slug) @@ -54,6 +53,10 @@ def add_firmware( log.warning("Empty filename, skipping") continue + log.info( + f"Uploading firmware {hl(file.filename)} to {hl(db_platform.custom_name or db_platform.name, color=BLUE)}" + ) + fs_firmware_handler.write_file(file=file, path=firmware_path) db_firmware = db_firmware_handler.get_firmware_by_filename( diff --git a/backend/endpoints/rom.py b/backend/endpoints/rom.py index 0c29fba06..74da81a81 100644 --- a/backend/endpoints/rom.py +++ b/backend/endpoints/rom.py @@ -651,7 +651,7 @@ async def update_rom( cleaned_data.update({"path_manual": path_manual}) log.debug( - f"Updating {hl(cleaned_data.get('name', ''), color=BLUE)} [{hl(id)}] with data {cleaned_data}" + f"Updating {hl(cleaned_data.get('name', ''), color=BLUE)} [{hl(cleaned_data.get('fs_name', ''))}] with data {cleaned_data}" ) db_rom_handler.update_rom(id, cleaned_data) diff --git a/backend/endpoints/saves.py b/backend/endpoints/saves.py index d01687285..f9e24256a 100644 --- a/backend/endpoints/saves.py +++ b/backend/endpoints/saves.py @@ -27,16 +27,6 @@ async def add_save( ) -> SaveSchema: data = await request.form() - rom = db_rom_handler.get_rom(rom_id) - if not rom: - raise RomNotFoundInDatabaseException(rom_id) - - log.info(f"Uploading save of {hl(rom.name, color=BLUE)}") - - saves_path = fs_asset_handler.build_saves_file_path( - user=request.user, platform_fs_slug=rom.platform.fs_slug, emulator=emulator - ) - if "saveFile" not in data: log.error("No save file provided") raise HTTPException( @@ -44,12 +34,23 @@ async def add_save( ) saveFile: UploadFile = data["saveFile"] # type: ignore + if not saveFile.filename: log.error("Save file has no filename") raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="Save file has no filename" ) + rom = db_rom_handler.get_rom(rom_id) + if not rom: + raise RomNotFoundInDatabaseException(rom_id) + + log.info(f"Uploading save {hl(saveFile.filename)} for {hl(rom.name, color=BLUE)}") + + saves_path = fs_asset_handler.build_saves_file_path( + user=request.user, platform_fs_slug=rom.platform.fs_slug, emulator=emulator + ) + fs_asset_handler.write_file(file=saveFile, path=saves_path) # Scan or update save @@ -215,7 +216,9 @@ async def delete_saves(request: Request) -> list[int]: db_save_handler.delete_save(save_id) - log.info(f"Deleting {hl(save.file_name)} from filesystem") + log.info( + f"Deleting save {hl(save.file_name)} [{save.rom.platform_slug}] from filesystem" + ) try: fs_asset_handler.remove_file( file_name=save.file_name, file_path=save.file_path diff --git a/backend/endpoints/states.py b/backend/endpoints/states.py index 3d3687836..f0d373fa6 100644 --- a/backend/endpoints/states.py +++ b/backend/endpoints/states.py @@ -27,16 +27,6 @@ async def add_state( ) -> StateSchema: data = await request.form() - rom = db_rom_handler.get_rom(rom_id) - if not rom: - raise RomNotFoundInDatabaseException(rom_id) - - log.info(f"Uploading state of {hl(rom.name, color=BLUE)}") - - states_path = fs_asset_handler.build_states_file_path( - user=request.user, platform_fs_slug=rom.platform.fs_slug, emulator=emulator - ) - if "stateFile" not in data: log.error("No state file provided") raise HTTPException( @@ -44,12 +34,23 @@ async def add_state( ) stateFile: UploadFile = data["stateFile"] # type: ignore + if not stateFile.filename: log.error("State file has no filename") raise HTTPException( status_code=status.HTTP_400_BAD_REQUEST, detail="State file has no filename" ) + rom = db_rom_handler.get_rom(rom_id) + if not rom: + raise RomNotFoundInDatabaseException(rom_id) + + log.info(f"Uploading state {hl(stateFile.filename)} for {hl(rom.name, color=BLUE)}") + + states_path = fs_asset_handler.build_states_file_path( + user=request.user, platform_fs_slug=rom.platform.fs_slug, emulator=emulator + ) + fs_asset_handler.write_file(file=stateFile, path=states_path) # Scan or update state @@ -218,7 +219,9 @@ async def delete_states(request: Request) -> list[int]: raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail=error) db_state_handler.delete_state(state_id) - log.info(f"Deleting {hl(state.file_name)} from filesystem") + log.info( + f"Deleting state {hl(state.file_name)} [{state.rom.platform_slug}] from filesystem" + ) try: fs_asset_handler.remove_file( diff --git a/backend/handler/filesystem/assets_handler.py b/backend/handler/filesystem/assets_handler.py index c0490dba1..e8a329945 100644 --- a/backend/handler/filesystem/assets_handler.py +++ b/backend/handler/filesystem/assets_handler.py @@ -26,7 +26,6 @@ class FSAssetsHandler(FSHandler): return Path(os.path.join(ASSETS_BASE_PATH, path)).mkdir(parents=True, exist_ok=True) - log.info(f" - Uploading {hl(file.filename)}") file_location = os.path.join(ASSETS_BASE_PATH, path, file.filename) with open(file_location, "wb") as f: diff --git a/backend/handler/filesystem/firmware_handler.py b/backend/handler/filesystem/firmware_handler.py index 84d89ed16..a55f781be 100644 --- a/backend/handler/filesystem/firmware_handler.py +++ b/backend/handler/filesystem/firmware_handler.py @@ -90,7 +90,6 @@ class FSFirmwareHandler(FSHandler): return Path(path).mkdir(parents=True, exist_ok=True) - log.info(f" - Uploading {hl(file.filename)}") file_location = os.path.join(path, file.filename) with open(file_location, "wb") as f: diff --git a/backend/handler/scan_handler.py b/backend/handler/scan_handler.py index d327fa299..80d276daa 100644 --- a/backend/handler/scan_handler.py +++ b/backend/handler/scan_handler.py @@ -150,8 +150,6 @@ def scan_firmware( ) -> Firmware: firmware_path = fs_firmware_handler.get_firmware_fs_structure(platform.fs_slug) - log.info(f"\t · {hl(file_name)}") - # Set default properties firmware_attrs = { "id": firmware.id if firmware else None, @@ -357,8 +355,6 @@ async def scan_rom( def _scan_asset(file_name: str, path: str): - log.info(f"\t · {hl(file_name)}") - file_size = fs_asset_handler.get_asset_size(file_name=file_name, asset_path=path) return { diff --git a/backend/scheduler.py b/backend/scheduler.py index e6d4bad6a..ec9385abd 100644 --- a/backend/scheduler.py +++ b/backend/scheduler.py @@ -4,7 +4,6 @@ from config import ( ENABLE_SCHEDULED_UPDATE_SWITCH_TITLEDB, SENTRY_DSN, ) -from logger.formatter import highlight as hl from logger.logger import log from tasks.scan_library import scan_library_task from tasks.tasks import tasks_scheduler diff --git a/backend/watcher.py b/backend/watcher.py index 90209264a..213345a2b 100644 --- a/backend/watcher.py +++ b/backend/watcher.py @@ -72,7 +72,7 @@ class EventHandler(FileSystemEventHandler): return time_delta = timedelta(minutes=RESCAN_ON_FILESYSTEM_CHANGE_DELAY) - rescan_in_msg = f"rescanning in {hl(RESCAN_ON_FILESYSTEM_CHANGE_DELAY, color=CYAN)} minutes." + rescan_in_msg = f"rescanning in {hl(str(RESCAN_ON_FILESYSTEM_CHANGE_DELAY), color=CYAN)} minutes." # Any change to a platform directory should trigger a full rescan if event.is_directory and event_src.count("/") == 1: diff --git a/backend/worker.py b/backend/worker.py index ab12b40be..e6e0ba621 100644 --- a/backend/worker.py +++ b/backend/worker.py @@ -1,9 +1,6 @@ -import logging - import sentry_sdk from config import SENTRY_DSN from handler.redis_handler import redis_client -from logger.formatter import common_date_format, common_log_format from rq import Queue, Worker from utils import get_version From b5ca8b437b002d6e85cacdec56003c3a49920956 Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 13:08:55 +0000 Subject: [PATCH 25/32] refactor: enhance logging by adding module name context and improving message formatting --- backend/endpoints/sockets/scan.py | 2 +- backend/handler/scan_handler.py | 22 ++++++++++++++++------ backend/logger/formatter.py | 7 ++++--- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/backend/endpoints/sockets/scan.py b/backend/endpoints/sockets/scan.py index c86dc669a..6cabdda0f 100644 --- a/backend/endpoints/sockets/scan.py +++ b/backend/endpoints/sockets/scan.py @@ -210,7 +210,7 @@ async def scan_platforms( for p in purged_platforms: log.warning(f" - {p.slug}") - log.info(emoji.emojize(" :check_mark: Scan completed ")) + log.info(emoji.emojize(":check_mark: Scan completed ")) await sm.emit("scan:done", scan_stats.__dict__) except ScanStoppedException: await stop_scan() diff --git a/backend/handler/scan_handler.py b/backend/handler/scan_handler.py index 80d276daa..4ae284bb5 100644 --- a/backend/handler/scan_handler.py +++ b/backend/handler/scan_handler.py @@ -20,6 +20,8 @@ from models.platform import Platform from models.rom import Rom from models.user import User +LOGGER_MODULE_NAME = {"module_name": "scan"} + class ScanType(Enum): NEW_PLATFORMS = "new_platforms" @@ -81,7 +83,8 @@ async def scan_platform( # Sometimes users change the name of the folder, so we try to match it with the config if fs_slug not in fs_platforms: log.warning( - f"{hl(fs_slug)} not found in file system, trying to match via config" + f"{hl(fs_slug)} not found in file system, trying to match via config", + extra=LOGGER_MODULE_NAME, ) if fs_slug in swapped_platform_bindings.keys(): platform = db_platform_handler.get_platform_by_fs_slug(fs_slug) @@ -131,13 +134,15 @@ async def scan_platform( log.info( emoji.emojize( f"Folder {hl(platform_attrs['fs_slug'])}[{hl(fs_slug)}] identified as {hl(platform_attrs['name'], color=BLUE)} :video_game:" - ) + ), + extra={"module_name": "scan"}, ) else: log.warning( emoji.emojize( f"Platform {hl(platform_attrs['slug'])} not identified :cross_mark:" - ) + ), + extra=LOGGER_MODULE_NAME, ) return Platform(**platform_attrs) @@ -338,18 +343,23 @@ async def scan_rom( and not ss_handler_rom.get("ss_id") ): log.warning( - emoji.emojize(f"{hl(rom_attrs['fs_name'])} not identified :cross_mark:") + emoji.emojize(f"{hl(rom_attrs['fs_name'])} not identified :cross_mark:"), + extra=LOGGER_MODULE_NAME, ) return Rom(**rom_attrs) log.info( emoji.emojize( f"{hl(rom_attrs['fs_name'])} identified as {hl(rom_attrs['name'], color=BLUE)} :alien_monster:" - ) + ), + extra=LOGGER_MODULE_NAME, ) if fs_rom.get("multi", False): for file in fs_rom["files"]: - log.info(f"\t · {hl(file.file_name, color=LIGHTYELLOW)}") + log.info( + f"\t · {hl(file.file_name, color=LIGHTYELLOW)}", + extra=LOGGER_MODULE_NAME, + ) return Rom(**rom_attrs) diff --git a/backend/logger/formatter.py b/backend/logger/formatter.py index 3910be5d0..df7e6bfe5 100644 --- a/backend/logger/formatter.py +++ b/backend/logger/formatter.py @@ -14,6 +14,7 @@ LIGHTMAGENTA = Fore.LIGHTMAGENTA_EX RESET = Fore.RESET RESET_ALL = Style.RESET_ALL +# TODO: use this for internal worker logs common_log_format = f"{GREEN}INFO{RESET}:\t {BLUE}[RomM]{LIGHTMAGENTA}[worker]{CYAN}[%(asctime)s] {RESET_ALL}%(message)s" common_date_format = "%Y-%m-%d %H:%M:%S" @@ -51,17 +52,17 @@ class Formatter(logging.Formatter): level = "%(levelname)s" dots = f"{RESET}:" identifier = ( - f"\t {BLUE}[RomM]{LIGHTMAGENTA}[{record.module.lower()}]" + f"\t {BLUE}[RomM]{LIGHTMAGENTA}[{record.module_name.lower()}]" if hasattr(record, "module_name") else f"\t {BLUE}[RomM]{LIGHTMAGENTA}[%(module)s]" ) identifier_warning = ( - f" {BLUE}[RomM]{LIGHTMAGENTA}[{record.module.lower()}]" + f" {BLUE}[RomM]{LIGHTMAGENTA}[{record.module_name.lower()}]" if hasattr(record, "module_name") else f" {BLUE}[RomM]{LIGHTMAGENTA}[%(module)s]" ) identifier_critical = ( - f" {BLUE}[RomM]{LIGHTMAGENTA}[{record.module.lower()}]" + f" {BLUE}[RomM]{LIGHTMAGENTA}[{record.module_name.lower()}]" if hasattr(record, "module_name") else f" {BLUE}[RomM]{LIGHTMAGENTA}[%(module)s]" ) From c4b9ecd3a6854ac0900efe0b154cdfcddd475b65 Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 13:35:34 +0000 Subject: [PATCH 26/32] refactor: update logger setup for improved clarity and consistency in worker logging --- backend/logger/logger.py | 2 +- backend/worker.py | 24 +++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/backend/logger/logger.py b/backend/logger/logger.py index c63189366..330284274 100644 --- a/backend/logger/logger.py +++ b/backend/logger/logger.py @@ -5,7 +5,7 @@ from config import LOGLEVEL from logger.formatter import Formatter # Set up logger -log = logging.getLogger("romm") +log = logging.getLogger() log.setLevel(LOGLEVEL) # Define stdout handler diff --git a/backend/worker.py b/backend/worker.py index e6e0ba621..cbccd47e1 100644 --- a/backend/worker.py +++ b/backend/worker.py @@ -1,9 +1,28 @@ +import logging + import sentry_sdk from config import SENTRY_DSN from handler.redis_handler import redis_client +from logger.logger import log from rq import Queue, Worker from utils import get_version +# Get the rq.worker logger +rq_logger = logging.getLogger("rq.worker") + +# Set its level (optional; you can match your app's LOGLEVEL if you want) +rq_logger.setLevel(log.level) + +# Apply the same formatter to rq.worker handlers +if not rq_logger.hasHandlers(): + # You can reuse the same handler as your app logger OR create a new one + for handler in log.handlers: + rq_logger.addHandler(handler) +else: + # If rq.worker already has handlers, just update their formatter + for handler in rq_logger.handlers: + handler.setFormatter(log.handlers[0].formatter) + listen = ("high", "default", "low") sentry_sdk.init( @@ -11,11 +30,6 @@ sentry_sdk.init( release=f"romm@{get_version()}", ) -# TODO: setup custom logger for background workers -# Set up custom logging for Worker logging -# logging.basicConfig(format=common_log_format, datefmt=common_date_format) - if __name__ == "__main__": - # Start the worker worker = Worker([Queue(name, connection=redis_client) for name in listen]) worker.work() From 1b38607eab85d33bd593e96268edb9a40b6bcd84 Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 15:23:32 +0000 Subject: [PATCH 27/32] refactor: update Alembic logging configuration and add ASCII art banner to init script --- backend/alembic.ini | 37 +------------------------------------ backend/alembic/env.py | 16 ++++++++++++---- backend/worker.py | 7 +------ docker/init_scripts/init | 9 +++++++++ 4 files changed, 23 insertions(+), 46 deletions(-) diff --git a/backend/alembic.ini b/backend/alembic.ini index 051854d0b..3944cdf1d 100644 --- a/backend/alembic.ini +++ b/backend/alembic.ini @@ -63,7 +63,7 @@ version_path_separator = os # Use os.pathsep. Default configuration used for ne sqlalchemy.url = '' -[post_write_hooks] +; [post_write_hooks] # post_write_hooks defines scripts or Python functions that are run # on newly generated revision scripts. See the documentation for further # detail and examples @@ -73,38 +73,3 @@ sqlalchemy.url = '' # black.type = console_scripts # black.entrypoint = black # black.options = -l 79 REVISION_SCRIPT_FILENAME - -# Logging configuration -[loggers] -keys = root,sqlalchemy,alembic - -[handlers] -keys = console - -[formatters] -keys = generic - -[logger_root] -level = WARN -handlers = console -qualname = - -[logger_sqlalchemy] -level = WARN -handlers = -qualname = sqlalchemy.engine - -[logger_alembic] -level = INFO -handlers = -qualname = alembic - -[handler_console] -class = StreamHandler -args = (sys.stderr,) -level = NOTSET -formatter = generic - -[formatter_generic] -format = %(levelname)s: [RomM][%(name)s] %(message)s -datefmt = %H:%M:%S diff --git a/backend/alembic/env.py b/backend/alembic/env.py index d99ef2fb5..110a5ddad 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -1,9 +1,11 @@ +import logging import sys from logging.config import fileConfig from pathlib import Path from alembic import context from config.config_manager import ConfigManager +from logger.logger import log from models.assets import Save, Screenshot, State # noqa from models.base import BaseModel from models.collection import VirtualCollection @@ -17,10 +19,16 @@ from sqlalchemy import create_engine # access to the values within the .ini file in use. config = context.config -# Interpret the config file for Python logging. -# This line sets up loggers basically. -if config.config_file_name is not None: - fileConfig(config.config_file_name, disable_existing_loggers=False) +# logger formatting +alembic_logger = logging.getLogger("alembic") +alembic_logger.setLevel(log.level) + +if not alembic_logger.hasHandlers(): + for handler in log.handlers: + alembic_logger.addHandler(handler) +else: + for handler in alembic_logger.handlers: + handler.setFormatter(log.handlers[0].formatter) # add your model's MetaData object here # for 'autogenerate' support diff --git a/backend/worker.py b/backend/worker.py index cbccd47e1..83eabfaf1 100644 --- a/backend/worker.py +++ b/backend/worker.py @@ -7,19 +7,14 @@ from logger.logger import log from rq import Queue, Worker from utils import get_version -# Get the rq.worker logger +# logger formatting rq_logger = logging.getLogger("rq.worker") - -# Set its level (optional; you can match your app's LOGLEVEL if you want) rq_logger.setLevel(log.level) -# Apply the same formatter to rq.worker handlers if not rq_logger.hasHandlers(): - # You can reuse the same handler as your app logger OR create a new one for handler in log.handlers: rq_logger.addHandler(handler) else: - # If rq.worker already has handlers, just update their formatter for handler in rq_logger.handlers: handler.setFormatter(log.handlers[0].formatter) diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 1ad6b7d13..53e10538b 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -205,6 +205,15 @@ shutdown() { stop_process_pid valkey-server } +# ASCII art banner +info_log " _____ __ __ " +info_log '| __ \ | \/ |' +info_log '| |__) |___ _ __ ___ | \ / |' +info_log "| _ // _ \\| '_ \` _ \\| |\\/| |" +info_log '| | \ \ (_) | | | | | | | | |' +info_log '|_| \_\___/|_| |_| |_|_| |_|' +info_log " " + # switch to backend directory cd /backend || { error_log "/backend directory doesn't seem to exist"; } From a9ac01cd077e85b805a2c3e7e915a06bca41f756 Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 15:33:45 +0000 Subject: [PATCH 28/32] refactor: update logging configuration for Alembic and RQ worker to unify logger formatting and level --- .trunk/trunk.yaml | 18 +++++++++--------- backend/alembic/env.py | 14 ++------------ backend/logger/logger.py | 18 ++++++++++++++++++ backend/worker.py | 13 ++----------- 4 files changed, 31 insertions(+), 32 deletions(-) diff --git a/.trunk/trunk.yaml b/.trunk/trunk.yaml index 65e033f98..8255f911c 100644 --- a/.trunk/trunk.yaml +++ b/.trunk/trunk.yaml @@ -2,7 +2,7 @@ # To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml version: 0.1 cli: - version: 1.22.12 + version: 1.22.15 # Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins) plugins: sources: @@ -21,25 +21,25 @@ lint: - pyright enabled: - markdownlint@0.44.0 - - eslint@9.25.1 + - eslint@9.26.0 - actionlint@1.7.7 - bandit@1.8.3 - black@25.1.0 - - checkov@3.2.408 + - checkov@3.2.416 - git-diff-check - isort@6.0.1 - mypy@1.15.0 - - osv-scanner@2.0.1 - - oxipng@9.1.4 + - osv-scanner@2.0.2 + - oxipng@9.1.5 - prettier@3.5.3 - - ruff@0.11.6 + - ruff@0.11.8 - shellcheck@0.10.0 - shfmt@3.6.0 - svgo@3.3.2 - taplo@0.9.3 - - trivy@0.61.1 - - trufflehog@3.88.25 - - yamllint@1.37.0 + - trivy@0.62.1 + - trufflehog@3.88.29 + - yamllint@1.37.1 ignore: - linters: [ALL] paths: diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 110a5ddad..97ccc6ee2 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -1,11 +1,10 @@ import logging import sys -from logging.config import fileConfig from pathlib import Path from alembic import context from config.config_manager import ConfigManager -from logger.logger import log +from logger.logger import unify_logger from models.assets import Save, Screenshot, State # noqa from models.base import BaseModel from models.collection import VirtualCollection @@ -19,16 +18,7 @@ from sqlalchemy import create_engine # access to the values within the .ini file in use. config = context.config -# logger formatting -alembic_logger = logging.getLogger("alembic") -alembic_logger.setLevel(log.level) - -if not alembic_logger.hasHandlers(): - for handler in log.handlers: - alembic_logger.addHandler(handler) -else: - for handler in alembic_logger.handlers: - handler.setFormatter(log.handlers[0].formatter) +unify_logger("alembic") # add your model's MetaData object here # for 'autogenerate' support diff --git a/backend/logger/logger.py b/backend/logger/logger.py index 330284274..166568b23 100644 --- a/backend/logger/logger.py +++ b/backend/logger/logger.py @@ -16,3 +16,21 @@ if not log.hasHandlers(): # Hush passlib warnings logging.getLogger("passlib").setLevel(logging.ERROR) + + +def unify_logger(logger: str) -> None: + """ + Unify the logger to use the same format and level as the main logger. + + Args: + logger (str): The name of the logger to unify. + """ + alembic_logger = logging.getLogger(logger) + alembic_logger.setLevel(log.level) + + if not alembic_logger.hasHandlers(): + for handler in log.handlers: + alembic_logger.addHandler(handler) + else: + for handler in alembic_logger.handlers: + handler.setFormatter(log.handlers[0].formatter) diff --git a/backend/worker.py b/backend/worker.py index 83eabfaf1..f04843958 100644 --- a/backend/worker.py +++ b/backend/worker.py @@ -3,20 +3,11 @@ import logging import sentry_sdk from config import SENTRY_DSN from handler.redis_handler import redis_client -from logger.logger import log +from logger.logger import log, unify_logger from rq import Queue, Worker from utils import get_version -# logger formatting -rq_logger = logging.getLogger("rq.worker") -rq_logger.setLevel(log.level) - -if not rq_logger.hasHandlers(): - for handler in log.handlers: - rq_logger.addHandler(handler) -else: - for handler in rq_logger.handlers: - handler.setFormatter(log.handlers[0].formatter) +unify_logger("rq.worker") listen = ("high", "default", "low") From 693f3e038b9f15d0597212430b3e17aa8092ab5a Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 15:55:58 +0000 Subject: [PATCH 29/32] refactor: add ASCII art banner to init script and improve debug log output --- docker/init_scripts/init | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/docker/init_scripts/init b/docker/init_scripts/init index 53e10538b..b005b99c1 100755 --- a/docker/init_scripts/init +++ b/docker/init_scripts/init @@ -28,8 +28,20 @@ BLUE='\033[0;34m' CYAN='\033[0;36m' RESET='\033[0;00m' -# print debug log output if enabled +print_banner() { + info_log " _____ __ __ " + info_log ' | __ \ | \/ |' + info_log ' | |__) |___ _ __ ___ | \ / |' + info_log " | _ // _ \\| '_ \` _ \\| |\\/| |" + info_log ' | | \ \ (_) | | | | | | | | |' + info_log ' |_| \_\___/|_| |_| |_|_| |_|' + info_log "" + info_log "The beautiful, powerful, self-hosted Rom Manager and player" + info_log "" +} + debug_log() { + # print debug log output if enabled if [[ ${LOGLEVEL} == "debug" ]]; then echo -e "${LIGHTMAGENTA}DEBUG: ${BLUE}[RomM]${LIGHTMAGENTA}[init]${CYAN}[$(date +"%Y-%m-%d %T")]${RESET}" "${@}" || true fi @@ -205,14 +217,7 @@ shutdown() { stop_process_pid valkey-server } -# ASCII art banner -info_log " _____ __ __ " -info_log '| __ \ | \/ |' -info_log '| |__) |___ _ __ ___ | \ / |' -info_log "| _ // _ \\| '_ \` _ \\| |\\/| |" -info_log '| | \ \ (_) | | | | | | | | |' -info_log '|_| \_\___/|_| |_| |_|_| |_|' -info_log " " +print_banner # switch to backend directory cd /backend || { error_log "/backend directory doesn't seem to exist"; } From 4b1fa77489c77b40daa9096384a133b9105c7fb2 Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 15:59:37 +0000 Subject: [PATCH 30/32] refactor: comment out post_write_hooks section in alembic.ini for clarity --- backend/alembic.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/alembic.ini b/backend/alembic.ini index 3944cdf1d..78c10646c 100644 --- a/backend/alembic.ini +++ b/backend/alembic.ini @@ -63,7 +63,7 @@ version_path_separator = os # Use os.pathsep. Default configuration used for ne sqlalchemy.url = '' -; [post_write_hooks] +# [post_write_hooks] # post_write_hooks defines scripts or Python functions that are run # on newly generated revision scripts. See the documentation for further # detail and examples From e39825ba83c511c25399abc02bec4750758a714b Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 16:01:16 +0000 Subject: [PATCH 31/32] refactor: remove unused logging import from worker.py --- backend/worker.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/backend/worker.py b/backend/worker.py index f04843958..2c58fbd87 100644 --- a/backend/worker.py +++ b/backend/worker.py @@ -1,9 +1,7 @@ -import logging - import sentry_sdk from config import SENTRY_DSN from handler.redis_handler import redis_client -from logger.logger import log, unify_logger +from logger.logger import unify_logger from rq import Queue, Worker from utils import get_version From 7f4f54795c3601f8eb69b6c5101627fd299266a5 Mon Sep 17 00:00:00 2001 From: zurdi Date: Fri, 9 May 2025 16:12:15 +0000 Subject: [PATCH 32/32] refactor: remove unused logging import from env.py --- backend/alembic/env.py | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 97ccc6ee2..9fed12b77 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -1,4 +1,3 @@ -import logging import sys from pathlib import Path