touchup igdb metadata fetching

This commit is contained in:
Georges-Antoine Assi
2024-02-06 10:30:18 -05:00
parent b6bd999c29
commit 4bca93f699
14 changed files with 252 additions and 335 deletions

View File

@@ -3,17 +3,19 @@ from typing import Optional
from endpoints.responses.assets import SaveSchema, ScreenshotSchema, StateSchema
from fastapi.responses import StreamingResponse
from handler import socket_handler
from handler.igdb_handler import IGDBRomExpansion, IGDBDlc, IGDBRelatedGame
from handler.igdb_handler import IGDBRelatedGame
from pydantic import BaseModel
from typing_extensions import TypedDict, NotRequired
class RomMetadata(TypedDict):
expansions: NotRequired[list[IGDBRomExpansion]]
dlcs: NotRequired[list[IGDBDlc]]
expansions: NotRequired[list[IGDBRelatedGame]]
dlcs: NotRequired[list[IGDBRelatedGame]]
remasters: NotRequired[list[IGDBRelatedGame]]
remakes: NotRequired[list[IGDBRelatedGame]]
expanded_games: NotRequired[list[IGDBRelatedGame]]
ports: list[IGDBRelatedGame]
similar_games: list[IGDBRelatedGame]
class RomSchema(BaseModel):
@@ -38,11 +40,14 @@ class RomSchema(BaseModel):
# Metadata fields
total_rating: Optional[str]
aggregated_rating: Optional[str]
first_release_date: Optional[int]
alternative_names: list[str]
genres: list[str]
franchises: list[str]
collections: list[str]
companies: list[str]
game_modes: list[str]
igdb_metadata: Optional[RomMetadata]
# Used for sorting on the frontend

View File

@@ -1,4 +1,3 @@
import datetime
import os
import shutil
from pathlib import Path
@@ -102,7 +101,6 @@ class FSResourceHandler(FSHandler):
file_name: name of rom file
size: size of the cover
"""
strtime = str(datetime.datetime.now().timestamp())
return f"{fs_slug}/{rom_name}/cover/{size.value}.png"
def get_rom_cover(
@@ -139,7 +137,6 @@ class FSResourceHandler(FSHandler):
@staticmethod
def build_artwork_path(rom_name: str, fs_slug: str, file_ext: str):
q_rom_name = quote(rom_name)
strtime = str(datetime.datetime.now().timestamp())
path_cover_l = f"{fs_slug}/{q_rom_name}/cover/{CoverSize.BIG.value}.{file_ext}"
path_cover_s = f"{fs_slug}/{q_rom_name}/cover/{CoverSize.SMALL.value}.{file_ext}"

View File

@@ -43,33 +43,37 @@ SWITCH_PRODUCT_ID_FILE: Final = os.path.join(
MAME_XML_FILE: Final = os.path.join(os.path.dirname(__file__), "fixtures", "mame.xml")
class IGDBRelatedGame(TypedDict):
class IGDBPlatform(TypedDict):
igdb_id: int
name: str
class IGDBRelatedGame(TypedDict):
id: int
name: str
slug: str
type: str
cover_url: str
class IGDBRomExpansion(IGDBRelatedGame):
id: int
slug: str
class IGDBDlc(IGDBRelatedGame):
slug: str
class IGDBMetadata(TypedDict):
total_rating: str
aggregated_rating: str
first_release_date: int | None
genres: list[str]
franchises: list[str]
alternative_names: list[str]
collections: list[str]
companies: list[str]
expansions: list[IGDBRomExpansion]
dlcs: list[IGDBDlc]
game_modes: list[str]
platforms: list[IGDBPlatform]
expansions: list[IGDBRelatedGame]
dlcs: list[IGDBRelatedGame]
remasters: list[IGDBRelatedGame]
remakes: list[IGDBRelatedGame]
expanded_games: list[IGDBRelatedGame]
ports: list[IGDBRelatedGame]
similar_games: list[IGDBRelatedGame]
class IGDBRom(TypedDict):
@@ -82,19 +86,25 @@ class IGDBRom(TypedDict):
igdb_metadata: Optional[IGDBMetadata]
class IGDBPlatform(TypedDict):
igdb_id: int
name: str
def extract_metadata_from_igdb_rom(rom: dict) -> IGDBMetadata:
return IGDBMetadata(
{
"total_rating": str(round(rom.get("total_rating", 0.0), 2)),
"aggregated_rating": str(round(rom.get("aggregated_rating", 0.0), 2)),
"first_release_date": rom.get("first_release_date", None),
"genres": pydash.map_(rom.get("genres", []), "name"),
"franchises": pydash.map_(rom.get("franchises", []), "name"),
"franchises": pydash.compact(
[rom.get("franchise.name", None)]
+ pydash.map_(rom.get("franchises", []), "name")
),
"alternative_names": pydash.map_(rom.get("alternative_names", []), "name"),
"collections": pydash.map_(rom.get("collections", []), "name"),
"game_modes": pydash.map_(rom.get("game_modes", []), "name"),
"companies": pydash.map_(rom.get("involved_companies", []), "company.name"),
"platforms": [
{"igdb_id": p.get("id", ""), "name": p.get("name", "")}
for p in rom.get("platforms", [])
],
"expansions": [
{"cover_url": pydash.get(e, "cover.url", ""), "type": "expansion", **e}
for e in rom.get("expansions", [])
@@ -115,7 +125,14 @@ def extract_metadata_from_igdb_rom(rom: dict) -> IGDBMetadata:
{"cover_url": pydash.get(g, "cover.url", ""), "type": "expanded", **g}
for g in rom.get("expanded_games", [])
],
"companies": pydash.map_(rom.get("involved_companies", []), "company.name"),
"ports": [
{"cover_url": pydash.get(p, "cover.url", ""), "type": "port", **p}
for p in rom.get("ports", [])
],
"similar_games": [
{"cover_url": pydash.get(s, "cover.url", ""), "type": "similar", **s}
for s in rom.get("similar_games", [])
],
}
)
@@ -562,34 +579,45 @@ GAMES_FIELDS = [
"summary",
"total_rating",
"aggregated_rating",
"genres.name",
"alternative_names.name",
"first_release_date",
"artworks.url",
"cover.url",
"screenshots.url",
"platforms.id",
"platforms.name",
"alternative_names.name",
"genres.name",
"franchise.name",
"franchises.name",
"collections.name",
"expansions.name",
"game_modes.name",
"involved_companies.company.name",
"expansions.id",
"expansions.slug",
"expansions.name",
"expansions.cover.url",
"expanded_games.id",
"expanded_games.slug",
"expanded_games.name",
"expanded_games.cover.url",
"dlcs.id",
"dlcs.name",
"dlcs.slug",
"dlcs.cover.url",
"remakes.id",
"remakes.slug",
"remakes.name",
"remakes.cover.url",
"remasters.id",
"remasters.slug",
"remasters.name",
"remasters.cover.url",
"involved_companies.company.name",
"platforms.name",
"first_release_date",
"game_modes.name",
"player_perspectives.name",
"ports.id",
"ports.slug",
"ports.name",
"ports.cover.url",
"similar_games.id",
"similar_games.slug",
"similar_games.name",
"language_supports.language.name",
"external_games.uid",
"external_games.category",
"similar_games.cover.url",
]

View File

@@ -42,7 +42,7 @@ class Rom(BaseModel):
path_cover_s: str = Column(Text, default="")
path_cover_l: str = Column(Text, default="")
url_cover: str = Column(Text, default="")
url_cover: str = Column(Text, default="", doc="URL to cover image stored in IGDB")
revision: str = Column(String(20))
regions: JSON = Column(JSON, default=[])
@@ -50,7 +50,7 @@ class Rom(BaseModel):
tags: JSON = Column(JSON, default=[])
path_screenshots: JSON = Column(JSON, default=[])
url_screenshots: JSON = Column(JSON, default=[])
url_screenshots: JSON = Column(JSON, default=[], doc="URLs to screenshots stored in IGDB")
multi: bool = Column(Boolean, default=False)
files: JSON = Column(JSON, default=[])
@@ -138,6 +138,14 @@ class Rom(BaseModel):
def total_rating(self) -> str:
return self.igdb_metadata.get("total_rating", "")
@property
def aggregated_rating(self) -> str:
return self.igdb_metadata.get("aggregated_rating", "")
@property
def alternative_names(self) -> list[str]:
return self.igdb_metadata.get("alternative_names", [])
@property
def first_release_date(self) -> int:
return self.igdb_metadata.get("first_release_date", 0)
@@ -157,6 +165,10 @@ class Rom(BaseModel):
@property
def companies(self) -> list[str]:
return self.igdb_metadata.get("companies", [])
@property
def game_modes(self) -> list[str]:
return self.igdb_metadata.get("game_modes", [])
def __repr__(self) -> str:
return self.file_name

View File

@@ -1,319 +1,197 @@
{
"id":191930,
"aggregated_rating":80.0,
"alternative_names":[
"id": 191930,
"name": "Pokémon Violet",
"slug": "pokemon-violet",
"summary": "The Pokémon Scarlet and Pokémon Violet games, the newest chapters in the Pokémon series, are coming to Nintendo Switch later this year. With these new titles, the Pokémon series takes a new evolutionary step, allowing you to explore freely in a richly expressed open world.\n\nVarious towns blend seamlessly into the wilderness with no borders. Youll be able to see the Pokémon of this region in the skies, in the seas, in the forests, on the streets—all over! Youll be able to experience the true joy of the Pokémon series—battling against wild Pokémon in order to catch them—now in an open-world game that players of any age can enjoy.\u200b",
"total_rating": 75.10255180007994,
"aggregated_rating": 80.0,
"first_release_date": 1668729600,
"alternative_names": [
{
"id":90678,
"name":"포켓몬스터바이올렛"
"id": 90678,
"name": "포켓몬스터바이올렛"
},
{
"id":107879,
"name":"ポケットモンスター バイオレット"
"id": 107879,
"name": "ポケットモンスター バイオレット"
},
{
"id":118572,
"name":"Pokémon Purpur"
"id": 118572,
"name": "Pokémon Purpur"
},
{
"id":118574,
"name":"Pokemon Violet"
"id": 118574,
"name": "Pokemon Violet"
},
{
"id":120436,
"name":"Pokémon Púrpura"
"id": 120436,
"name": "Pokémon Púrpura"
},
{
"id":137600,
"name":"Pocket Monsters Violet"
"id": 137600,
"name": "Pocket Monsters Violet"
}
],
"artworks":[
"artworks": [
{
"id":74870,
"url":"//images.igdb.com/igdb/image/upload/t_thumb/ar1lrq.jpg"
"id": 74870,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/ar1lrq.jpg"
}
],
"cover":{
"id":270118,
"url":"//images.igdb.com/igdb/image/upload/t_thumb/co5sfa.jpg"
"cover": {
"id": 270118,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/co5sfa.jpg"
},
"expansions":[
"screenshots": [
{
"id":239930,
"cover":{
"id":307621,
"url":"//images.igdb.com/igdb/image/upload/t_thumb/co6ld1.jpg"
},
"name":"Pokémon Violet: The Hidden Treasure of Area Zero - Part 1: The Teal Mask"
"id": 740299,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/scfv7v.jpg"
},
{
"id":239933,
"cover":{
"id":307620,
"url":"//images.igdb.com/igdb/image/upload/t_thumb/co6ld0.jpg"
},
"name":"Pokémon Violet: The Hidden Treasure of Area Zero - Part 2: The Indigo Disk"
"id": 740300,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/scfv7w.jpg"
},
{
"id": 740301,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/scfv7x.jpg"
},
{
"id": 740302,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/scfv7y.jpg"
}
],
"external_games":[
"franchise": {
"id": 60,
"name": "Pokémon"
},
"franchises": [
{
"id":2280257,
"category":20,
"uid":"B09TTZZWW6"
},
{
"id":2328045,
"category":20,
"uid":"B09VPP3X5W"
},
{
"id":2445770,
"category":20,
"uid":"B0B2X4BMV2"
},
{
"id":2445855,
"category":20,
"uid":"B0B31X6NKB"
},
{
"id":2529113,
"category":20,
"uid":"B0B81JV184"
},
{
"id":2529198,
"category":20,
"uid":"B0B8BPG7YF"
},
{
"id":2530534,
"category":20,
"uid":"B0B8R7Q42W"
},
{
"id":2530552,
"category":20,
"uid":"B0B814NSK9"
},
{
"id":2604272,
"category":20,
"uid":"B0BFYDDFP7"
},
{
"id":2630441,
"category":20,
"uid":"B0B2X4BMV2"
},
{
"id":2630443,
"category":20,
"uid":"B0B2X4BMV2"
},
{
"id":2630444,
"category":20,
"uid":"B0B2X4BMV2"
},
{
"id":2870935,
"category":20,
"uid":"B0B31X6NKB"
"id": 60,
"name": "Pokémon"
}
],
"first_release_date":1668729600,
"franchises":[
"genres": [
{
"id":60,
"name":"Pokémon"
}
],
"game_modes":[
{
"id":1,
"name":"Single player"
"id": 12,
"name": "Role-playing (RPG)"
},
{
"id":2,
"name":"Multiplayer"
}
],
"genres":[
{
"id":12,
"name":"Role-playing (RPG)"
"id": 16,
"name": "Turn-based strategy (TBS)"
},
{
"id":16,
"name":"Turn-based strategy (TBS)"
},
{
"id":31,
"name":"Adventure"
"id": 31,
"name": "Adventure"
}
],
"involved_companies":[
"game_modes": [
{
"id":164695,
"company":{
"id":1617,
"name":"Game Freak"
"id": 1,
"name": "Single player"
},
{
"id": 2,
"name": "Multiplayer"
}
],
"platforms": [
{
"id": 130,
"slug": "switch",
"name": "Nintendo Switch"
}
],
"collections": [
{
"id": 314,
"name": "Pokémon"
}
],
"involved_companies": [
{
"id": 164695,
"company": {
"id": 1617,
"name": "Game Freak"
}
},
{
"id":164700,
"company":{
"id":70,
"name":"Nintendo"
"id": 164700,
"company": {
"id": 70,
"name": "Nintendo"
}
}
],
"name":"Pokémon Violet",
"platforms":[
"expansions": [
{
"id":130,
"name":"Nintendo Switch"
}
],
"player_perspectives":[
{
"id":2,
"name":"Third person"
}
],
"screenshots":[
{
"id":740299,
"url":"//images.igdb.com/igdb/image/upload/t_thumb/scfv7v.jpg"
},
{
"id":740300,
"url":"//images.igdb.com/igdb/image/upload/t_thumb/scfv7w.jpg"
},
{
"id":740301,
"url":"//images.igdb.com/igdb/image/upload/t_thumb/scfv7x.jpg"
},
{
"id":740302,
"url":"//images.igdb.com/igdb/image/upload/t_thumb/scfv7y.jpg"
}
],
"similar_games":[
{
"id":27092,
"name":"GreedFall"
},
{
"id":55038,
"name":"Immortal: Unchained"
},
{
"id":55199,
"name":"Dragon: Marked for Death"
},
{
"id":81249,
"name":"The Elder Scrolls VI"
},
{
"id":96217,
"name":"Eternity: The Last Unicorn"
},
{
"id":103168,
"name":"Warhammer: Chaosbane"
},
{
"id":103303,
"name":"The Elder Scrolls: Blades"
},
{
"id":106987,
"name":"Torchlight III"
},
{
"id":113360,
"name":"Hytale"
},
{
"id":115653,
"name":"Pokémon Shield"
}
],
"slug":"pokemon-violet",
"summary":"The Pokémon Scarlet and Pokémon Violet games, the newest chapters in the Pokémon series, are coming to Nintendo Switch later this year. With these new titles, the Pokémon series takes a new evolutionary step, allowing you to explore freely in a richly expressed open world.\n\nVarious towns blend seamlessly into the wilderness with no borders. Youll be able to see the Pokémon of this region in the skies, in the seas, in the forests, on the streets—all over! Youll be able to experience the true joy of the Pokémon series—battling against wild Pokémon in order to catch them—now in an open-world game that players of any age can enjoy.\u200b",
"total_rating":75.10255180007994,
"language_supports":[
{
"id":349083,
"language":{
"id":7,
"name":"English"
"id": 239930,
"slug": "pokemon-violet-the-hidden-treasure-of-area-zero-part-1-the-teal-mask",
"name": "Pokémon Violet: The Hidden Treasure of Area Zero - Part 1: The Teal Mask",
"cover": {
"id": 307621,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/co6ld1.jpg"
}
},
{
"id":349084,
"language":{
"id":12,
"name":"French"
}
},
{
"id":349086,
"language":{
"id":27,
"name":"German"
}
},
{
"id":349088,
"language":{
"id":15,
"name":"Italian"
}
},
{
"id":349090,
"language":{
"id":16,
"name":"Japanese"
}
},
{
"id":349092,
"language":{
"id":17,
"name":"Korean"
}
},
{
"id":349093,
"language":{
"id":2,
"name":"Chinese (Simplified)"
}
},
{
"id":349094,
"language":{
"id":9,
"name":"Spanish (Spain)"
}
},
{
"id":349095,
"language":{
"id":3,
"name":"Chinese (Traditional)"
"id": 239933,
"slug": "pokemon-violet-the-hidden-treasure-of-area-zero-part-2-the-indigo-disk",
"name": "Pokémon Violet: The Hidden Treasure of Area Zero - Part 2: The Indigo Disk",
"cover": {
"id": 307620,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/co6ld0.jpg"
}
}
],
"collections":[
"expanded_games": [],
"dlcs": [],
"remakes": [],
"remasters": [],
"similar_games": [
{
"id":314,
"name":"Pokémon"
"id": 27092,
"slug": "greedfall",
"name": "GreedFall",
"cover": {
"id": 270119,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/co5sfb.jpg"
}
},
{
"id": 55038,
"slug": "immortal-unchained",
"name": "Immortal: Unchained",
"cover": {
"id": 270120,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/co5sfc.jpg"
}
},
{
"id": 55199,
"slug": "dragon-marked-for-death",
"name": "Dragon: Marked for Death",
"cover": {
"id": 270121,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/co5sfd.jpg"
}
},
{
"id": 81249,
"slug": "the-elder-scrolls-vi",
"name": "The Elder Scrolls VI",
"cover": {
"id": 270122,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/co5sfe.jpg"
}
},
{
"id": 96217,
"slug": "eternity-the-last-unicorn",
"name": "Eternity: The Last Unicorn",
"cover": {
"id": 270123,
"url": "//images.igdb.com/igdb/image/upload/t_thumb/co5sff.jpg"
}
}
]
}
}

View File

@@ -16,10 +16,9 @@ export type { CursorPage_RomSchema_ } from './models/CursorPage_RomSchema_';
export type { EnhancedRomSchema } from './models/EnhancedRomSchema';
export type { HeartbeatResponse } from './models/HeartbeatResponse';
export type { HTTPValidationError } from './models/HTTPValidationError';
export type { IGDBDlc } from './models/IGDBDlc';
export type { IGDBMetadata } from './models/IGDBMetadata';
export type { IGDBPlatform } from './models/IGDBPlatform';
export type { IGDBRelatedGame } from './models/IGDBRelatedGame';
export type { IGDBRomExpansion } from './models/IGDBRomExpansion';
export type { MessageResponse } from './models/MessageResponse';
export type { PlatformSchema } from './models/PlatformSchema';
export type { Role } from './models/Role';

View File

@@ -26,11 +26,14 @@ export type EnhancedRomSchema = {
slug: (string | null);
summary: (string | null);
total_rating: (string | null);
aggregated_rating: (string | null);
first_release_date: (number | null);
alternative_names: Array<string>;
genres: Array<string>;
franchises: Array<string>;
collections: Array<string>;
companies: Array<string>;
game_modes: Array<string>;
igdb_metadata: (RomMetadata | null);
sort_comparator: string;
path_cover_s: (string | null);

View File

@@ -3,21 +3,26 @@
/* tslint:disable */
/* eslint-disable */
import type { IGDBDlc } from './IGDBDlc';
import type { IGDBPlatform } from './IGDBPlatform';
import type { IGDBRelatedGame } from './IGDBRelatedGame';
import type { IGDBRomExpansion } from './IGDBRomExpansion';
export type IGDBMetadata = {
total_rating: string;
aggregated_rating: string;
first_release_date: (number | null);
genres: Array<string>;
franchises: Array<string>;
alternative_names: Array<string>;
collections: Array<string>;
companies: Array<string>;
expansions: Array<IGDBRomExpansion>;
dlcs: Array<IGDBDlc>;
game_modes: Array<string>;
platforms: Array<IGDBPlatform>;
expansions: Array<IGDBRelatedGame>;
dlcs: Array<IGDBRelatedGame>;
remasters: Array<IGDBRelatedGame>;
remakes: Array<IGDBRelatedGame>;
expanded_games: Array<IGDBRelatedGame>;
ports: Array<IGDBRelatedGame>;
similar_games: Array<IGDBRelatedGame>;
};

View File

@@ -3,10 +3,8 @@
/* tslint:disable */
/* eslint-disable */
export type IGDBDlc = {
export type IGDBPlatform = {
igdb_id: number;
name: string;
type: string;
cover_url: string;
slug: string;
};

View File

@@ -4,7 +4,9 @@
/* eslint-disable */
export type IGDBRelatedGame = {
id: number;
name: string;
slug: string;
type: string;
cover_url: string;
};

View File

@@ -1,13 +0,0 @@
/* generated using openapi-typescript-codegen -- do no edit */
/* istanbul ignore file */
/* tslint:disable */
/* eslint-disable */
export type IGDBRomExpansion = {
name: string;
type: string;
cover_url: string;
id: number;
slug: string;
};

View File

@@ -3,15 +3,15 @@
/* tslint:disable */
/* eslint-disable */
import type { IGDBDlc } from './IGDBDlc';
import type { IGDBRelatedGame } from './IGDBRelatedGame';
import type { IGDBRomExpansion } from './IGDBRomExpansion';
export type RomMetadata = {
expansions?: Array<IGDBRomExpansion>;
dlcs?: Array<IGDBDlc>;
expansions?: Array<IGDBRelatedGame>;
dlcs?: Array<IGDBRelatedGame>;
remasters?: Array<IGDBRelatedGame>;
remakes?: Array<IGDBRelatedGame>;
expanded_games?: Array<IGDBRelatedGame>;
ports: Array<IGDBRelatedGame>;
similar_games: Array<IGDBRelatedGame>;
};

View File

@@ -25,11 +25,14 @@ export type RomSchema = {
slug: (string | null);
summary: (string | null);
total_rating: (string | null);
aggregated_rating: (string | null);
first_release_date: (number | null);
alternative_names: Array<string>;
genres: Array<string>;
franchises: Array<string>;
collections: Array<string>;
companies: Array<string>;
game_modes: Array<string>;
igdb_metadata: (RomMetadata | null);
sort_comparator: string;
path_cover_s: (string | null);

View File

@@ -12,6 +12,6 @@ export type SearchRomSchema = {
summary: (string | null);
url_cover: string;
url_screenshots: Array<string>;
metadata: (IGDBMetadata | null);
igdb_metadata: (IGDBMetadata | null);
};