diff --git a/backend/alembic/env.py b/backend/alembic/env.py index 4b278d2b93..409a702c33 100644 --- a/backend/alembic/env.py +++ b/backend/alembic/env.py @@ -12,6 +12,7 @@ from models.firmware import Firmware # noqa from models.music import MusicFavoriteTrack, MusicPlaylist, MusicPlaylistTrack # noqa from models.platform import Platform # noqa +from models.recommendation import RomSimilarity # noqa from models.rom import Rom, RomFacets, RomMetadata, SiblingRom # noqa from models.user import User # noqa diff --git a/backend/alembic/versions/0108_rom_similarity.py b/backend/alembic/versions/0108_rom_similarity.py new file mode 100644 index 0000000000..7930de74a6 --- /dev/null +++ b/backend/alembic/versions/0108_rom_similarity.py @@ -0,0 +1,69 @@ +"""Precomputed item-item similarity edges for the recommendations engine + +Recommendations previously came straight from IGDB's ``similar_games``, which +knows nothing about which of those games are actually in the library and is +absent entirely for anything IGDB never matched. ``rom_similarity`` holds a +library-relative similarity graph built from the normalised metadata, the IGDB +prior, collection co-membership and co-play, so both the "Similar games" +section and the personalised feed read a single indexed table. + +The table is rewritten wholesale by the recommendations task rather than +maintained incrementally, because the IDF weighting that makes the scores +library-relative shifts as the library grows. Rows are bounded at roughly +``rom_count * MAX_NEIGHBOURS``. + +Revision ID: 0108_rom_similarity +Revises: 0107_roms_dedup_cover_index +Create Date: 2026-08-07 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op # type: ignore[attr-defined] + +from utils.database import CustomJSON + +# revision identifiers, used by Alembic. +revision = "0108_rom_similarity" +down_revision = "0107_roms_dedup_cover_index" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.create_table( + "rom_similarity", + sa.Column("rom_id", sa.Integer(), nullable=False), + sa.Column("related_rom_id", sa.Integer(), nullable=False), + sa.Column("score", sa.Float(), nullable=False), + sa.Column("reasons", CustomJSON(), nullable=True), + sa.Column( + "created_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.Column( + "updated_at", + sa.TIMESTAMP(timezone=True), + server_default=sa.func.now(), + nullable=False, + ), + sa.ForeignKeyConstraint(["rom_id"], ["roms.id"], ondelete="CASCADE"), + sa.ForeignKeyConstraint(["related_rom_id"], ["roms.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("rom_id", "related_rom_id"), + # Reads are always "top N neighbours of this ROM", so the score rides + # along in the index to keep the ordering off a filesort. + sa.Index("idx_rom_similarity_rom_score", "rom_id", "score"), + # Backs the cascade: without it Postgres seq-scans this table on every + # ROM delete. Declared inline rather than via a following create_index + # so InnoDB adopts it for the foreign key instead of silently adding a + # second index on the same column. + sa.Index("idx_rom_similarity_related_rom_id", "related_rom_id"), + ) + + +def downgrade() -> None: + # Dropping the table takes its indexes and constraints with it. Dropping + # the indexes first fails on MariaDB, which needs them for the foreign keys. + op.drop_table("rom_similarity") diff --git a/backend/alembic/versions/0109_igdb_tag_columns.py b/backend/alembic/versions/0109_igdb_tag_columns.py new file mode 100644 index 0000000000..8a6c0f877d --- /dev/null +++ b/backend/alembic/versions/0109_igdb_tag_columns.py @@ -0,0 +1,240 @@ +"""Surface IGDB keywords, themes and player perspectives as facet columns + +The recommendations index scores on `roms_facets` / `roms_metadata` rather than +the raw provider blobs, so these three fields have to travel the same route as +genres and franchises: a STORED generated column on `roms`, exposed by the +`roms_metadata` view and mirrored into `roms_facets` by its triggers. + +Only IGDB supplies them (plus `manual_metadata`, so a user override still +wins), which makes the COALESCE chain much shorter than the existing facets. + +Existing libraries carry no such data until it is fetched -- the columns are +generated from the metadata blob, so they stay empty until a rescan or +`tools/backfill_igdb_tags.py` populates the source. + +Revision ID: 0109_igdb_tag_columns +Revises: 0108_rom_similarity +Create Date: 2026-08-08 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op # type: ignore[attr-defined] + +from utils.database import CustomJSON, is_postgresql + +# revision identifiers, used by Alembic. +revision = "0109_igdb_tag_columns" +down_revision = "0108_rom_similarity" +branch_labels = None +depends_on = None + + +# (generated column, roms_facets column). A user override in `manual_metadata` +# takes precedence, matching every other generated facet. +_TAG_COLUMNS = [ + ("generated_keywords", "keywords"), + ("generated_themes", "themes"), + ("generated_player_perspectives", "player_perspectives"), +] +_SOURCES = ["manual_metadata", "igdb_metadata"] + +# Every column the `roms_metadata` view projects, old and new. The view is +# replaced wholesale, so the pre-existing projections have to be restated. +_VIEW_COLUMNS = [ + ("generated_genres", "genres"), + ("generated_franchises", "franchises"), + ("generated_collections", "collections"), + ("generated_companies", "companies"), + ("generated_game_modes", "game_modes"), + ("generated_age_ratings", "age_ratings"), + ("generated_first_release_date", "first_release_date"), + ("generated_average_rating", "average_rating"), + ("generated_player_count", "player_count"), +] + _TAG_COLUMNS + + +def _maria_array_expr(column: str) -> str: + key = column[len("generated_") :] + branches = [ + f"CASE WHEN JSON_LENGTH(JSON_EXTRACT({src}, '$.{key}')) > 0 " + f"THEN JSON_EXTRACT({src}, '$.{key}') ELSE NULL END" + for src in _SOURCES + ] + branches.append("JSON_ARRAY()") + return "COALESCE(" + ", ".join(branches) + ")" + + +def _postgres_array_expr(column: str) -> str: + key = column[len("generated_") :] + branches = [f"NULLIF({src} -> '{key}', '[]'::jsonb)" for src in _SOURCES] + branches.append("'[]'::jsonb") + return "COALESCE(" + ", ".join(branches) + ")" + + +def _view_sql(is_pg: bool) -> str: + projections = ",\n ".join( + # The view exposed player_count as text and PostgreSQL cannot change a + # column's type through CREATE OR REPLACE VIEW, so the cast has to stay. + ( + f"{name}::text AS {alias}" + if is_pg and alias == "player_count" + else f"{name} AS {alias}" + ) + for name, alias in _VIEW_COLUMNS + ) + return ( + "CREATE OR REPLACE VIEW roms_metadata AS\n" # nosec B608 + "SELECT\n" + " id AS rom_id,\n" + " NOW() AS created_at,\n" + " NOW() AS updated_at,\n" + f" {projections}\n" + "FROM roms" + ) + + +def upgrade() -> None: + bind = op.get_bind() + is_pg = is_postgresql(bind) + json_type = "JSONB" if is_pg else "JSON" + + for generated, _ in _TAG_COLUMNS: + expr = ( + _postgres_array_expr(generated) if is_pg else _maria_array_expr(generated) + ) + op.execute( + f"ALTER TABLE roms ADD COLUMN {generated} {json_type} " # nosec B608 + f"GENERATED ALWAYS AS ({expr}) STORED" + ) + + op.execute(_view_sql(is_pg)) + + for _, facet in _TAG_COLUMNS: + op.add_column("roms_facets", sa.Column(facet, CustomJSON(), nullable=True)) + + # The triggers copy a fixed column list, so they are rebuilt rather than + # amended. Backfill first so existing rows are correct either way. + _backfill_facets() + _rebuild_triggers( + is_pg, _MIRRORED_COLUMNS + [(facet, gen) for gen, facet in _TAG_COLUMNS] + ) + + +def _backfill_facets() -> None: + assignments = ", ".join( + f"f.{facet} = r.{generated}" for generated, facet in _TAG_COLUMNS + ) + bind = op.get_bind() + if is_postgresql(bind): + set_clause = ", ".join( + f"{facet} = r.{generated}" for generated, facet in _TAG_COLUMNS + ) + op.execute( + f"UPDATE roms_facets f SET {set_clause} FROM roms r " # nosec B608 + "WHERE r.id = f.rom_id" + ) + else: + op.execute( + f"UPDATE roms_facets f JOIN roms r ON r.id = f.rom_id SET {assignments}" # nosec B608 + ) + + +# Mirrored into roms_facets by the triggers, matching migration 0100's list +# with the three new columns appended. +_MIRRORED_COLUMNS = [ + ("platform_id", "platform_id"), + ("genres", "generated_genres"), + ("franchises", "generated_franchises"), + ("collections", "generated_collections"), + ("companies", "generated_companies"), + ("game_modes", "generated_game_modes"), + ("age_ratings", "generated_age_ratings"), + ("player_count", "generated_player_count"), + ("regions", "regions"), + ("languages", "languages"), + ("tags", "tags"), + ("igdb_id", "igdb_id"), + ("ss_id", "ss_id"), + ("moby_id", "moby_id"), + ("launchbox_id", "launchbox_id"), + ("ra_id", "ra_id"), + ("hasheous_id", "hasheous_id"), + ("tgdb_id", "tgdb_id"), + ("flashpoint_id", "flashpoint_id"), + ("hltb_id", "hltb_id"), + ("gamelist_id", "gamelist_id"), + ("libretro_id", "libretro_id"), +] + +_MYSQL_TRIGGERS = { + "roms_facets_after_insert": "AFTER INSERT", + "roms_facets_after_update": "AFTER UPDATE", +} + + +def _rebuild_triggers(is_pg: bool, mirrored: list[tuple[str, str]]) -> None: + """Recreate the roms_facets sync triggers over the given column list. + + The triggers copy a fixed set of columns, so adding one means replacing + them wholesale rather than amending in place. + """ + targets = ", ".join(target for target, _ in mirrored) + values = ", ".join(f"NEW.{source}" for _, source in mirrored) + + if is_pg: + assignments = ", ".join( + f"{target} = EXCLUDED.{target}" for target, _ in mirrored + ) + op.execute(f""" +CREATE OR REPLACE FUNCTION romm_sync_rom_facets() RETURNS trigger + LANGUAGE plpgsql AS $$ +BEGIN + INSERT INTO roms_facets (rom_id, {targets}) + VALUES (NEW.id, {values}) + ON CONFLICT (rom_id) DO UPDATE SET + {assignments}, + updated_at = NOW(); + RETURN NULL; +END $$ +""") # nosec B608 + return + + updates = ",\n".join(f"{target} = VALUES({target})" for target, _ in mirrored) + body = ( + f"INSERT INTO roms_facets (rom_id, {targets})\n" # nosec B608 + f"VALUES (NEW.id, {values})\n" + f"ON DUPLICATE KEY UPDATE\n{updates},\nupdated_at = CURRENT_TIMESTAMP" + ) + for name, timing in _MYSQL_TRIGGERS.items(): + op.execute(f"DROP TRIGGER IF EXISTS {name}") + op.execute(f"CREATE TRIGGER {name} {timing} ON roms\nFOR EACH ROW\n{body}") + + +def downgrade() -> None: + bind = op.get_bind() + is_pg = is_postgresql(bind) + + # Drop the view's dependency on the new columns before dropping them. + remaining = [entry for entry in _VIEW_COLUMNS if entry not in _TAG_COLUMNS] + projections = ",\n ".join( + ( + f"{name}::text AS {alias}" + if is_pg and alias == "player_count" + else f"{name} AS {alias}" + ) + for name, alias in remaining + ) + op.execute("DROP VIEW IF EXISTS roms_metadata") + op.execute( + "CREATE VIEW roms_metadata AS\n" # nosec B608 + "SELECT\n id AS rom_id,\n NOW() AS created_at,\n" + f" NOW() AS updated_at,\n {projections}\nFROM roms" + ) + + for _, facet in _TAG_COLUMNS: + op.drop_column("roms_facets", facet) + for generated, _ in _TAG_COLUMNS: + op.execute(f"ALTER TABLE roms DROP COLUMN {generated}") + + _rebuild_triggers(is_pg, _MIRRORED_COLUMNS) diff --git a/backend/alembic/versions/0110_rating_count_column.py b/backend/alembic/versions/0110_rating_count_column.py new file mode 100644 index 0000000000..7752bfffde --- /dev/null +++ b/backend/alembic/versions/0110_rating_count_column.py @@ -0,0 +1,107 @@ +"""Expose IGDB's rating count so a rating can be weighted by its confidence + +`average_rating` averages whatever providers rated a game, which makes a +single ScreenScraper 10/10 indistinguishable from a broad consensus. On a real +15k library sixteen games score a perfect 100, every one of them with no IGDB +votes behind it -- and the cold-start feed, which orders by rating alone, +recommended all sixteen alphabetically. + +Storing IGDB's vote count lets that feed shrink a rating toward the library +mean in proportion to how little evidence backs it. + +Only IGDB reports a count, and `manual_metadata` can override it like every +other generated facet. + +Revision ID: 0110_rating_count_column +Revises: 0109_igdb_tag_columns +Create Date: 2026-08-08 00:00:00.000000 + +""" + +from alembic import op # type: ignore[attr-defined] + +from utils.database import is_postgresql + +# revision identifiers, used by Alembic. +revision = "0110_rating_count_column" +down_revision = "0109_igdb_tag_columns" +branch_labels = None +depends_on = None + +_COLUMN = "generated_rating_count" +_SOURCES = ["manual_metadata", "igdb_metadata"] + +# Restated in full because CREATE OR REPLACE VIEW rewrites every projection. +_VIEW_COLUMNS = [ + ("generated_genres", "genres"), + ("generated_franchises", "franchises"), + ("generated_collections", "collections"), + ("generated_companies", "companies"), + ("generated_game_modes", "game_modes"), + ("generated_age_ratings", "age_ratings"), + ("generated_keywords", "keywords"), + ("generated_themes", "themes"), + ("generated_player_perspectives", "player_perspectives"), + ("generated_first_release_date", "first_release_date"), + ("generated_average_rating", "average_rating"), + ("generated_player_count", "player_count"), +] + + +def _maria_expr() -> str: + branches = [ + f"NULLIF(JSON_VALUE({src}, '$.total_rating_count'), '')" for src in _SOURCES + ] + return "CAST(COALESCE(" + ", ".join(branches) + ", 0) AS SIGNED)" + + +def _postgres_expr() -> str: + branches = [f"({src} ->> 'total_rating_count')" for src in _SOURCES] + return ( + "COALESCE(" + + ", ".join(f"NULLIF({b}, '')::numeric" for b in branches) + + ", 0)::bigint" + ) + + +def _view_sql(is_pg: bool, columns: list[tuple[str, str]]) -> str: + projections = ",\n ".join( + # The view exposed player_count as text and PostgreSQL cannot change a + # column's type through CREATE OR REPLACE VIEW. + ( + f"{name}::text AS {alias}" + if is_pg and alias == "player_count" + else f"{name} AS {alias}" + ) + for name, alias in columns + ) + return ( + "CREATE OR REPLACE VIEW roms_metadata AS\n" # nosec B608 + "SELECT\n" + " id AS rom_id,\n" + " NOW() AS created_at,\n" + " NOW() AS updated_at,\n" + f" {projections}\n" + "FROM roms" + ) + + +def upgrade() -> None: + is_pg = is_postgresql(op.get_bind()) + expr = _postgres_expr() if is_pg else _maria_expr() + column_type = "BIGINT" if is_pg else "BIGINT" + + op.execute( + f"ALTER TABLE roms ADD COLUMN {_COLUMN} {column_type} " # nosec B608 + f"GENERATED ALWAYS AS ({expr}) STORED" + ) + op.execute(_view_sql(is_pg, _VIEW_COLUMNS + [(_COLUMN, "rating_count")])) + + +def downgrade() -> None: + is_pg = is_postgresql(op.get_bind()) + + # The view has to stop referencing the column before it can be dropped. + op.execute("DROP VIEW IF EXISTS roms_metadata") + op.execute(_view_sql(is_pg, _VIEW_COLUMNS).replace("CREATE OR REPLACE", "CREATE")) + op.execute(f"ALTER TABLE roms DROP COLUMN {_COLUMN}") diff --git a/backend/alembic/versions/0111_company_role_columns.py b/backend/alembic/versions/0111_company_role_columns.py new file mode 100644 index 0000000000..b9f78d630b --- /dev/null +++ b/backend/alembic/versions/0111_company_role_columns.py @@ -0,0 +1,222 @@ +"""Split companies into developers and publishers for recommendation scoring + +`companies` flattens every IGDB involvement role into one list, so a studio +that made a game is indistinguishable from a label that shipped it or a +regional distributor that boxed it. The roles carry very different weight for +similarity: a developer's games genuinely resemble each other, while a +publisher spans everything it ever shipped. + +The distinction is not academic. On a 15k-game library the most common +"company" is Tec Toy, Sega's Brazilian distributor, on 774 games, ahead of +Nintendo on 756 -- dense enough that IDF alone does not suppress them, so +matches get explained as "same distributor". + +`companies` stays exactly as it is for display; these are additive, and the +scorer prefers them where present and falls back to the merged list where a +game was matched by a provider that reports no roles. + +Revision ID: 0111_company_role_columns +Revises: 0110_rating_count_column +Create Date: 2026-08-08 00:00:00.000000 + +""" + +import sqlalchemy as sa +from alembic import op # type: ignore[attr-defined] + +from utils.database import CustomJSON, is_postgresql + +# revision identifiers, used by Alembic. +revision = "0111_company_role_columns" +down_revision = "0110_rating_count_column" +branch_labels = None +depends_on = None + +# (generated column, roms_facets column). Only IGDB reports roles, plus +# manual_metadata so a user override still wins. +_ROLE_COLUMNS = [ + ("generated_developers", "developers"), + ("generated_publishers", "publishers"), +] +_SOURCES = ["manual_metadata", "igdb_metadata"] + +# Restated in full: CREATE OR REPLACE VIEW rewrites every projection. +_VIEW_COLUMNS = [ + ("generated_genres", "genres"), + ("generated_franchises", "franchises"), + ("generated_collections", "collections"), + ("generated_companies", "companies"), + ("generated_game_modes", "game_modes"), + ("generated_age_ratings", "age_ratings"), + ("generated_keywords", "keywords"), + ("generated_themes", "themes"), + ("generated_player_perspectives", "player_perspectives"), + ("generated_first_release_date", "first_release_date"), + ("generated_average_rating", "average_rating"), + ("generated_player_count", "player_count"), + ("generated_rating_count", "rating_count"), +] + +# Mirrored into roms_facets by the triggers. Matches migration 0109's list +# with the two role columns appended. +_MIRRORED_COLUMNS = [ + ("platform_id", "platform_id"), + ("genres", "generated_genres"), + ("franchises", "generated_franchises"), + ("collections", "generated_collections"), + ("companies", "generated_companies"), + ("game_modes", "generated_game_modes"), + ("age_ratings", "generated_age_ratings"), + ("keywords", "generated_keywords"), + ("themes", "generated_themes"), + ("player_perspectives", "generated_player_perspectives"), + ("player_count", "generated_player_count"), + ("regions", "regions"), + ("languages", "languages"), + ("tags", "tags"), + ("igdb_id", "igdb_id"), + ("ss_id", "ss_id"), + ("moby_id", "moby_id"), + ("launchbox_id", "launchbox_id"), + ("ra_id", "ra_id"), + ("hasheous_id", "hasheous_id"), + ("tgdb_id", "tgdb_id"), + ("flashpoint_id", "flashpoint_id"), + ("hltb_id", "hltb_id"), + ("gamelist_id", "gamelist_id"), + ("libretro_id", "libretro_id"), +] + +_MYSQL_TRIGGERS = { + "roms_facets_after_insert": "AFTER INSERT", + "roms_facets_after_update": "AFTER UPDATE", +} + + +def _maria_expr(column: str) -> str: + key = column[len("generated_") :] + branches = [ + f"CASE WHEN JSON_LENGTH(JSON_EXTRACT({src}, '$.{key}')) > 0 " + f"THEN JSON_EXTRACT({src}, '$.{key}') ELSE NULL END" + for src in _SOURCES + ] + branches.append("JSON_ARRAY()") + return "COALESCE(" + ", ".join(branches) + ")" + + +def _postgres_expr(column: str) -> str: + key = column[len("generated_") :] + branches = [f"NULLIF({src} -> '{key}', '[]'::jsonb)" for src in _SOURCES] + branches.append("'[]'::jsonb") + return "COALESCE(" + ", ".join(branches) + ")" + + +def _view_sql(is_pg: bool, columns: list[tuple[str, str]]) -> str: + projections = ",\n ".join( + # The view exposed player_count as text and PostgreSQL cannot change a + # column's type through CREATE OR REPLACE VIEW. + ( + f"{name}::text AS {alias}" + if is_pg and alias == "player_count" + else f"{name} AS {alias}" + ) + for name, alias in columns + ) + return ( + "CREATE OR REPLACE VIEW roms_metadata AS\n" # nosec B608 + "SELECT\n" + " id AS rom_id,\n" + " NOW() AS created_at,\n" + " NOW() AS updated_at,\n" + f" {projections}\n" + "FROM roms" + ) + + +def _rebuild_triggers(is_pg: bool, mirrored: list[tuple[str, str]]) -> None: + """Recreate the roms_facets sync triggers over the given column list.""" + targets = ", ".join(target for target, _ in mirrored) + values = ", ".join(f"NEW.{source}" for _, source in mirrored) + + if is_pg: + assignments = ", ".join( + f"{target} = EXCLUDED.{target}" for target, _ in mirrored + ) + op.execute(f""" +CREATE OR REPLACE FUNCTION romm_sync_rom_facets() RETURNS trigger + LANGUAGE plpgsql AS $$ +BEGIN + INSERT INTO roms_facets (rom_id, {targets}) + VALUES (NEW.id, {values}) + ON CONFLICT (rom_id) DO UPDATE SET + {assignments}, + updated_at = NOW(); + RETURN NULL; +END $$ +""") # nosec B608 + return + + updates = ",\n".join(f"{target} = VALUES({target})" for target, _ in mirrored) + body = ( + f"INSERT INTO roms_facets (rom_id, {targets})\n" # nosec B608 + f"VALUES (NEW.id, {values})\n" + f"ON DUPLICATE KEY UPDATE\n{updates},\nupdated_at = CURRENT_TIMESTAMP" + ) + for name, timing in _MYSQL_TRIGGERS.items(): + op.execute(f"DROP TRIGGER IF EXISTS {name}") + op.execute(f"CREATE TRIGGER {name} {timing} ON roms\nFOR EACH ROW\n{body}") + + +def upgrade() -> None: + is_pg = is_postgresql(op.get_bind()) + json_type = "JSONB" if is_pg else "JSON" + + for generated, _ in _ROLE_COLUMNS: + expr = _postgres_expr(generated) if is_pg else _maria_expr(generated) + op.execute( + f"ALTER TABLE roms ADD COLUMN {generated} {json_type} " # nosec B608 + f"GENERATED ALWAYS AS ({expr}) STORED" + ) + + op.execute(_view_sql(is_pg, _VIEW_COLUMNS + [(g, f) for g, f in _ROLE_COLUMNS])) + + for _, facet in _ROLE_COLUMNS: + op.add_column("roms_facets", sa.Column(facet, CustomJSON(), nullable=True)) + + _backfill_facets(is_pg) + _rebuild_triggers( + is_pg, _MIRRORED_COLUMNS + [(facet, gen) for gen, facet in _ROLE_COLUMNS] + ) + + +def _backfill_facets(is_pg: bool) -> None: + if is_pg: + assignments = ", ".join( + f"{facet} = r.{generated}" for generated, facet in _ROLE_COLUMNS + ) + op.execute( + f"UPDATE roms_facets f SET {assignments} FROM roms r " # nosec B608 + "WHERE r.id = f.rom_id" + ) + return + + assignments = ", ".join( + f"f.{facet} = r.{generated}" for generated, facet in _ROLE_COLUMNS + ) + op.execute( + f"UPDATE roms_facets f JOIN roms r ON r.id = f.rom_id SET {assignments}" # nosec B608 + ) + + +def downgrade() -> None: + is_pg = is_postgresql(op.get_bind()) + + op.execute("DROP VIEW IF EXISTS roms_metadata") + op.execute(_view_sql(is_pg, _VIEW_COLUMNS).replace("CREATE OR REPLACE", "CREATE")) + + for _, facet in _ROLE_COLUMNS: + op.drop_column("roms_facets", facet) + for generated, _ in _ROLE_COLUMNS: + op.execute(f"ALTER TABLE roms DROP COLUMN {generated}") + + _rebuild_triggers(is_pg, _MIRRORED_COLUMNS) diff --git a/backend/config/__init__.py b/backend/config/__init__.py index 500ae95714..6609627d19 100644 --- a/backend/config/__init__.py +++ b/backend/config/__init__.py @@ -259,6 +259,15 @@ def _get_env(var: str, fallback: str | None = None) -> str | None: "SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC_CRON", "0 4 * * *", # At 4:00 AM every day ) +# On by default: the similarity index is what both the "Similar games" section +# and the personalised feed read, so leaving it off silently empties them. +ENABLE_SCHEDULED_BUILD_RECOMMENDATIONS: Final[bool] = safe_str_to_bool( + _get_env("ENABLE_SCHEDULED_BUILD_RECOMMENDATIONS", "true") +) +SCHEDULED_BUILD_RECOMMENDATIONS_CRON: Final[str] = _get_env( + "SCHEDULED_BUILD_RECOMMENDATIONS_CRON", + "30 5 * * *", # At 5:30 AM every day, after the nightly scan and metadata tasks +) # SYNC SYNC_BASE_PATH: Final[str] = f"{ROMM_BASE_PATH}/sync" diff --git a/backend/endpoints/recommendations.py b/backend/endpoints/recommendations.py new file mode 100644 index 0000000000..e1d4064789 --- /dev/null +++ b/backend/endpoints/recommendations.py @@ -0,0 +1,69 @@ +from typing import Annotated + +from fastapi import Query, Request + +from decorators.auth import protected_route +from endpoints.responses.recommendation import RecommendedRomSchema +from endpoints.responses.rom import SimpleRomSchema +from handler.auth.constants import Scope +from handler.auth.dependencies import get_permissions +from handler.recommendation import FeedBuilder, get_cached_feed, set_cached_feed +from utils.router import APIRouter + +router = APIRouter( + prefix="/recommendations", + tags=["recommendations"], +) + +DEFAULT_FEED_LIMIT = 20 +MAX_FEED_LIMIT = 50 + +# How much deeper to rank for a user whose visibility rules will drop entries +# from the ranked list. Everyone else ranks exactly as many as they asked for. +VISIBILITY_OVERFETCH = 3 + + +@protected_route(router.get, "", [Scope.ROMS_READ]) +def get_recommendations( + request: Request, + limit: Annotated[ + int, + Query(ge=1, le=MAX_FEED_LIMIT, description="Maximum recommendations to return"), + ] = DEFAULT_FEED_LIMIT, + refresh: Annotated[ + bool, Query(description="Bypass the cached feed and rank again") + ] = False, +) -> list[RecommendedRomSchema]: + """Personalised game recommendations for the current user. + + Ranked on demand from the precomputed similarity graph plus the user's + live play history, so a game played minutes ago already steers the feed. + """ + user_id = request.user.id + perms = get_permissions(request) + + # Ranking exactly `limit` and filtering afterwards hands a user with hidden + # ROMs a short row, or an empty one when the hidden games rank highest. + hides_anything = not perms.is_admin and bool( + perms.hidden_rom_ids or perms.hidden_platform_ids + ) + ranked_limit = limit * VISIBILITY_OVERFETCH if hides_anything else limit + + feed = None if refresh else get_cached_feed(user_id, ranked_limit) + if feed is None: + feed = FeedBuilder(user_id).build(limit=ranked_limit) + set_cached_feed(user_id, ranked_limit, feed) + + visible = [ + RecommendedRomSchema( + rom=SimpleRomSchema.from_orm_with_request(item.rom, request), + score=item.score, + reasons=item.reasons, # type: ignore[arg-type] + seed_rom_id=item.seed_rom_id, + seed_rom_name=item.seed_rom_name, + ) + for item in feed + if perms.can_see_rom(item.rom.id, item.rom.platform_id) + ] + + return visible[:limit] diff --git a/backend/endpoints/responses/recommendation.py b/backend/endpoints/responses/recommendation.py new file mode 100644 index 0000000000..564dd7ff07 --- /dev/null +++ b/backend/endpoints/responses/recommendation.py @@ -0,0 +1,31 @@ +from __future__ import annotations + +from pydantic import BaseModel + +from endpoints.responses.rom import SimpleRomSchema + + +class SimilarityReasonSchema(BaseModel): + """Why two games were linked, e.g. {"facet": "franchise", "value": "Metroid"}. + + `facet` is one of the metadata facets the engine scores on (genre, + franchise, collection, company, game_mode, decade), or "igdb" when the + link came from IGDB's own related-games list, or "top_rated" for the + cold-start feed. The frontend maps it to a translated label. + """ + + facet: str + value: str + + +class SimilarRomSchema(BaseModel): + rom: SimpleRomSchema + score: float + reasons: list[SimilarityReasonSchema] + + +class RecommendedRomSchema(SimilarRomSchema): + # The played game that pulled this recommendation in, for "Because you + # played X". Absent on cold-start results, which have no seed. + seed_rom_id: int | None = None + seed_rom_name: str | None = None diff --git a/backend/endpoints/roms/__init__.py b/backend/endpoints/roms/__init__.py index e1375e5d5a..5605ea3849 100644 --- a/backend/endpoints/roms/__init__.py +++ b/backend/endpoints/roms/__init__.py @@ -39,6 +39,7 @@ ) from decorators.auth import protected_route from endpoints.responses import BulkOperationResponse +from endpoints.responses.recommendation import SimilarRomSchema from endpoints.responses.rom import ( DetailedRomSchema, RomFiltersDict, @@ -53,7 +54,12 @@ assert_rom_visible, get_permissions, ) -from handler.database import db_collection_handler, db_rom_handler, db_save_handler +from handler.database import ( + db_collection_handler, + db_recommendation_handler, + db_rom_handler, + db_save_handler, +) from handler.database.base_handler import sync_session from handler.filesystem import fs_resource_handler, fs_rom_handler from handler.filesystem.assets_handler import validate_image_upload @@ -68,6 +74,7 @@ ) from handler.metadata.launchbox_handler.media import populate_rom_specific_paths from handler.metadata.ss_handler import add_ss_auth_to_url, get_preferred_media_types +from handler.recommendation import cap_by_series from handler.rom_conversion import promote_single_file_to_folder from logger.formatter import BLUE from logger.formatter import highlight as hl @@ -117,6 +124,7 @@ STATUS_MEMBERSHIP_FIELDS = frozenset({"status", "now_playing", "backlogged", "hidden"}) +# RomUser fields that feed the recommendation ranking. def safe_int_or_none(value: Any) -> int | None: if value is None or value == "": return None @@ -1263,6 +1271,67 @@ def get_rom_simple( return SimpleRomSchema.from_orm_with_request(rom, request) +@protected_route( + router.get, + "/{id}/similar", + [] if DISABLE_DOWNLOAD_ENDPOINT_AUTH else [Scope.ROMS_READ], + responses={status.HTTP_404_NOT_FOUND: {}}, +) +def get_similar_roms( + request: Request, + id: Annotated[int, PathVar(description="Rom internal id.", ge=1)], + limit: Annotated[ + int, Query(ge=1, le=50, description="Maximum similar roms to return") + ] = 12, +) -> list[SimilarRomSchema]: + """Games in this library that resemble the given one. + + Reads the precomputed similarity graph, which blends library-relative + metadata overlap with IGDB's related games, collection co-membership and + co-play. Unlike the raw IGDB list, every result is a game the server + actually holds. + """ + + rom = db_rom_handler.get_rom_simple(id) + + if not rom: + raise RomNotFoundInDatabaseException(id) + + assert_rom_visible(request, rom) + + perms = get_permissions(request) + # Over-fetch: permission filtering and the per-series cap below both drop + # entries, and a shelf deep in one franchise drops a lot of them. + edges = db_recommendation_handler.get_similar_rom_edges(id, limit=limit * 4) + + similar_roms = { + similar.id: similar + for similar in db_rom_handler.get_roms_simple_by_ids( + [edge.rom_id for edge in edges] + ) + if not similar.missing_from_fs + and perms.can_see_rom(similar.id, similar.platform_id) + } + + # Without the cap this section is just the franchise the user is already + # looking at -- five Metroid games for Super Metroid, which a franchise + # filter already gives them. + selected = cap_by_series( + edges, lambda edge: similar_roms.get(edge.rom_id), limit=limit + ) + + return [ + SimilarRomSchema( + rom=SimpleRomSchema.from_orm_with_request( + similar_roms[edge.rom_id], request + ), + score=edge.score, + reasons=edge.reasons, # type: ignore[arg-type] + ) + for edge in selected + ] + + @protected_route( router.get, "/{id}", diff --git a/backend/endpoints/tasks.py b/backend/endpoints/tasks.py index d007dc3a9e..ceed9acf89 100644 --- a/backend/endpoints/tasks.py +++ b/backend/endpoints/tasks.py @@ -38,6 +38,7 @@ recompute_save_content_hashes_task, ) from tasks.manual.sync_folder_scan import sync_folder_scan_task +from tasks.scheduled.build_recommendations import build_recommendations_task from tasks.scheduled.cleanup_orphaned_resources import cleanup_orphaned_resources_task from tasks.scheduled.cleanup_zip_cache import cleanup_zip_cache_task from tasks.scheduled.convert_images_to_webp import convert_images_to_webp_task @@ -88,6 +89,13 @@ class ManualTask(ScheduledTask): "task": update_switch_titledb_task, } ), + ScheduledTask( + { + "name": "build_recommendations", + "type": TaskType.UPDATE, + "task": build_recommendations_task, + } + ), ScheduledTask( { "name": "convert_images_to_webp", diff --git a/backend/handler/database/__init__.py b/backend/handler/database/__init__.py index 1d2f4cfe49..5a5a0937fc 100644 --- a/backend/handler/database/__init__.py +++ b/backend/handler/database/__init__.py @@ -7,6 +7,7 @@ from .permissions_handler import DBPermissionsHandler from .platforms_handler import DBPlatformsHandler from .play_sessions_handler import DBPlaySessionsHandler +from .recommendations_handler import DBRecommendationsHandler from .roms_handler import DBRomsHandler from .saves_handler import DBSavesHandler from .screenshots_handler import DBScreenshotsHandler @@ -24,6 +25,7 @@ db_permission_handler = DBPermissionsHandler() db_platform_handler = DBPlatformsHandler() db_play_session_handler = DBPlaySessionsHandler() +db_recommendation_handler = DBRecommendationsHandler() db_rom_handler = DBRomsHandler() db_save_handler = DBSavesHandler() db_screenshot_handler = DBScreenshotsHandler() diff --git a/backend/handler/database/recommendations_handler.py b/backend/handler/database/recommendations_handler.py new file mode 100644 index 0000000000..6de1c01718 --- /dev/null +++ b/backend/handler/database/recommendations_handler.py @@ -0,0 +1,436 @@ +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from typing import Any, NamedTuple + +from sqlalchemy import delete, func, insert, select +from sqlalchemy.orm import Session + +from decorators.database import begin_session +from models.collection import CollectionRom +from models.play_session import PlaySession +from models.recommendation import RomSimilarity +from models.rom import Rom, RomFacets, RomMetadata, RomUser + +from .base_handler import DBBaseHandler + +# Streaming chunk for the wide `roms` scan that reads the IGDB metadata blobs. +IGDB_SCAN_CHUNK_SIZE = 500 + +# Rows per INSERT when rewriting the similarity table. +EDGE_INSERT_CHUNK_SIZE = 1_000 + +# Co-occurrence sets larger than this say more about the collector than the +# games: a 900-ROM "Everything" collection would otherwise emit 400k pairs and +# relate its entire contents to itself. +MAX_CO_OCCURRENCE_SET_SIZE = 250 + +# Votes a rating needs before it is trusted on its own in the cold-start feed. +# Below this it is blended with the library mean; well above it, the raw rating +# carries. Tuned so a handful of votes cannot float an obscure game to the top. +BAYESIAN_PRIOR_VOTES = 50 + + +class RomFeatureRow(NamedTuple): + """The narrow slice of metadata the similarity build reads per ROM.""" + + rom_id: int + platform_id: int + title_key: str | None + genres: list[str] | None + franchises: list[str] | None + collections: list[str] | None + companies: list[str] | None + developers: list[str] | None + publishers: list[str] | None + game_modes: list[str] | None + keywords: list[str] | None + themes: list[str] | None + player_perspectives: list[str] | None + first_release_date: int | None + average_rating: float | None + + +class UserAffinityRow(NamedTuple): + """One ROM the user has engaged with, and how strongly.""" + + rom_id: int + rating: int | None + difficulty: int | None + completion: int | None + status: str | None + last_played: Any | None + now_playing: bool + backlogged: bool + hidden: bool + playtime_ms: int + + +class SimilarRomEdge(NamedTuple): + rom_id: int + score: float + reasons: list[dict[str, Any]] + + +class DBRecommendationsHandler(DBBaseHandler): + # --- Similarity build inputs ------------------------------------------------- + + @begin_session + def get_feature_rows( + self, session: Session = None # type: ignore + ) -> list[RomFeatureRow]: + """Every ROM's facet values, for building the library-wide IDF. + + Reads `roms_facets` rather than `roms` on purpose: the facet values are + mirrored there precisely so aggregations never touch the wide rows with + their provider-metadata blobs. + """ + stmt = ( + select( + RomFacets.rom_id, + RomFacets.platform_id, + Rom.name_sort_key, + RomFacets.genres, + RomFacets.franchises, + RomFacets.collections, + RomFacets.companies, + RomFacets.developers, + RomFacets.publishers, + RomFacets.game_modes, + RomFacets.keywords, + RomFacets.themes, + RomFacets.player_perspectives, + RomMetadata.first_release_date, + RomMetadata.average_rating, + ) + .join(Rom, Rom.id == RomFacets.rom_id) + .outerjoin(RomMetadata, RomMetadata.rom_id == RomFacets.rom_id) + .where(Rom.missing_from_fs.is_(False)) + ) + + return [RomFeatureRow(*row) for row in session.execute(stmt).all()] + + @begin_session + def get_rom_igdb_ids( + self, session: Session = None # type: ignore + ) -> dict[int, int]: + """ROM id -> IGDB id. + + Keyed by ROM id, not IGDB id: the relationship is many-to-one (region + and revision variants of one game share an IGDB id), and keying the + other way would silently drop every duplicate but one -- which is + exactly the set the duplicate suppression needs to see. + """ + stmt = select(RomFacets.rom_id, RomFacets.igdb_id).where( + RomFacets.igdb_id.is_not(None) + ) + return {rom_id: igdb_id for rom_id, igdb_id in session.execute(stmt).all()} + + @begin_session + def iter_igdb_related( + self, session: Session = None # type: ignore + ) -> Iterator[tuple[int, list[int]]]: + """Stream each ROM's IGDB related-game ids out of its metadata blob. + + Yields in chunks because this is the one query that has to read the + wide `roms` row; everything else in the build works off narrow tables. + """ + stmt = ( + select(Rom.id, Rom.igdb_metadata) + .where(Rom.igdb_id.is_not(None), Rom.missing_from_fs.is_(False)) + .execution_options(yield_per=IGDB_SCAN_CHUNK_SIZE) + ) + + for rom_id, metadata in session.execute(stmt): + if not metadata: + continue + + related_ids: list[int] = [] + # `ports` is deliberately absent. A port is the same product on + # other hardware, so the relation says nothing about whether one + # is worth suggesting to someone who played the other. Where the + # port is faithful it is a duplicate, and where it was rebuilt for + # the target hardware it can earn a place on its own facets. + for bucket in ( + "similar_games", + "remakes", + "remasters", + "expanded_games", + "expansions", + "dlcs", + ): + for entry in metadata.get(bucket) or (): + entry_id = entry.get("id") if isinstance(entry, dict) else None + if isinstance(entry_id, int): + related_ids.append(entry_id) + + if related_ids: + yield rom_id, related_ids + + @begin_session + def get_collection_membership_sets( + self, session: Session = None # type: ignore + ) -> list[list[int]]: + """ROM ids grouped by user collection, for co-membership scoring.""" + stmt = select(CollectionRom.collection_id, CollectionRom.rom_id).order_by( + CollectionRom.collection_id + ) + return self._group_second_by_first(session.execute(stmt).all()) + + @begin_session + def get_played_sets( + self, session: Session = None # type: ignore + ) -> list[list[int]]: + """ROM ids grouped by user, restricted to games they actually played. + + This is the item-based collaborative signal. On a single-user server it + degrades gracefully into "things I play together" rather than vanishing. + """ + stmt = ( + select(RomUser.user_id, RomUser.rom_id) + .where(RomUser.last_played.is_not(None)) + .order_by(RomUser.user_id) + ) + return self._group_second_by_first(session.execute(stmt).all()) + + @staticmethod + def _group_second_by_first(rows: Sequence[Any]) -> list[list[int]]: + """Bucket (key, rom_id) rows into per-key id lists, dropping the noisy ones. + + Singletons carry no pair information and oversized sets carry mostly + noise, so neither is worth handing to the pair counter. + """ + grouped: dict[int, list[int]] = {} + for key, rom_id in rows: + grouped.setdefault(key, []).append(rom_id) + + return [ + ids + for ids in grouped.values() + if 1 < len(ids) <= MAX_CO_OCCURRENCE_SET_SIZE + ] + + # --- Similarity build output ------------------------------------------------- + + @begin_session + def replace_similarity_edges( + self, + rom_ids: Sequence[int], + edges: Sequence[dict[str, Any]], + session: Session = None, # type: ignore + ) -> int: + """Swap in a batch of ROMs' edges. + + Scoped to `rom_ids` rather than truncating the table so the build can + commit incrementally: a task that dies halfway leaves stale edges for + the ROMs it never reached, not an empty recommendations table. + """ + if not rom_ids: + return 0 + + session.execute(delete(RomSimilarity).where(RomSimilarity.rom_id.in_(rom_ids))) + + written = 0 + for start in range(0, len(edges), EDGE_INSERT_CHUNK_SIZE): + chunk = edges[start : start + EDGE_INSERT_CHUNK_SIZE] + if chunk: + session.execute(insert(RomSimilarity), chunk) + written += len(chunk) + + return written + + @begin_session + def delete_all_similarity_edges( + self, session: Session = None # type: ignore + ) -> None: + session.execute(delete(RomSimilarity)) + + @begin_session + def count_similarity_edges(self, session: Session = None) -> int: # type: ignore + return session.scalar(select(func.count()).select_from(RomSimilarity)) or 0 + + # --- Reads ------------------------------------------------------------------- + + @begin_session + def get_similar_rom_edges( + self, + rom_id: int, + limit: int = 20, + session: Session = None, # type: ignore + ) -> list[SimilarRomEdge]: + """Top precomputed neighbours of one ROM, best first. + + Returns ids rather than ROMs so every caller hydrates through the same + `SimpleRomSchema` load path instead of each growing its own. + """ + stmt = ( + select( + RomSimilarity.related_rom_id, + RomSimilarity.score, + RomSimilarity.reasons, + ) + .where(RomSimilarity.rom_id == rom_id) + .order_by(RomSimilarity.score.desc(), RomSimilarity.related_rom_id.asc()) + .limit(limit) + ) + + return [ + SimilarRomEdge(rom_id=related_id, score=score, reasons=reasons or []) + for related_id, score, reasons in session.execute(stmt).all() + ] + + @begin_session + def get_neighbours_for_roms( + self, + rom_ids: Sequence[int], + limit_per_rom: int = 20, + session: Session = None, # type: ignore + ) -> list[tuple[int, int, float, list[dict[str, Any]]]]: + """Edges fanning out from a set of seed ROMs, for the personalised feed. + + Returns raw (seed_rom_id, related_rom_id, score, reasons) tuples; the + ranking handler hydrates only the ROMs that survive its cut. + """ + if not rom_ids: + return [] + + stmt = ( + select( + RomSimilarity.rom_id, + RomSimilarity.related_rom_id, + RomSimilarity.score, + RomSimilarity.reasons, + ) + .where(RomSimilarity.rom_id.in_(rom_ids)) + .order_by(RomSimilarity.rom_id, RomSimilarity.score.desc()) + ) + + # Trim per seed in Python: a per-group LIMIT needs a window function, + # and the row counts here are already bounded by the build's top-N cut. + per_seed: dict[int, int] = {} + results: list[tuple[int, int, float, list[dict[str, Any]]]] = [] + for seed_id, related_id, score, reasons in session.execute(stmt): + taken = per_seed.get(seed_id, 0) + if taken >= limit_per_rom: + continue + per_seed[seed_id] = taken + 1 + results.append((seed_id, related_id, score, reasons or [])) + + return results + + @begin_session + def get_user_affinity( + self, + user_id: int, + session: Session = None, # type: ignore + ) -> list[UserAffinityRow]: + """Everything the user has done with each ROM, plus total playtime. + + Playtime comes from `play_sessions` (the accurate signal) while the + rest comes from `rom_user`; a ROM can appear with either or both. + """ + playtime_subq = ( + select( + PlaySession.rom_id.label("rom_id"), + func.coalesce(func.sum(PlaySession.duration_ms), 0).label( + "playtime_ms" + ), + ) + .where(PlaySession.user_id == user_id, PlaySession.rom_id.is_not(None)) + .group_by(PlaySession.rom_id) + .subquery() + ) + + stmt = ( + select( + RomUser.rom_id, + RomUser.rating, + RomUser.difficulty, + RomUser.completion, + RomUser.status, + RomUser.last_played, + RomUser.now_playing, + RomUser.backlogged, + RomUser.hidden, + func.coalesce(playtime_subq.c.playtime_ms, 0), + ) + .outerjoin(playtime_subq, playtime_subq.c.rom_id == RomUser.rom_id) + .where(RomUser.user_id == user_id) + ) + + return [ + UserAffinityRow( + rom_id=row[0], + rating=row[1], + difficulty=row[2], + completion=row[3], + status=row[4].value if hasattr(row[4], "value") else row[4], + last_played=row[5], + now_playing=bool(row[6]), + backlogged=bool(row[7]), + hidden=bool(row[8]), + playtime_ms=int(row[9] or 0), + ) + for row in session.execute(stmt).all() + ] + + @begin_session + def get_rom_names( + self, + rom_ids: Sequence[int], + session: Session = None, # type: ignore + ) -> dict[int, str]: + """Display names only, for the "Because you played X" attribution.""" + if not rom_ids: + return {} + + stmt = select(Rom.id, Rom.name, Rom.fs_name).where(Rom.id.in_(rom_ids)) + return { + rom_id: name or fs_name + for rom_id, name, fs_name in session.execute(stmt).all() + } + + @begin_session + def get_fallback_rom_ids( + self, + limit: int, + exclude_rom_ids: Sequence[int] = (), + session: Session = None, # type: ignore + ) -> list[int]: + """Cold-start feed: the best-reviewed games in the library. + + Ranked by a Bayesian average rather than the raw rating. A rating + backed by few votes is pulled toward the library mean in proportion to + how little evidence supports it, so a lone provider's perfect score no + longer outranks a broadly-liked classic. Without this the feed was + fourteen games that one source rated 100, listed alphabetically. + """ + mean_rating = session.scalar( + select(func.avg(RomMetadata.average_rating)).where( + RomMetadata.average_rating.is_not(None) + ) + ) + prior = float(mean_rating or 0.0) + + votes = func.coalesce(RomMetadata.rating_count, 0) + # (v * R + m * C) / (v + m): the standard shrinkage estimator, with m + # acting as "how many votes it takes to be believed on your own". + bayesian = ( + votes * RomMetadata.average_rating + BAYESIAN_PRIOR_VOTES * prior + ) / (votes + BAYESIAN_PRIOR_VOTES) + + stmt = ( + select(Rom.id) + .join(RomMetadata, RomMetadata.rom_id == Rom.id) + .where( + Rom.missing_from_fs.is_(False), + RomMetadata.average_rating.is_not(None), + ) + .order_by(bayesian.desc(), Rom.name_sort_key.asc()) + .limit(limit) + ) + + if exclude_rom_ids: + stmt = stmt.where(Rom.id.not_in(exclude_rom_ids)) + + return list(session.execute(stmt).scalars().all()) diff --git a/backend/handler/database/roms_handler.py b/backend/handler/database/roms_handler.py index a6528571fb..f48f59e2c8 100644 --- a/backend/handler/database/roms_handler.py +++ b/backend/handler/database/roms_handler.py @@ -420,6 +420,25 @@ def wrapper(*args, **kwargs): return wrapper +# The fields the recommendation feed scores on. Every writer of `rom_user` +# goes through `update_rom_user`, so the cached feed is dropped there rather +# than at each call site: play sessions, save and state uploads and the +# RetroAchievements sync all move these without touching the ROM endpoints. +RECOMMENDATION_SEED_FIELDS = frozenset( + {"rating", "status", "last_played", "now_playing", "hidden"} +) + + +def _invalidate_feed_if_seed_changed(user_id: int, data: dict) -> None: + if not RECOMMENDATION_SEED_FIELDS & data.keys(): + return + + # Imported here because the recommendation package reads this module. + from handler.recommendation import invalidate_cached_feed + + invalidate_cached_feed(user_id) + + class DBRomsHandler(DBBaseHandler): @begin_session @with_details @@ -471,6 +490,20 @@ def get_roms_by_ids( return [] return session.scalars(query.filter(Rom.id.in_(ids))).all() + @begin_session + @with_simple_details + def get_roms_simple_by_ids( + self, + ids: Sequence[int], + *, + query: Query = None, # type: ignore + session: Session = None, # type: ignore + ) -> Sequence[Rom]: + """Get multiple ROMs by ID with only the loads `SimpleRomSchema` needs.""" + if not ids: + return [] + return session.scalars(query.filter(Rom.id.in_(ids))).all() + def get_files_for_roms( self, rom_ids: list[int], @@ -1952,6 +1985,8 @@ def update_rom_user( if not rom_user: return None + _invalidate_feed_if_seed_changed(rom_user.user_id, data) + if not data.get("is_main_sibling", False): return rom_user diff --git a/backend/handler/metadata/igdb_handler.py b/backend/handler/metadata/igdb_handler.py index 9a11a54618..2b3c7b9100 100644 --- a/backend/handler/metadata/igdb_handler.py +++ b/backend/handler/metadata/igdb_handler.py @@ -1,5 +1,6 @@ import re -from typing import Final, NotRequired, TypedDict +from collections.abc import Sequence +from typing import Any, Final, NotRequired, TypedDict import httpx import pydash @@ -118,14 +119,26 @@ class IGDBMetadataMultiplayerMode(TypedDict): class IGDBMetadata(TypedDict): total_rating: str | None + # How many votes back total_rating. A 10/10 from one source is not the + # same claim as 9/10 from a thousand, and the cold-start feed needs to + # tell them apart. + total_rating_count: int | None aggregated_rating: str | None first_release_date: int | None youtube_video_id: str | None genres: list[str] + keywords: list[str] + themes: list[str] + player_perspectives: list[str] franchises: list[str] alternative_names: list[str] collections: list[str] companies: list[str] + # Split by role. A developer is a strong similarity signal (Treasure's + # games resemble each other); a publisher spans everything it ships, and + # regional distributors sit here too. + developers: list[str] + publishers: list[str] game_modes: list[str] age_ratings: list[IGDBAgeRating] platforms: list[IGDBMetadataPlatform] @@ -162,6 +175,35 @@ def build_related_game( ) +def _expanded_names(entries: Sequence[Any]) -> list[str]: + """Names from an IGDB expandable field. + + A field that was requested without `.name` comes back as a bare id rather + than an object, so entries that are not dicts are skipped instead of + raising. + """ + return [ + name + for entry in entries + if isinstance(entry, dict) and (name := entry.get("name")) + ] + + +def _companies_with_role(entries: Sequence[Any], role: str) -> list[str]: + """Names of the companies flagged with a given IGDB involvement role. + + A company can hold more than one role on the same game, so the lists + overlap where a studio both made and shipped a title. + """ + return pydash.uniq( + [ + entry["company"]["name"] + for entry in entries + if isinstance(entry, dict) and entry.get(role) and entry.get("company") + ] + ) + + def extract_metadata_from_igdb_rom( self: MetadataHandler, rom: Game, platform_igdb_id: int | None ) -> IGDBMetadata: @@ -175,6 +217,9 @@ def extract_metadata_from_igdb_rom( franchises = rom.get("franchises", []) game_modes = rom.get("game_modes", []) genres = rom.get("genres", []) + keywords = rom.get("keywords", []) + themes = rom.get("themes", []) + player_perspectives = rom.get("player_perspectives", []) involved_companies = rom.get("involved_companies", []) platforms = rom.get("platforms", []) multiplayer_modes = rom.get("multiplayer_modes", []) @@ -238,12 +283,22 @@ def extract_metadata_from_igdb_rom( { "youtube_video_id": videos[0].get("video_id") if videos else None, "total_rating": str(round(rom.get("total_rating", 0.0), 2)), + "total_rating_count": rom.get("total_rating_count", 0), "aggregated_rating": str(round(rom.get("aggregated_rating", 0.0), 2)), "first_release_date": rom.get("first_release_date", None), "genres": [g.get("name", "") for g in genres if g.get("name")], - "franchises": pydash.compact( - [franchise.get("name") if franchise else None] - + [f.get("name", "") for f in franchises if f.get("name")] + # Community tags ("metroidvania", "roguelike") describing how a game + # plays, which the coarse genre list does not capture. + "keywords": _expanded_names(keywords), + "themes": _expanded_names(themes), + "player_perspectives": _expanded_names(player_perspectives), + # IGDB reports the main franchise both on its own and inside the + # list, so the two sources overlap for most games that have one. + "franchises": pydash.uniq( + pydash.compact( + [franchise.get("name") if franchise else None] + + [f.get("name", "") for f in franchises if f.get("name")] + ) ), "alternative_names": [ n.get("name", "") for n in alternative_names if n.get("name") @@ -253,6 +308,8 @@ def extract_metadata_from_igdb_rom( "companies": [ c["company"]["name"] for c in involved_companies if c.get("company") ], + "developers": _companies_with_role(involved_companies, "developer"), + "publishers": _companies_with_role(involved_companies, "publisher"), "platforms": [ IGDBMetadataPlatform(igdb_id=p["id"], name=p.get("name", "")) for p in platforms @@ -1035,6 +1092,8 @@ async def get_oauth_token(self) -> str: "collections.name", "game_modes.name", "involved_companies.company.name", + "involved_companies.developer", + "involved_companies.publisher", "expansions.id", "expansions.slug", "expansions.name", @@ -1059,6 +1118,10 @@ async def get_oauth_token(self) -> str: "ports.slug", "ports.name", "ports.cover.url", + "total_rating_count", + "keywords.name", + "themes.name", + "player_perspectives.name", "similar_games.id", "similar_games.slug", "similar_games.name", diff --git a/backend/handler/recommendation/__init__.py b/backend/handler/recommendation/__init__.py new file mode 100644 index 0000000000..31f4c46b80 --- /dev/null +++ b/backend/handler/recommendation/__init__.py @@ -0,0 +1,23 @@ +from .builder import BuildStats, SimilarityBuilder +from .diversity import MAX_PER_SERIES, cap_by_series +from .feed import ( + FeedBuilder, + RecommendedRom, + get_cached_feed, + invalidate_all_cached_feeds, + invalidate_cached_feed, + set_cached_feed, +) + +__all__ = [ + "MAX_PER_SERIES", + "BuildStats", + "FeedBuilder", + "RecommendedRom", + "SimilarityBuilder", + "cap_by_series", + "get_cached_feed", + "invalidate_all_cached_feeds", + "invalidate_cached_feed", + "set_cached_feed", +] diff --git a/backend/handler/recommendation/builder.py b/backend/handler/recommendation/builder.py new file mode 100644 index 0000000000..d099684e4e --- /dev/null +++ b/backend/handler/recommendation/builder.py @@ -0,0 +1,415 @@ +"""Builds the precomputed item-item similarity graph. + +Run from the scheduled recommendations task. The whole graph is derived from +one consistent snapshot of the library, because the IDF weighting that makes +scores library-relative changes as the shelf grows. +""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Callable, Iterable, Sequence +from dataclasses import asdict, dataclass, field +from itertools import combinations +from typing import Any, Final + +from handler.database import db_recommendation_handler +from handler.database.recommendations_handler import RomFeatureRow +from logger.logger import log + +from .scoring import ( + RomFeatures, + blend, + build_inverted_index, + build_normalised_vectors, + candidate_ids, + compute_idf, + content_similarity, + extract_tokens, + has_taste_signal, + normalise_co_occurrence, + shared_reasons, +) + +# Neighbours kept per ROM. Enough to fill a "Similar games" shelf several times +# over and to give the personalised feed room to diversify, without letting the +# table grow to rom_count * library_size. +MAX_NEIGHBOURS: Final = 24 + +# Below this a "recommendation" is just two games that share the word Action. +MIN_EDGE_SCORE: Final = 0.05 + +# ROMs per write batch. Bounds peak memory and lets the task commit as it goes. +BUILD_BATCH_SIZE: Final = 500 + +# Neighbours from any one franchise allowed into the stored graph. Read-time +# diversity can only reorder what was stored, so a game sitting deep in a big +# series would otherwise have all 24 slots taken by that series and nothing +# left to promote. +MAX_STORED_PER_SERIES: Final = 6 + +# Hard ceiling on candidates scored per ROM. Reached only by games whose every +# facet is rare, where the tail is noise anyway. +MAX_CANDIDATES_PER_ROM: Final = 1_500 + + +@dataclass +class BuildStats: + roms_indexed: int = 0 + edges_written: int = 0 + roms_without_metadata: int = 0 + total: int = 0 + + def to_dict(self) -> dict[str, int]: + return asdict(self) + + +@dataclass +class _PairSignals: + """Sparse, symmetric side-signals keyed by an ordered ROM id pair.""" + + igdb: dict[tuple[int, int], float] = field(default_factory=dict) + co_play: dict[tuple[int, int], float] = field(default_factory=dict) + co_collection: dict[tuple[int, int], float] = field(default_factory=dict) + play_totals: dict[int, int] = field(default_factory=dict) + collection_totals: dict[int, int] = field(default_factory=dict) + + # ROM id -> every ROM linked to it by a non-content signal. Built once + # after collection: deriving it per ROM would rescan all three pair maps + # for every ROM in the library. + adjacency: defaultdict[int, set[int]] = field( + default_factory=lambda: defaultdict(set) + ) + + def index_adjacency(self) -> None: + for source in (self.igdb, self.co_play, self.co_collection): + for left, right in source: + self.adjacency[left].add(right) + self.adjacency[right].add(left) + + def partners_of(self, rom_id: int) -> set[int]: + return self.adjacency.get(rom_id, set()) + + +def _series_tokens(feature: RomFeatures) -> set[str]: + """Every franchise token a game carries. + + All of them rather than the first, for the same reason the serving-side + cap uses every value: a game listing both "Madden" and "NFL" would + otherwise be reserved against whichever happened to come first. + """ + return {token for token in feature.tokens if token.startswith("franchise:")} + + +def _pair_key(left: int, right: int) -> tuple[int, int]: + return (left, right) if left < right else (right, left) + + +class SimilarityBuilder: + """Assembles the similarity graph and writes it to `rom_similarity`.""" + + def __init__(self, progress: Callable[[BuildStats], None] | None = None) -> None: + self._progress = progress + self.stats = BuildStats() + + def build(self) -> BuildStats: + feature_rows = db_recommendation_handler.get_feature_rows() + if not feature_rows: + log.info("No ROMs to index for recommendations") + return self.stats + + features = self._build_features(feature_rows) + self.stats.total = len(features) + self._report() + + total_documents = len(features) + idf = compute_idf( + (feature.tokens for feature in features.values()), total_documents + ) + vectors = build_normalised_vectors( + {rom_id: feature.tokens for rom_id, feature in features.items()}, idf + ) + postings = build_inverted_index(features) + + igdb_ids = { + rom_id: igdb_id + for rom_id, igdb_id in db_recommendation_handler.get_rom_igdb_ids().items() + if rom_id in features + } + # Resolving an IGDB id back to a ROM is one-to-many; any owned copy of + # the game is an equally good target for the edge. + igdb_to_rom = {igdb_id: rom_id for rom_id, igdb_id in igdb_ids.items()} + signals = self._collect_pair_signals(features, igdb_ids, igdb_to_rom) + + log.info(f"Scoring similarity for {total_documents} ROMs") + self._score_and_write(features, vectors, postings, signals, igdb_ids) + + log.info( + f"Recommendations index built: {self.stats.roms_indexed} ROMs, " + f"{self.stats.edges_written} edges" + ) + return self.stats + + # --- Inputs ------------------------------------------------------------------ + + def _build_features(self, rows: Sequence[RomFeatureRow]) -> dict[int, RomFeatures]: + features: dict[int, RomFeatures] = {} + + for row in rows: + tokens = extract_tokens( + platform_id=row.platform_id, + genres=row.genres, + franchises=row.franchises, + collections=row.collections, + companies=row.companies, + developers=row.developers, + publishers=row.publishers, + game_modes=row.game_modes, + keywords=row.keywords, + themes=row.themes, + player_perspectives=row.player_perspectives, + first_release_date=row.first_release_date, + ) + + # Platform and decade alone describe a shelf, not a game: two + # unmatched files from the same folder would otherwise normalise to + # identical vectors and score a perfect match against each other. + if not has_taste_signal(tokens): + self.stats.roms_without_metadata += 1 + continue + + features[row.rom_id] = RomFeatures( + rom_id=row.rom_id, + platform_id=row.platform_id, + tokens=tokens, + average_rating=row.average_rating, + title_key=row.title_key, + ) + + return features + + def _collect_pair_signals( + self, + features: dict[int, RomFeatures], + igdb_ids: dict[int, int], + igdb_to_rom: dict[int, int], + ) -> _PairSignals: + signals = _PairSignals() + + for rom_id, related_igdb_ids in db_recommendation_handler.iter_igdb_related(): + if rom_id not in features: + continue + for related_igdb_id in related_igdb_ids: + related_rom_id = igdb_to_rom.get(related_igdb_id) + # IGDB's list is mostly games the user does not own; only the + # ones actually on the shelf are worth an edge. + if related_rom_id is None or related_rom_id == rom_id: + continue + if self._is_duplicate(rom_id, related_rom_id, igdb_ids): + continue + signals.igdb[_pair_key(rom_id, related_rom_id)] = 1.0 + + self._count_co_occurrence( + db_recommendation_handler.get_played_sets(), + features, + igdb_ids, + signals.co_play, + signals.play_totals, + ) + self._count_co_occurrence( + db_recommendation_handler.get_collection_membership_sets(), + features, + igdb_ids, + signals.co_collection, + signals.collection_totals, + ) + + signals.index_adjacency() + + log.debug( + f"Pair signals: igdb={len(signals.igdb)}, " + f"co_play={len(signals.co_play)}, " + f"co_collection={len(signals.co_collection)}" + ) + return signals + + def _count_co_occurrence( + self, + id_sets: Iterable[Sequence[int]], + features: dict[int, RomFeatures], + igdb_ids: dict[int, int], + pair_counts: dict[tuple[int, int], float], + totals: dict[int, int], + ) -> None: + raw: defaultdict[tuple[int, int], int] = defaultdict(int) + + for id_set in id_sets: + known = sorted({rom_id for rom_id in id_set if rom_id in features}) + if len(known) < 2: + continue + + for rom_id in known: + totals[rom_id] = totals.get(rom_id, 0) + 1 + + for left, right in combinations(known, 2): + if self._is_duplicate(left, right, igdb_ids): + continue + raw[(left, right)] += 1 + + for (left, right), count in raw.items(): + pair_counts[(left, right)] = normalise_co_occurrence( + count, totals.get(left, 0), totals.get(right, 0) + ) + + @staticmethod + def _is_duplicate(left: int, right: int, igdb_ids: dict[int, int]) -> bool: + """Two files of the same game (regions, revisions) are not a recommendation.""" + left_igdb = igdb_ids.get(left) + return left_igdb is not None and left_igdb == igdb_ids.get(right) + + # --- Scoring ----------------------------------------------------------------- + + def _score_and_write( + self, + features: dict[int, RomFeatures], + vectors: dict[int, dict[str, float]], + postings: dict[str, list[int]], + signals: _PairSignals, + igdb_ids: dict[int, int], + ) -> None: + total_documents = len(features) + batch_rom_ids: list[int] = [] + batch_edges: list[dict[str, Any]] = [] + + for rom_id, feature in features.items(): + edges = self._score_one( + feature, features, vectors, postings, signals, igdb_ids, total_documents + ) + + batch_rom_ids.append(rom_id) + batch_edges.extend(edges) + self.stats.roms_indexed += 1 + + if len(batch_rom_ids) >= BUILD_BATCH_SIZE: + self._flush(batch_rom_ids, batch_edges) + batch_rom_ids, batch_edges = [], [] + + self._flush(batch_rom_ids, batch_edges) + + def _score_one( + self, + feature: RomFeatures, + features: dict[int, RomFeatures], + vectors: dict[int, dict[str, float]], + postings: dict[str, list[int]], + signals: _PairSignals, + igdb_ids: dict[int, int], + total_documents: int, + ) -> list[dict[str, Any]]: + rom_id = feature.rom_id + source_vector = vectors.get(rom_id, {}) + + candidates = candidate_ids(feature, postings, total_documents) + # A game IGDB relates to, or that users play alongside this one, is + # worth scoring even when they share no metadata facet at all. + candidates |= signals.partners_of(rom_id) + candidates.discard(rom_id) + + if len(candidates) > MAX_CANDIDATES_PER_ROM: + candidates = set( + sorted( + candidates, + key=lambda cid: content_similarity( + source_vector, vectors.get(cid, {}) + ), + reverse=True, + )[:MAX_CANDIDATES_PER_ROM] + ) + + scored: list[tuple[float, int, list[dict[str, str]]]] = [] + for candidate_id in candidates: + if self._is_duplicate(rom_id, candidate_id, igdb_ids): + continue + + candidate_vector = vectors.get(candidate_id, {}) + content = content_similarity(source_vector, candidate_vector) + key = _pair_key(rom_id, candidate_id) + + score = blend( + content=content, + igdb_prior=signals.igdb.get(key, 0.0), + co_play=signals.co_play.get(key, 0.0), + co_collection=signals.co_collection.get(key, 0.0), + average_rating=features[candidate_id].average_rating, + ) + + if score < MIN_EDGE_SCORE: + continue + + reasons = shared_reasons(source_vector, candidate_vector) + if key in signals.igdb: + reasons.append({"facet": "igdb", "value": "similar"}) + + scored.append((score, candidate_id, reasons)) + + scored.sort(key=lambda item: (-item[0], item[1])) + + # Second pass, over the ranked list: drop neighbours that duplicate the + # source or each other. The per-candidate check above compares against + # the source alone, so two discs of one release would each take a slot. + edges: list[dict[str, Any]] = [] + taken_igdb_ids: set[int] = set() + taken_titles: set[str] = set() + series_counts: dict[str, int] = {} + source_title = feature.title_key + + for score, candidate_id, reasons in scored: + candidate_igdb_id = igdb_ids.get(candidate_id) + if candidate_igdb_id is not None: + if candidate_igdb_id in taken_igdb_ids: + continue + taken_igdb_ids.add(candidate_igdb_id) + + # The same game on another platform is not a recommendation, and + # IGDB gives every port its own id, so the id check above cannot + # catch it. Two ports collide with each other as readily as with + # the source, hence both comparisons. + candidate_title = features[candidate_id].title_key + if candidate_title: + if candidate_title == source_title or candidate_title in taken_titles: + continue + taken_titles.add(candidate_title) + + series = _series_tokens(features[candidate_id]) + if series and any( + series_counts.get(token, 0) >= MAX_STORED_PER_SERIES for token in series + ): + continue + for token in series: + series_counts[token] = series_counts.get(token, 0) + 1 + + edges.append( + { + "rom_id": rom_id, + "related_rom_id": candidate_id, + "score": round(score, 6), + "reasons": reasons, + } + ) + if len(edges) >= MAX_NEIGHBOURS: + break + + return edges + + def _flush(self, rom_ids: list[int], edges: list[dict[str, Any]]) -> None: + if not rom_ids: + return + + self.stats.edges_written += db_recommendation_handler.replace_similarity_edges( + rom_ids, edges + ) + self._report() + + def _report(self) -> None: + if self._progress: + self._progress(self.stats) diff --git a/backend/handler/recommendation/diversity.py b/backend/handler/recommendation/diversity.py new file mode 100644 index 0000000000..b319dab9b3 --- /dev/null +++ b/backend/handler/recommendation/diversity.py @@ -0,0 +1,108 @@ +"""Keeps a recommendation list from collapsing into one series. + +Similarity ranking alone puts every Metroid game above every Metroidvania, +which makes "Similar games" a duplicate of a franchise filter. Someone who +owns Super Metroid already knows Metroid exists; the useful suggestion is +Castlevania. + +Applied when serving rather than when building, so the policy can change +without rebuilding the index. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from typing import Final, TypeVar + +from models.rom import Rom + +# Neighbours allowed from any one series before the rest are dropped. Low +# enough that a deep franchise cannot fill the section on its own, high enough +# that a close same-series match is not traded away for a far weaker unrelated +# one: past the franchise, scores fall off a cliff. +MAX_PER_SERIES: Final = 3 + +T = TypeVar("T") + + +def series_keys(rom: Rom) -> set[str]: + """Every series a game belongs to, franchises and collections alike. + + All of them, not just the first: IGDB lists a game's franchises in no + stable order, so keying on one entry splits a single real series across + several counters. Madden titles carry both "Madden" and "NFL", and the + cap let four through -- two counted against each. + """ + metadatum = rom.metadatum + if metadatum is None: + return set() + + return { + str(value) + for values in (metadatum.franchises, metadatum.collections) + for value in (values or []) + if value + } + + +def cap_by_series( + items: Iterable[T], + resolve_rom: Callable[[T], Rom | None], + *, + limit: int, + max_per_series: int = MAX_PER_SERIES, + max_per_platform: int | None = None, +) -> list[T]: + """Take items in order, allowing at most `max_per_series` from each series. + + Games with no series are never capped: they have nothing to cluster on, so + treating them as one giant group would suppress most of an unmatched shelf. + + `max_per_platform` additionally stops one platform owning the result, which + the personalised feed wants and a single game's "Similar games" does not. + """ + # Positions are tracked so backfilled entries slot back into score order + # rather than being appended after lower-scoring ones. + selected: list[tuple[int, T]] = [] + overflow: list[tuple[int, T]] = [] + counts: dict[str, int] = {} + platform_counts: dict[int, int] = {} + + for position, item in enumerate(items): + rom = resolve_rom(item) + if rom is None: + continue + + keys = series_keys(rom) + # Saturated on any one of its series is enough: a game sharing a + # franchise with two already-picked entries is the repetition the cap + # exists to stop, whichever of its franchises that happens to be. + if keys and any(counts.get(key, 0) >= max_per_series for key in keys): + overflow.append((position, item)) + continue + + if ( + max_per_platform is not None + and platform_counts.get(rom.platform_id, 0) >= max_per_platform + ): + overflow.append((position, item)) + continue + + for key in keys: + counts[key] = counts.get(key, 0) + 1 + platform_counts[rom.platform_id] = platform_counts.get(rom.platform_id, 0) + 1 + + selected.append((position, item)) + if len(selected) >= limit: + return [item for _, item in selected] + + # A shelf sitting deep in one franchise can cap away nearly everything, + # leaving a section with two entries or none. A slightly repetitive row + # beats an empty one, so the capped-out candidates backfill it. + for entry in overflow: + if len(selected) >= limit: + break + selected.append(entry) + + selected.sort(key=lambda entry: entry[0]) + return [item for _, item in selected] diff --git a/backend/handler/recommendation/feed.py b/backend/handler/recommendation/feed.py new file mode 100644 index 0000000000..db75da5159 --- /dev/null +++ b/backend/handler/recommendation/feed.py @@ -0,0 +1,358 @@ +"""Builds a user's personalised recommendation feed. + +Computed on demand from the precomputed similarity edges plus live activity, +rather than precomputed per user: a feed built nightly would ignore the game +someone played an hour ago, which is exactly the signal that matters most. +""" + +from __future__ import annotations + +import json +import math +from collections.abc import Sequence +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Final + +from handler.database import db_recommendation_handler, db_rom_handler +from handler.database.recommendations_handler import UserAffinityRow +from handler.recommendation.diversity import cap_by_series +from handler.redis_handler import sync_cache +from logger.logger import log +from models.rom import Rom, RomUserStatus + +# The user rating scale is 1-10; 5.5 is the indifference point, so anything +# below it pushes similar games *away* rather than merely not pulling them in. +NEUTRAL_RATING: Final = 5.5 +MAX_RATING: Final = 10.0 + +# Playtime saturates: the difference between 1h and 10h says a lot, the +# difference between 100h and 200h says almost nothing. +PLAYTIME_SATURATION_HOURS: Final = 20.0 + +# Taste drifts. A game played last week should steer the feed more than one +# finished two years ago, without the old one falling off entirely. +RECENCY_HALFLIFE_DAYS: Final = 60.0 +MIN_RECENCY_FACTOR: Final = 0.15 + +STATUS_AFFINITY: Final[dict[str, float]] = { + RomUserStatus.COMPLETED_100.value: 1.0, + RomUserStatus.FINISHED.value: 0.8, + RomUserStatus.RETIRED.value: 0.4, + RomUserStatus.INCOMPLETE.value: 0.3, +} + +# Games the user has already played still belong in the feed (a sequel to +# something you finished is a fine suggestion), but an unplayed game you own +# is the outcome this feature exists to produce. +PLAYED_NOVELTY_FACTOR: Final = 0.3 + +# Disliked games steer, but less forcefully than loved ones. +NEGATIVE_SEED_DAMPING: Final = 0.5 + +# Ceiling applied during diversification, so one platform cannot own the row. +# The per-series ceiling is shared with the similar-games surface. +MAX_PER_PLATFORM: Final = 4 + +# Seeds are read in descending affinity; beyond this the contribution is noise. +MAX_SEEDS: Final = 60 + +# How many candidates to hydrate before diversifying down to the final count. +OVERFETCH_FACTOR: Final = 5 + +FEED_CACHE_TTL_SECONDS: Final = 900 +FEED_CACHE_PREFIX: Final = "recommendations:feed" + + +@dataclass +class RecommendedRom: + rom: Rom + score: float + reasons: list[dict[str, str]] = field(default_factory=list) + seed_rom_id: int | None = None + seed_rom_name: str | None = None + + +@dataclass +class _Candidate: + rom_id: int + score: float = 0.0 + best_seed_id: int | None = None + best_seed_contribution: float = 0.0 + reasons: list[dict[str, str]] = field(default_factory=list) + + +def seed_affinity(row: UserAffinityRow, *, now: datetime | None = None) -> float: + """How strongly one played game should steer the feed, in roughly [-1, 1]. + + Negative for games the user rated below the midpoint, which is what lets + the feed learn "not this kind of thing". + """ + signals: list[float] = [] + + if row.playtime_ms > 0: + hours = row.playtime_ms / 3_600_000 + signals.append( + min(1.0, math.log1p(hours) / math.log1p(PLAYTIME_SATURATION_HOURS)) + ) + + if row.rating: + # Maps 10 -> +1.0 and 1 -> -1.0, crossing zero at the scale midpoint. + signals.append( + max( + -1.0, + min(1.0, (row.rating - NEUTRAL_RATING) / (MAX_RATING - NEUTRAL_RATING)), + ) + ) + + if row.status and row.status in STATUS_AFFINITY: + signals.append(STATUS_AFFINITY[row.status]) + + if row.now_playing: + signals.append(1.0) + + if not signals: + # Played at some point, but nothing else is known about it. + return ( + 0.2 * _recency_factor(row.last_played, now=now) if row.last_played else 0.0 + ) + + affinity = sum(signals) / len(signals) + if affinity < 0: + affinity *= NEGATIVE_SEED_DAMPING + + return affinity * _recency_factor(row.last_played, now=now) + + +def _recency_factor( + last_played: datetime | None, *, now: datetime | None = None +) -> float: + """Exponential decay on time since last played, floored so old loves persist.""" + if last_played is None: + return MIN_RECENCY_FACTOR + + reference = now or datetime.now(timezone.utc) + if last_played.tzinfo is None: + last_played = last_played.replace(tzinfo=timezone.utc) + + days = max(0.0, (reference - last_played).total_seconds() / 86_400) + decayed = math.pow(0.5, days / RECENCY_HALFLIFE_DAYS) + return max(MIN_RECENCY_FACTOR, decayed) + + +class FeedBuilder: + """Ranks candidates for one user from the precomputed similarity graph.""" + + def __init__(self, user_id: int) -> None: + self.user_id = user_id + + def build(self, limit: int = 20) -> list[RecommendedRom]: + affinity_rows = db_recommendation_handler.get_user_affinity(self.user_id) + + seeds, excluded = self._partition(affinity_rows) + if not seeds: + return self._cold_start(limit, excluded) + + candidates = self._accumulate(seeds, excluded) + if not candidates: + return self._cold_start(limit, excluded) + + played_rom_ids = { + row.rom_id for row in affinity_rows if row.last_played is not None + } + return self._rank(candidates, played_rom_ids, seeds, limit) + + # --- Stages ------------------------------------------------------------------ + + def _partition( + self, rows: Sequence[UserAffinityRow] + ) -> tuple[dict[int, float], set[int]]: + """Split the user's library interactions into seeds and hard exclusions.""" + seeds: dict[int, float] = {} + excluded: set[int] = set() + + for row in rows: + if row.hidden or row.status == RomUserStatus.NEVER_PLAYING.value: + excluded.add(row.rom_id) + # Never-played-on-purpose is a filtering decision, not a taste + # signal, so it contributes nothing to the seed set. + if row.status == RomUserStatus.NEVER_PLAYING.value: + continue + + affinity = seed_affinity(row) + if abs(affinity) > 0.01: + seeds[row.rom_id] = affinity + + # The seeds themselves are never their own recommendations. + excluded.update(seeds) + + if len(seeds) > MAX_SEEDS: + strongest = sorted(seeds.items(), key=lambda kv: -abs(kv[1]))[:MAX_SEEDS] + seeds = dict(strongest) + + return seeds, excluded + + def _accumulate( + self, seeds: dict[int, float], excluded: set[int] + ) -> dict[int, _Candidate]: + """Fan out from every seed, summing weighted edge scores per candidate.""" + edges = db_recommendation_handler.get_neighbours_for_roms(list(seeds)) + candidates: dict[int, _Candidate] = {} + + for seed_id, related_id, edge_score, reasons in edges: + if related_id in excluded: + continue + + contribution = edge_score * seeds[seed_id] + candidate = candidates.get(related_id) + if candidate is None: + candidate = _Candidate(rom_id=related_id) + candidates[related_id] = candidate + + candidate.score += contribution + # Attribute the recommendation to whichever seed pulled hardest, so + # "Because you played X" names the game that actually caused it. + if contribution > candidate.best_seed_contribution: + candidate.best_seed_contribution = contribution + candidate.best_seed_id = seed_id + candidate.reasons = list(reasons) + + return { + rom_id: candidate + for rom_id, candidate in candidates.items() + if candidate.score > 0 + } + + def _rank( + self, + candidates: dict[int, _Candidate], + played_rom_ids: set[int], + seeds: dict[int, float], + limit: int, + ) -> list[RecommendedRom]: + for candidate in candidates.values(): + if candidate.rom_id in played_rom_ids: + candidate.score *= PLAYED_NOVELTY_FACTOR + + ordered = sorted(candidates.values(), key=lambda c: (-c.score, c.rom_id))[ + : limit * OVERFETCH_FACTOR + ] + + roms = _hydrate([candidate.rom_id for candidate in ordered]) + seed_names = db_recommendation_handler.get_rom_names( + [ + candidate.best_seed_id + for candidate in ordered + if candidate.best_seed_id is not None + ] + ) + + # Shared with the "Similar games" surface so a series is counted by + # every name it goes under, not by one representative. + chosen = cap_by_series( + ordered, + lambda candidate: roms.get(candidate.rom_id), + limit=limit, + max_per_platform=MAX_PER_PLATFORM, + ) + + return [ + RecommendedRom( + rom=roms[candidate.rom_id], + score=round(candidate.score, 6), + reasons=candidate.reasons, + seed_rom_id=candidate.best_seed_id, + seed_rom_name=seed_names.get(candidate.best_seed_id or -1), + ) + for candidate in chosen + ] + + def _cold_start(self, limit: int, excluded: set[int]) -> list[RecommendedRom]: + """No usable activity yet, so fall back to the library's best-reviewed.""" + log.debug(f"No recommendation seeds for user {self.user_id}, using fallback") + + rom_ids = db_recommendation_handler.get_fallback_rom_ids( + limit, exclude_rom_ids=list(excluded) + ) + roms = _hydrate(rom_ids) + + return [ + RecommendedRom( + rom=roms[rom_id], + score=0.0, + reasons=[{"facet": "top_rated", "value": ""}], + ) + for rom_id in rom_ids + if rom_id in roms + ] + + +def _hydrate(rom_ids: Sequence[int]) -> dict[int, Rom]: + """Load ROMs through the shared `SimpleRomSchema` load path. + + Missing-from-disk ROMs are dropped here rather than filtered in SQL, so + the edge queries never have to join the wide `roms` table. + """ + return { + rom.id: rom + for rom in db_rom_handler.get_roms_simple_by_ids(list(rom_ids)) + if not rom.missing_from_fs + } + + +def get_cached_feed(user_id: int, limit: int) -> list[RecommendedRom] | None: + """Read a cached feed, re-hydrating the ROMs so visibility stays live.""" + raw = sync_cache.get(_cache_key(user_id, limit)) + if not raw: + return None + + try: + entries: list[dict[str, Any]] = json.loads(raw) + except (ValueError, TypeError): + return None + + roms = _hydrate([entry["rom_id"] for entry in entries]) + + return [ + RecommendedRom( + rom=roms[entry["rom_id"]], + score=entry["score"], + reasons=entry.get("reasons") or [], + seed_rom_id=entry.get("seed_rom_id"), + seed_rom_name=entry.get("seed_rom_name"), + ) + for entry in entries + if entry["rom_id"] in roms + ] + + +def set_cached_feed(user_id: int, limit: int, feed: Sequence[RecommendedRom]) -> None: + """Cache only the ranking, never the ROM rows themselves.""" + payload = json.dumps( + [ + { + "rom_id": item.rom.id, + "score": item.score, + "reasons": item.reasons, + "seed_rom_id": item.seed_rom_id, + "seed_rom_name": item.seed_rom_name, + } + for item in feed + ] + ) + sync_cache.set(_cache_key(user_id, limit), payload, ex=FEED_CACHE_TTL_SECONDS) + + +def invalidate_cached_feed(user_id: int) -> None: + for key in sync_cache.scan_iter(f"{FEED_CACHE_PREFIX}:{user_id}:*"): + sync_cache.delete(key) + + +def invalidate_all_cached_feeds() -> None: + """Drop every user's ranking, for when the graph underneath it changes.""" + for key in sync_cache.scan_iter(f"{FEED_CACHE_PREFIX}:*"): + sync_cache.delete(key) + + +def _cache_key(user_id: int, limit: int) -> str: + return f"{FEED_CACHE_PREFIX}:{user_id}:{limit}" diff --git a/backend/handler/recommendation/scoring.py b/backend/handler/recommendation/scoring.py new file mode 100644 index 0000000000..823f8d7a06 --- /dev/null +++ b/backend/handler/recommendation/scoring.py @@ -0,0 +1,458 @@ +"""Pure scoring primitives for the recommendation engine. + +Deliberately free of ORM and I/O so the ranking maths can be exercised +directly in tests. Everything here operates on plain dataclasses and dicts. + +The engine is library-relative: a facet is only as informative as it is rare +*in this library*. A shelf of 4000 arcade games learns that "Action" says +nothing and that "Metroidvania" says a great deal, without anyone tuning it. +""" + +from __future__ import annotations + +import math +from collections import Counter, defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Final + +# Relative pull of each facet before IDF weighting. A shared series or +# franchise is far stronger evidence of "you will like this too" than a shared +# genre, and platform/decade are context rather than taste. +FACET_WEIGHTS: Final[Mapping[str, float]] = { + "collection": 3.0, + "franchise": 2.5, + "genre": 1.0, + # IGDB's curated viewpoint list. "Side view" versus "First person" says + # more about how a game plays than most genre labels do. + "perspective": 1.0, + # A secondary genre axis (Horror, Comedy, Fantasy), curated and low + # cardinality, so it earns close to a genre's weight. + "theme": 0.9, + # Community tags. High cardinality and mixed quality ("motorcycle" sits + # beside "metroidvania"), so IDF does most of the work and the weight + # stays below the curated facets. This is the least settled of the + # weights: on a 12.7k library 0.8 surfaced real golf games for Golf while + # 0.5 kept 2D Mario platformers ahead of Mario Tennis. Revisit with the + # inspection tool against a real shelf before trusting it. + "keyword": 0.7, + # Who actually made it. Set to what the merged `company` facet carried + # before the split, so separating the roles redistributes that weight + # rather than adding new influence. + # + # Sweeping it from 1.0 down to 0.4 barely moved results: tight studios + # (Treasure, Sacnoth) hold their matches at every value because their + # games also share genre and theme, and wide-ranging ones (Neversoft) + # only improve at the very bottom of the range. + "developer": 0.7, + # Who shipped it. A label spans everything it ever released, and regional + # distributors land here too -- Tec Toy alone covers 774 games on a 15k + # library, dense enough that IDF does not suppress it on its own. + "publisher": 0.25, + # Used only where no provider reported roles, so the role is unknown and + # the value could be either. Below genre for the same reason publisher is. + "company": 0.7, + # Nearly every game is "Single player", so this mostly rides along; IDF + # already flattens it and the low weight keeps it from breaking ties. + "game_mode": 0.4, + "platform": 0.4, + "decade": 0.3, +} + +# Facets present on more than this share of the library describe the library, +# not the game. They still contribute to the score (with a tiny IDF) but are +# skipped when generating candidates, so "Action" never expands into a +# postings list covering most of the shelf. +MAX_CANDIDATE_DF_RATIO: Final = 0.20 +MAX_CANDIDATE_POSTINGS: Final = 2_000 +# ...but the ratio only makes sense once there is a library to take a ratio of. +# Below this, expanding every token is cheap, and skipping them would leave a +# small shelf with no candidates at all. +MIN_CANDIDATE_DF: Final = 50 + +# Blend of the four independent signals. These sum to 1.0 so a raw score is +# readable as "fraction of maximum possible relatedness". +CONTENT_WEIGHT: Final = 0.55 +IGDB_PRIOR_WEIGHT: Final = 0.20 +CO_PLAY_WEIGHT: Final = 0.15 +CO_COLLECTION_WEIGHT: Final = 0.10 + +# Similar *and* worth playing: a small nudge from critic rating, capped low +# enough that it reorders ties without overriding genuine relatedness. +MAX_QUALITY_BONUS: Final = 0.05 + +# Length-normalisation strength: 1.0 is plain L2, 0.0 scales every vector by +# the library average instead of its own length. +# +# Zero, against the 0.75 that text retrieval uses, because facet counts are not +# verbosity. A long document repeating a word is not more relevant, which is +# why retrieval normalises it away; but a game tagged with three genres and two +# franchises genuinely has more in common than one carrying a single tag, and +# dividing by its own length punished it for being well documented. +# +# Measured on a 12.7k-game library: at 0.75 every top match for a Mario +# compilation was a 6-8 token entry (Golf, F-1 Race, Pinball); at 0.0 they were +# 12-16 token entries (Yoshi's Island, Super Mario 64, Super Mario Kart). The +# feared popularity bias did not appear -- across 300 sampled games the most +# repeated recommendation fell from 6 lists to 3, and distinct results rose +# from 1363 to 1384. +PIVOT_B: Final = 0.0 + +# Release proximity matters, but only softly: a decade token already carries +# most of the era signal. +SAME_DECADE_TOKEN: Final = "decade" + + +# Facets that describe the game itself rather than where it sits on the shelf. +# A ROM carrying none of these has nothing to be similar *about*: platform and +# decade alone would make every unmatched file in a folder a perfect match for +# every other, since both vectors normalise to the same thing. +TASTE_FACETS: Final[frozenset[str]] = frozenset( + { + "genre", + "franchise", + "collection", + "company", + "game_mode", + "developer", + "publisher", + "keyword", + "theme", + "perspective", + } +) + + +def has_taste_signal(tokens: Sequence[str]) -> bool: + """Whether a ROM carries any facet worth computing similarity from.""" + return any(token_facet(token) in TASTE_FACETS for token in tokens) + + +def make_token(facet: str, value: str) -> str: + """Namespace a facet value so genre:Action never collides with tag:Action.""" + return f"{facet}:{value}" + + +def token_facet(token: str) -> str: + return token.split(":", 1)[0] + + +def token_value(token: str) -> str: + return token.split(":", 1)[1] if ":" in token else token + + +@dataclass(slots=True) +class RomFeatures: + """Everything the scorer needs about one ROM.""" + + rom_id: int + platform_id: int + tokens: tuple[str, ...] = () + average_rating: float | None = None + # Normalised title, used to spot the same game released on another + # platform, which IGDB indexes as a separate id. + title_key: str | None = None + + +@dataclass(slots=True) +class ScoredNeighbour: + """One edge of the item-item graph, with its explanation.""" + + rom_id: int + score: float + reasons: list[dict[str, str]] = field(default_factory=list) + + +def extract_tokens( + *, + platform_id: int, + genres: Sequence[str] | None = None, + franchises: Sequence[str] | None = None, + collections: Sequence[str] | None = None, + companies: Sequence[str] | None = None, + developers: Sequence[str] | None = None, + publishers: Sequence[str] | None = None, + game_modes: Sequence[str] | None = None, + keywords: Sequence[str] | None = None, + themes: Sequence[str] | None = None, + player_perspectives: Sequence[str] | None = None, + first_release_date: int | None = None, +) -> tuple[str, ...]: + """Flatten a ROM's metadata into namespaced, deduplicated feature tokens.""" + tokens: list[str] = [] + + # Prefer the role-split lists where a provider reported them, and fall back + # to the merged one otherwise. Emitting both would count an IGDB-matched + # game's studio twice while a game matched elsewhere counted once. + has_roles = bool(developers) or bool(publishers) + company_facets: tuple[tuple[str, Sequence[str] | None], ...] = ( + (("developer", developers), ("publisher", publishers)) + if has_roles + else (("company", companies),) + ) + + for facet, values in ( + ("genre", genres), + ("franchise", franchises), + ("collection", collections), + *company_facets, + ("game_mode", game_modes), + ("keyword", keywords), + ("theme", themes), + ("perspective", player_perspectives), + ): + for value in values or (): + cleaned = (value or "").strip() + if cleaned: + tokens.append(make_token(facet, cleaned)) + + tokens.append(make_token("platform", str(platform_id))) + + year = release_year_from_epoch(first_release_date) + if year is not None: + tokens.append(make_token(SAME_DECADE_TOKEN, str(year // 10 * 10))) + + # dict.fromkeys keeps first-seen order, which keeps reasons deterministic. + return tuple(dict.fromkeys(tokens)) + + +def release_year_from_epoch(first_release_date: int | None) -> int | None: + """Metadata stores release dates as a UTC epoch in seconds.""" + if not first_release_date: + return None + try: + # Guard against the occasional millisecond value from a bad provider row. + seconds = ( + first_release_date // 1000 + if abs(first_release_date) > 10_000_000_000 + else first_release_date + ) + return 1970 + int(seconds // 31_556_952) + except (TypeError, ValueError, OverflowError): + return None + + +def compute_idf( + documents: Iterable[Sequence[str]], total_documents: int +) -> dict[str, float]: + """Inverse document frequency over the library's token vocabulary. + + Uses the BM25 form, ``ln(1 + (N - df + 0.5) / (df + 0.5))``. The simpler + ``ln(1 + N / (1 + df))`` was tried first and discriminates far too weakly: + a token on every ROM still scored ~0.69 against ~1.9 for a rare one, so + "Single player" (present on nearly every game) kept enough weight to pull + unrelated titles above genuine genre matches. BM25 drives the universal + token to ~0.02 while leaving the rare one untouched, and stays positive on + tiny libraries where a plain ``ln(N / df)`` collapses every token to zero. + """ + if total_documents <= 0: + return {} + + document_frequency: Counter[str] = Counter() + for tokens in documents: + document_frequency.update(set(tokens)) + + return { + token: math.log(1 + (total_documents - df + 0.5) / (df + 0.5)) + for token, df in document_frequency.items() + } + + +def build_vector(tokens: Sequence[str], idf: Mapping[str, float]) -> dict[str, float]: + """Raw facet-weighted IDF vector, before any length normalisation. + + A facet's weight is split across however many values it holds, so one + franchise counts for more than one of six. Without it, a compilation + carrying several franchises matches strongly on all of them: the SNES + Mario compilation pulled in Mario Tennis and Mario Party ahead of the 2D + platformers it actually resembles. + + Splitting by sqrt rather than the count itself keeps a multi-value facet + worth more in total than a single-value one -- three genres really is more + information than one -- while stopping it from scaling linearly. + """ + facet_counts = Counter(token_facet(token) for token in tokens) + + raw = { + token: ( + FACET_WEIGHTS.get(token_facet(token), 1.0) + * idf.get(token, 0.0) + / math.sqrt(facet_counts[token_facet(token)]) + ) + for token in tokens + } + return {token: weight for token, weight in raw.items() if weight > 0} + + +def vector_norm(vector: Mapping[str, float]) -> float: + return math.sqrt(sum(weight * weight for weight in vector.values())) + + +def pivot_length(norm: float, average_norm: float, *, b: float = PIVOT_B) -> float: + """Blend a vector's own length with the library average. + + Plain L2 normalisation (b=1) divides by the vector's own length, which + hands sparsely-tagged games an advantage: with only a few tokens each one + carries enormous weight, so a game sharing one broad facet outscores a + richly-tagged game sharing three. See PIVOT_B for why the default is 0. + """ + if average_norm <= 0: + return norm or 1.0 + return (1.0 - b) * average_norm + b * norm + + +def normalise(vector: Mapping[str, float], pivot: float) -> dict[str, float]: + if pivot <= 0: + return {} + return {token: weight / pivot for token, weight in vector.items()} + + +def build_normalised_vectors( + token_sets: Mapping[int, Sequence[str]], idf: Mapping[str, float] +) -> dict[int, dict[str, float]]: + """Vectors for a whole library, pivot-normalised against its average length. + + Needs the full set up front because the pivot is relative to the library, + the same way the IDF weighting is. + """ + raw = {key: build_vector(tokens, idf) for key, tokens in token_sets.items()} + norms = {key: vector_norm(vector) for key, vector in raw.items()} + + populated = [norm for norm in norms.values() if norm > 0] + average_norm = sum(populated) / len(populated) if populated else 0.0 + + return { + key: normalise(vector, pivot_length(norms[key], average_norm)) + for key, vector in raw.items() + } + + +def content_similarity(left: Mapping[str, float], right: Mapping[str, float]) -> float: + """Dot product of two pivot-normalised vectors.""" + # Iterate the smaller side; token overlap is sparse. + if len(left) > len(right): + left, right = right, left + return sum( + weight * right[token] for token, weight in left.items() if token in right + ) + + +def shared_reasons( + left: Mapping[str, float], + right: Mapping[str, float], + *, + limit: int = 3, +) -> list[dict[str, str]]: + """The facets that actually drove the score, strongest first. + + These are what the UI renders as "Same series as Super Metroid" rather + than an unexplained list of covers. + """ + contributions = [ + (weight * right[token], token) + for token, weight in left.items() + if token in right + ] + # Keywords are ranked last regardless of contribution. They are the rarest + # tokens, so they carry the highest IDF and would otherwise always win the + # slot -- explaining a match with "drawbridge" or "frankenstein's monster" + # when the two games are really both Castlevanias. They still earn a slot + # once the curated facets are exhausted, where "interconnected-world" says + # something no genre can. + contributions.sort( + key=lambda pair: (token_facet(pair[1]) == "keyword", -pair[0], pair[1]) + ) + + reasons: list[dict[str, str]] = [] + seen_facets: set[str] = set() + for _, token in contributions: + facet = token_facet(token) + # One reason per facet: three shared genres reads worse than a genre, + # a company and a decade. + if facet in seen_facets or facet == "platform": + continue + seen_facets.add(facet) + reasons.append({"facet": facet, "value": token_value(token)}) + if len(reasons) >= limit: + break + + return reasons + + +def quality_bonus(average_rating: float | None) -> float: + """Map a 0-100 critic rating onto a small additive bonus.""" + if not average_rating: + return 0.0 + normalised = max(0.0, min(1.0, average_rating / 100.0)) + return MAX_QUALITY_BONUS * normalised + + +def blend( + *, + content: float, + igdb_prior: float = 0.0, + co_play: float = 0.0, + co_collection: float = 0.0, + average_rating: float | None = None, +) -> float: + """Combine the independent signals into a single 0-1-ish score.""" + return ( + CONTENT_WEIGHT * _clamp(content) + + IGDB_PRIOR_WEIGHT * _clamp(igdb_prior) + + CO_PLAY_WEIGHT * _clamp(co_play) + + CO_COLLECTION_WEIGHT * _clamp(co_collection) + + quality_bonus(average_rating) + ) + + +def _clamp(value: float) -> float: + return max(0.0, min(1.0, value)) + + +def build_inverted_index( + features: Mapping[int, RomFeatures], +) -> dict[str, list[int]]: + """Token -> ROM ids, used to avoid the O(n^2) all-pairs comparison. + + Only ROMs sharing at least one *discriminative* token are ever scored + against each other, which is what keeps a 50k-ROM library tractable. + """ + postings: dict[str, list[int]] = defaultdict(list) + for rom_id, feature in features.items(): + for token in feature.tokens: + postings[token].append(rom_id) + return dict(postings) + + +def candidate_ids( + feature: RomFeatures, + postings: Mapping[str, Sequence[int]], + total_documents: int, +) -> set[int]: + """Candidate neighbours for one ROM, drawn from its rarest facets.""" + df_cap = max(MIN_CANDIDATE_DF, int(total_documents * MAX_CANDIDATE_DF_RATIO)) + candidates: set[int] = set() + + for token in feature.tokens: + bucket = postings.get(token) + if not bucket: + continue + if len(bucket) > df_cap or len(bucket) > MAX_CANDIDATE_POSTINGS: + continue + candidates.update(bucket) + + candidates.discard(feature.rom_id) + return candidates + + +def normalise_co_occurrence( + pair_count: float, left_total: int, right_total: int +) -> float: + """Cosine-style normalisation of a raw co-occurrence count. + + Without this, whatever ROM sits in the most collections (or has the most + play sessions) would look related to everything. + """ + if pair_count <= 0 or left_total <= 0 or right_total <= 0: + return 0.0 + return min(1.0, pair_count / math.sqrt(left_total * right_total)) diff --git a/backend/main.py b/backend/main.py index bba55b4706..3cecbefbbd 100644 --- a/backend/main.py +++ b/backend/main.py @@ -48,6 +48,7 @@ from endpoints.permissions import router as permissions_router from endpoints.platform import router as platform_router from endpoints.play_sessions import router as play_sessions_router +from endpoints.recommendations import router as recommendations_router from endpoints.roms import router as rom_router from endpoints.saves import router as saves_router from endpoints.screenshots import router as screenshots_router @@ -177,6 +178,7 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None]: app.include_router(play_sessions_router, prefix="/api") app.include_router(platform_router, prefix="/api") app.include_router(rom_router, prefix="/api") +app.include_router(recommendations_router, prefix="/api") app.include_router(music_router, prefix="/api") app.include_router(music_playlists_router, prefix="/api") app.include_router(search_router, prefix="/api") diff --git a/backend/models/recommendation.py b/backend/models/recommendation.py new file mode 100644 index 0000000000..e19c0b1a47 --- /dev/null +++ b/backend/models/recommendation.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from sqlalchemy import Float, ForeignKey, Index +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from models.base import BaseModel +from utils.database import CustomJSON + +if TYPE_CHECKING: + from models.rom import Rom + + +class RomSimilarity(BaseModel): + """A precomputed edge of the item-item similarity graph. + + Written wholesale by the recommendations task, never incrementally, so the + rows are always consistent with a single IDF snapshot of the library. Only + the top neighbours of each ROM are kept, which bounds the table at + roughly ``rom_count * MAX_NEIGHBOURS`` rows. + + Edges are stored in both directions. The scoring itself is symmetric, but + the per-ROM top-N cut is not (a niche game's best neighbour may not + reciprocate), and duplicating avoids an OR across two indexed columns on + every read. + """ + + __tablename__ = "rom_similarity" + + __table_args__ = ( + Index("idx_rom_similarity_rom_score", "rom_id", "score"), + Index("idx_rom_similarity_related_rom_id", "related_rom_id"), + ) + + rom_id: Mapped[int] = mapped_column( + ForeignKey("roms.id", ondelete="CASCADE"), primary_key=True + ) + related_rom_id: Mapped[int] = mapped_column( + ForeignKey("roms.id", ondelete="CASCADE"), primary_key=True + ) + + score: Mapped[float] = mapped_column(Float(), nullable=False) + + # The facets that drove the score, e.g. [{"facet": "franchise", + # "value": "Metroid"}], so the UI can say why without recomputing. + reasons: Mapped[list[dict[str, Any]] | None] = mapped_column( + CustomJSON(), default=[] + ) + + # No ORM-level delete cascade: ROMs are removed with a bulk `delete()` + # (see `db_rom_handler.delete_rom`), which never runs one. The foreign + # keys' ON DELETE CASCADE is what actually clears both directions. + rom: Mapped[Rom] = relationship( + "Rom", + foreign_keys=[rom_id], + back_populates="similar_roms", + lazy="raise", + passive_deletes=True, + ) + related_rom: Mapped[Rom] = relationship( + "Rom", foreign_keys=[related_rom_id], lazy="raise", passive_deletes=True + ) diff --git a/backend/models/rom.py b/backend/models/rom.py index ec4681b30a..22fa16b8d5 100644 --- a/backend/models/rom.py +++ b/backend/models/rom.py @@ -87,6 +87,7 @@ def compute_name_sort_key(name: str | None) -> str: from models.assets import Save, Screenshot, State from models.collection import Collection from models.platform import Platform + from models.recommendation import RomSimilarity from models.user import User @@ -255,11 +256,25 @@ class RomMetadata(BaseModel): franchises: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) collections: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) companies: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) + # `companies` split by IGDB involvement role. A developer's games really do + # resemble each other; a publisher spans everything it ever shipped. + developers: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) + publishers: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) game_modes: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) age_ratings: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) + # IGDB-only descriptors: community tags plus the curated theme and + # viewpoint lists. Far more specific about how a game plays than genre. + keywords: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) + themes: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) + player_perspectives: Mapped[list[str] | None] = mapped_column( + CustomJSON(), default=[] + ) player_count: Mapped[str | None] = mapped_column(String(length=100), default="1") first_release_date: Mapped[int | None] = mapped_column(BigInteger(), default=None) average_rating: Mapped[float | None] = mapped_column(default=None) + # Votes behind `average_rating`, from IGDB. Zero where no provider + # reported one, which is how an unbacked perfect score is spotted. + rating_count: Mapped[int | None] = mapped_column(BigInteger(), default=0) rom: Mapped[Rom] = relationship(lazy="joined", back_populates="metadatum") @@ -287,8 +302,15 @@ class RomFacets(BaseModel): franchises: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) collections: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) companies: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) + developers: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) + publishers: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) game_modes: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) age_ratings: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) + keywords: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) + themes: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) + player_perspectives: Mapped[list[str] | None] = mapped_column( + CustomJSON(), default=[] + ) player_count: Mapped[str | None] = mapped_column(String(length=100), default="1") regions: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) languages: Mapped[list[str] | None] = mapped_column(CustomJSON(), default=[]) @@ -494,6 +516,13 @@ class Rom(BaseModel): lazy="raise", back_populates="roms", ) + similar_roms: Mapped[list[RomSimilarity]] = relationship( + "RomSimilarity", + foreign_keys="RomSimilarity.rom_id", + lazy="raise", + back_populates="rom", + passive_deletes=True, + ) def __init__(self, *args: Any, **kwargs: Any) -> None: super().__init__(*args, **kwargs) diff --git a/backend/startup.py b/backend/startup.py index 4d58a41cf3..655ebec864 100644 --- a/backend/startup.py +++ b/backend/startup.py @@ -7,6 +7,7 @@ from rq.job import Job from config import ( + ENABLE_SCHEDULED_BUILD_RECOMMENDATIONS, ENABLE_SCHEDULED_CONVERT_IMAGES_TO_WEBP, ENABLE_SCHEDULED_RESCAN, ENABLE_SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC, @@ -32,6 +33,7 @@ from tasks.manual.recompute_save_content_hashes import ( recompute_save_content_hashes_task, ) +from tasks.scheduled.build_recommendations import build_recommendations_task from tasks.scheduled.cleanup_netplay import cleanup_netplay_task from tasks.scheduled.cleanup_orphaned_resources import cleanup_orphaned_resources_task from tasks.scheduled.cleanup_upload_tmp import cleanup_upload_tmp_task @@ -159,6 +161,9 @@ async def main() -> None: log.info("Starting scheduled convert images to webp") convert_images_to_webp_task.init() _enqueue_convert_images_to_webp() + if ENABLE_SCHEDULED_BUILD_RECOMMENDATIONS: + log.info("Starting scheduled recommendations index build") + build_recommendations_task.init() if ENABLE_SCHEDULED_RETROACHIEVEMENTS_PROGRESS_SYNC: log.info("Starting scheduled RetroAchievements progress sync") sync_retroachievements_progress_task.init() diff --git a/backend/tasks/scheduled/build_recommendations.py b/backend/tasks/scheduled/build_recommendations.py new file mode 100644 index 0000000000..6683a18dac --- /dev/null +++ b/backend/tasks/scheduled/build_recommendations.py @@ -0,0 +1,64 @@ +"""Rebuilds the item-item similarity graph that backs recommendations.""" + +from config import ( + ENABLE_SCHEDULED_BUILD_RECOMMENDATIONS, + SCHEDULED_BUILD_RECOMMENDATIONS_CRON, +) +from handler.recommendation import ( + BuildStats, + SimilarityBuilder, + invalidate_all_cached_feeds, +) +from logger.logger import log +from tasks.tasks import PeriodicTask, TaskType +from utils.context import initialize_context + +from . import UpdateStats + + +class BuildRecommendationsTask(PeriodicTask): + def __init__(self): + super().__init__( + title="Build recommendations index", + description=( + "Rebuilds the similar-games index from library metadata, " + "play history and collections" + ), + task_type=TaskType.UPDATE, + enabled=ENABLE_SCHEDULED_BUILD_RECOMMENDATIONS, + manual_run=True, + cron_string=SCHEDULED_BUILD_RECOMMENDATIONS_CRON, + func="tasks.scheduled.build_recommendations.build_recommendations_task.run", + ) + + @initialize_context() + async def run(self, force: bool = False) -> dict[str, int]: + if not self.enabled and not force: + log.info(f"Scheduled {self.description} not enabled, unscheduling...") + self.unschedule() + return UpdateStats().to_dict() + + log.info("Building recommendations index...") + + update_stats = UpdateStats() + + def report(stats: BuildStats) -> None: + update_stats.update(processed=stats.roms_indexed, total=stats.total) + + try: + build_stats = SimilarityBuilder(progress=report).build() + except Exception: + log.error("Failed to build recommendations index", exc_info=True) + raise + + # Every cached ranking was computed against the previous graph. + invalidate_all_cached_feeds() + + log.info( + f"Recommendations index rebuilt: {build_stats.edges_written} edges " + f"across {build_stats.roms_indexed} ROMs" + ) + return update_stats.to_dict() + + +build_recommendations_task = BuildRecommendationsTask() diff --git a/backend/tests/endpoints/test_recommendations.py b/backend/tests/endpoints/test_recommendations.py new file mode 100644 index 0000000000..947c2d9423 --- /dev/null +++ b/backend/tests/endpoints/test_recommendations.py @@ -0,0 +1,275 @@ +from datetime import datetime, timezone + +from fastapi import status +from fastapi.testclient import TestClient +from tests.handler.recommendation.test_builder import make_rom + +from handler.database import db_rom_handler +from handler.recommendation import SimilarityBuilder, invalidate_cached_feed +from models.platform import Platform +from models.rom import Rom +from models.user import User + + +def build_library(platform: Platform) -> dict[str, Rom]: + library = { + "metroid": make_rom( + platform, + "Super Metroid", + igdb_id=5001, + genres=["Platform", "Adventure"], + franchises=["Metroid"], + companies=["Nintendo"], + ), + "metroid_2": make_rom( + platform, + "Metroid Fusion", + igdb_id=5002, + genres=["Platform", "Adventure"], + franchises=["Metroid"], + companies=["Nintendo"], + ), + "castlevania": make_rom( + platform, + "Castlevania SOTN", + igdb_id=5003, + genres=["Platform", "Adventure"], + franchises=["Castlevania"], + companies=["Konami"], + ), + } + SimilarityBuilder().build() + return library + + +def test_similar_roms_returns_library_games( + client: TestClient, access_token: str, platform: Platform +): + library = build_library(platform) + + response = client.get( + f"/api/roms/{library['metroid'].id}/similar", + headers={"Authorization": f"Bearer {access_token}"}, + ) + assert response.status_code == status.HTTP_200_OK + + body = response.json() + assert body + returned_ids = [entry["rom"]["id"] for entry in body] + assert library["metroid_2"].id in returned_ids + # The source game is never among its own recommendations. + assert library["metroid"].id not in returned_ids + + +def test_similar_roms_are_ordered_by_score( + client: TestClient, access_token: str, platform: Platform +): + library = build_library(platform) + + body = client.get( + f"/api/roms/{library['metroid'].id}/similar", + headers={"Authorization": f"Bearer {access_token}"}, + ).json() + + scores = [entry["score"] for entry in body] + assert scores == sorted(scores, reverse=True) + + +def test_similar_roms_include_reasons( + client: TestClient, access_token: str, platform: Platform +): + library = build_library(platform) + + body = client.get( + f"/api/roms/{library['metroid'].id}/similar", + headers={"Authorization": f"Bearer {access_token}"}, + ).json() + + match = next( + entry for entry in body if entry["rom"]["id"] == library["metroid_2"].id + ) + assert {"facet": "franchise", "value": "Metroid"} in match["reasons"] + + +def test_similar_roms_respects_the_limit( + client: TestClient, access_token: str, platform: Platform +): + library = build_library(platform) + + body = client.get( + f"/api/roms/{library['metroid'].id}/similar?limit=1", + headers={"Authorization": f"Bearer {access_token}"}, + ).json() + + assert len(body) == 1 + + +def test_similar_roms_rejects_an_out_of_range_limit( + client: TestClient, access_token: str, rom: Rom +): + response = client.get( + f"/api/roms/{rom.id}/similar?limit=999", + headers={"Authorization": f"Bearer {access_token}"}, + ) + assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT + + +def test_similar_roms_404s_for_an_unknown_rom(client: TestClient, access_token: str): + response = client.get( + "/api/roms/99999999/similar", + headers={"Authorization": f"Bearer {access_token}"}, + ) + assert response.status_code == status.HTTP_404_NOT_FOUND + + +def test_similar_roms_requires_auth(client: TestClient, rom: Rom): + assert client.get(f"/api/roms/{rom.id}/similar").status_code in ( + status.HTTP_401_UNAUTHORIZED, + status.HTTP_403_FORBIDDEN, + ) + + +def test_similar_roms_is_empty_before_the_index_is_built( + client: TestClient, access_token: str, rom: Rom +): + response = client.get( + f"/api/roms/{rom.id}/similar", + headers={"Authorization": f"Bearer {access_token}"}, + ) + assert response.status_code == status.HTTP_200_OK + assert response.json() == [] + + +def test_recommendations_are_seeded_by_play_history( + client: TestClient, access_token: str, admin_user: User, platform: Platform +): + library = build_library(platform) + + rom_user = db_rom_handler.get_rom_user(library["metroid"].id, admin_user.id) + if rom_user is None: + rom_user = db_rom_handler.add_rom_user(library["metroid"].id, admin_user.id) + db_rom_handler.update_rom_user( + rom_user.id, + {"rating": 10, "last_played": datetime.now(timezone.utc)}, + ) + invalidate_cached_feed(admin_user.id) + + response = client.get( + "/api/recommendations", + headers={"Authorization": f"Bearer {access_token}"}, + ) + assert response.status_code == status.HTTP_200_OK + + body = response.json() + returned_ids = [entry["rom"]["id"] for entry in body] + + # Loving Super Metroid should surface Metroid Fusion, and never re-suggest + # the seed itself. + assert library["metroid_2"].id in returned_ids + assert library["metroid"].id not in returned_ids + + +def test_recommendations_attribute_the_seed_game( + client: TestClient, access_token: str, admin_user: User, platform: Platform +): + library = build_library(platform) + + rom_user = db_rom_handler.get_rom_user(library["metroid"].id, admin_user.id) + if rom_user is None: + rom_user = db_rom_handler.add_rom_user(library["metroid"].id, admin_user.id) + db_rom_handler.update_rom_user( + rom_user.id, + {"rating": 10, "last_played": datetime.now(timezone.utc)}, + ) + invalidate_cached_feed(admin_user.id) + + body = client.get( + "/api/recommendations", + headers={"Authorization": f"Bearer {access_token}"}, + ).json() + + match = next( + entry for entry in body if entry["rom"]["id"] == library["metroid_2"].id + ) + assert match["seed_rom_id"] == library["metroid"].id + assert match["seed_rom_name"] == "Super Metroid" + + +def test_rating_a_game_reshapes_the_feed_without_an_explicit_refresh( + client: TestClient, access_token: str, admin_user: User, platform: Platform +): + """Updating rom_user must drop the cached feed, not wait out its TTL.""" + library = build_library(platform) + headers = {"Authorization": f"Bearer {access_token}"} + invalidate_cached_feed(admin_user.id) + + # Prime the cache while there is no history at all. + before = client.get("/api/recommendations", headers=headers).json() + assert library["metroid_2"].id not in [entry["rom"]["id"] for entry in before] + + response = client.put( + f"/api/roms/{library['metroid'].id}/props?update_last_played=true", + headers=headers, + json={"rating": 10}, + ) + assert response.status_code == status.HTTP_200_OK + + after = client.get("/api/recommendations", headers=headers).json() + assert library["metroid_2"].id in [entry["rom"]["id"] for entry in after] + + +def test_recommendations_fall_back_when_there_is_no_history( + client: TestClient, access_token: str, admin_user: User, platform: Platform +): + make_rom( + platform, + "Acclaimed Game", + igdb_id=6001, + genres=["RPG"], + average_rating=95.0, + ) + SimilarityBuilder().build() + invalidate_cached_feed(admin_user.id) + + response = client.get( + "/api/recommendations", + headers={"Authorization": f"Bearer {access_token}"}, + ) + assert response.status_code == status.HTTP_200_OK + + body = response.json() + assert body + assert all( + entry["reasons"] == [{"facet": "top_rated", "value": ""}] for entry in body + ) + + +def test_recommendations_respect_the_limit( + client: TestClient, access_token: str, admin_user: User, platform: Platform +): + build_library(platform) + invalidate_cached_feed(admin_user.id) + + body = client.get( + "/api/recommendations?limit=1", + headers={"Authorization": f"Bearer {access_token}"}, + ).json() + + assert len(body) <= 1 + + +def test_recommendations_reject_an_out_of_range_limit( + client: TestClient, access_token: str +): + response = client.get( + "/api/recommendations?limit=500", + headers={"Authorization": f"Bearer {access_token}"}, + ) + assert response.status_code == status.HTTP_422_UNPROCESSABLE_CONTENT + + +def test_recommendations_require_auth(client: TestClient): + assert client.get("/api/recommendations").status_code in ( + status.HTTP_401_UNAUTHORIZED, + status.HTTP_403_FORBIDDEN, + ) diff --git a/backend/tests/handler/metadata/test_igdb_handler.py b/backend/tests/handler/metadata/test_igdb_handler.py index 7a41799ce2..5642469fed 100644 --- a/backend/tests/handler/metadata/test_igdb_handler.py +++ b/backend/tests/handler/metadata/test_igdb_handler.py @@ -16,6 +16,7 @@ IGDBHandler, _build_platforms_where, _platform_igdb_ids_with_twin, + extract_metadata_from_igdb_rom, get_igdb_preferred_locale, ) from handler.redis_handler import async_cache @@ -879,3 +880,83 @@ async def mock_list_games( assert result is not None assert result["id"] == 42 search_mock.assert_not_awaited() + + +def _extract_metadata(**overrides) -> dict: + """Run the extractor over a minimal game with the given fields overridden.""" + game = _make_game(1, "Test Game") + game.update(overrides) + return extract_metadata_from_igdb_rom(IGDBHandler(), game, GENESIS_IGDB_ID) + + +class TestFranchiseDeduplication: + """IGDB sends the main franchise both on its own and inside `franchises`. + + Measured on a 14,952-game library: 1,080 of 8,788 games carrying a + franchise carried it twice (12.3%), reaching the details page as + "Happy Feet, Happy Feet". + """ + + def test_the_main_franchise_is_not_repeated_inside_the_list(self): + metadata = _extract_metadata( + franchise={"name": "Happy Feet"}, + franchises=[{"name": "Happy Feet"}, {"name": "Mumble"}], + ) + + assert metadata["franchises"] == ["Happy Feet", "Mumble"] + + def test_the_main_franchise_stays_first(self): + """`gamelist` exports `franchises[0]` as , so order matters.""" + metadata = _extract_metadata( + franchise={"name": "Metroid"}, + franchises=[{"name": "Metroid"}, {"name": "Super Metroid"}], + ) + + assert metadata["franchises"][0] == "Metroid" + + def test_distinct_franchises_are_both_kept(self): + metadata = _extract_metadata( + franchise={"name": "Madden"}, + franchises=[{"name": "NFL"}], + ) + + assert metadata["franchises"] == ["Madden", "NFL"] + + +class TestCompanyRoleDeduplication: + """`involved_companies` carries one entry per involvement, not per company. + + A studio credited as both developer and publisher therefore appears twice + in its role list. Measured on a 14,952-game library: 235 developer lists + and 131 publisher lists repeated a name. + """ + + def test_a_studio_credited_twice_in_one_role_is_listed_once(self): + involved = [ + {"company": {"name": "Cavia"}, "developer": True, "publisher": False}, + {"company": {"name": "Cavia"}, "developer": True, "publisher": False}, + ] + + assert _extract_metadata(involved_companies=involved)["developers"] == ["Cavia"] + + def test_a_studio_that_both_made_and_shipped_a_game_holds_both_roles(self): + """The two lists legitimately overlap; neither may repeat internally.""" + involved = [ + {"company": {"name": "Nintendo"}, "developer": True, "publisher": True}, + ] + + metadata = _extract_metadata(involved_companies=involved) + + assert metadata["developers"] == ["Nintendo"] + assert metadata["publishers"] == ["Nintendo"] + + def test_distinct_developers_keep_their_order(self): + involved = [ + {"company": {"name": "Crystal Dynamics"}, "developer": True}, + {"company": {"name": "Nixxes Software"}, "developer": True}, + ] + + assert _extract_metadata(involved_companies=involved)["developers"] == [ + "Crystal Dynamics", + "Nixxes Software", + ] diff --git a/backend/tests/handler/recommendation/__init__.py b/backend/tests/handler/recommendation/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/tests/handler/recommendation/test_builder.py b/backend/tests/handler/recommendation/test_builder.py new file mode 100644 index 0000000000..9e9acf198f --- /dev/null +++ b/backend/tests/handler/recommendation/test_builder.py @@ -0,0 +1,408 @@ +"""End-to-end checks for the similarity index build. + +Metadata is seeded by writing `roms.igdb_metadata`, because `roms_metadata` is +a view over generated columns and `roms_facets` is trigger-maintained from the +same source: writing the blob is what drives both. +""" + +import pytest + +from handler.database import ( + db_platform_handler, + db_recommendation_handler, + db_rom_handler, +) +from handler.recommendation import SimilarityBuilder +from models.platform import Platform +from models.rom import Rom + + +def make_rom( + platform: Platform, + name: str, + *, + igdb_id: int | None = None, + genres: list[str] | None = None, + franchises: list[str] | None = None, + collections: list[str] | None = None, + companies: list[str] | None = None, + similar_igdb_ids: list[int] | None = None, + port_igdb_ids: list[int] | None = None, + average_rating: float | None = None, + rating_votes: int | None = None, +) -> Rom: + metadata: dict = { + "genres": genres or [], + "franchises": franchises or [], + "collections": collections or [], + "companies": companies or [], + "game_modes": [], + } + if average_rating is not None: + # IGDB's total_rating is carried as a string; the generated column casts it. + metadata["total_rating"] = str(average_rating) + if rating_votes is not None: + metadata["total_rating_count"] = rating_votes + if similar_igdb_ids: + metadata["similar_games"] = [ + {"id": similar_id, "name": f"game-{similar_id}", "type": "similar"} + for similar_id in similar_igdb_ids + ] + if port_igdb_ids: + metadata["ports"] = [ + {"id": port_id, "name": f"game-{port_id}", "type": "port"} + for port_id in port_igdb_ids + ] + + # Set on insert rather than updated afterwards: the generated columns (and + # the roms_facets triggers) derive from this blob, so one write is enough. + return db_rom_handler.add_rom( + Rom( + platform_id=platform.id, + name=name, + slug=name.lower().replace(" ", "-"), + fs_name=f"{name}.zip", + fs_name_no_tags=name, + fs_name_no_ext=name, + fs_extension="zip", + fs_path=f"{platform.slug}/roms", + igdb_id=igdb_id, + igdb_metadata=metadata, + ) + ) + + +@pytest.fixture +def library(platform: Platform) -> dict[str, Rom]: + """A small library with one tight cluster and one unrelated outlier. + + Deliberately padded with unrelated games. IDF is relative to the library, + so on a four-game shelf where three are platformers "Platform" carries + almost no weight and a genre-only match drops under MIN_EDGE_SCORE -- a + degenerate case that says nothing about ranking on a real library. + """ + for index, (genre, franchise, company) in enumerate( + [ + ("Racing", "Outrun", "Sega"), + ("Shooter", "Gradius", "Konami"), + ("Sport", "Tennis", "Namco"), + ("Fighting", "Street Fighter", "Capcom"), + ("Simulation", "Sim", "Maxis"), + ] + ): + make_rom( + platform, + f"Filler {index}", + igdb_id=9000 + index, + genres=[genre], + franchises=[franchise], + companies=[company], + ) + + return { + "metroid": make_rom( + platform, + "Super Metroid", + igdb_id=1001, + genres=["Platform", "Adventure"], + franchises=["Metroid"], + companies=["Nintendo"], + ), + "metroid_2": make_rom( + platform, + "Metroid Fusion", + igdb_id=1002, + genres=["Platform", "Adventure"], + franchises=["Metroid"], + companies=["Nintendo"], + ), + "castlevania": make_rom( + platform, + "Castlevania SOTN", + igdb_id=1003, + genres=["Platform", "Adventure"], + franchises=["Castlevania"], + companies=["Konami"], + ), + "puzzle": make_rom( + platform, + "Tetris", + igdb_id=1004, + genres=["Puzzle"], + companies=["Nintendo"], + ), + } + + +def test_build_creates_edges_for_related_games(library: dict[str, Rom]): + # Counts are >= rather than ==: the builder indexes the whole library, so + # ROMs left behind by other tests in this worker's database count too. + stats = SimilarityBuilder().build() + + assert stats.roms_indexed >= len(library) + assert stats.edges_written > 0 + + edges = db_recommendation_handler.get_similar_rom_edges(library["metroid"].id) + assert [edge.rom_id for edge in edges] + + +def test_same_franchise_outranks_same_genre(library: dict[str, Rom]): + SimilarityBuilder().build() + + edges = db_recommendation_handler.get_similar_rom_edges(library["metroid"].id) + ranked = [edge.rom_id for edge in edges] + + # Metroid Fusion shares the franchise; Castlevania only shares genres. + assert ranked[0] == library["metroid_2"].id + + # A genre-only match may fall below MIN_EDGE_SCORE and be dropped entirely, + # which is a stronger version of the same result. Assert the ordering only + # when it survived, so the test measures ranking rather than the threshold. + if library["castlevania"].id in ranked: + assert ranked.index(library["metroid_2"].id) < ranked.index( + library["castlevania"].id + ) + + +def test_unrelated_game_scores_below_a_franchise_match(library: dict[str, Rom]): + SimilarityBuilder().build() + + by_rom = { + edge.rom_id: edge.score + for edge in db_recommendation_handler.get_similar_rom_edges( + library["metroid"].id + ) + } + + assert by_rom[library["metroid_2"].id] > by_rom.get(library["puzzle"].id, 0.0) + + +def test_edges_carry_a_human_readable_reason(library: dict[str, Rom]): + SimilarityBuilder().build() + + edges = db_recommendation_handler.get_similar_rom_edges(library["metroid"].id) + match = next(edge for edge in edges if edge.rom_id == library["metroid_2"].id) + + assert {"facet": "franchise", "value": "Metroid"} in match.reasons + + +def test_a_rom_is_never_similar_to_itself(library: dict[str, Rom]): + SimilarityBuilder().build() + + for rom in library.values(): + neighbours = db_recommendation_handler.get_similar_rom_edges(rom.id) + assert rom.id not in {edge.rom_id for edge in neighbours} + + +def test_region_duplicates_are_not_recommendations(platform: Platform): + """Two files of the same game must never recommend each other.""" + usa = make_rom( + platform, + "Chrono Trigger (USA)", + igdb_id=2001, + genres=["RPG"], + franchises=["Chrono"], + ) + europe = make_rom( + platform, + "Chrono Trigger (Europe)", + igdb_id=2001, + genres=["RPG"], + franchises=["Chrono"], + ) + + SimilarityBuilder().build() + + neighbours = db_recommendation_handler.get_similar_rom_edges(usa.id) + assert europe.id not in {edge.rom_id for edge in neighbours} + + +def test_ports_of_one_game_take_a_single_slot(platform: Platform): + """Regression: the title check only compared candidates to the source. + + IGDB gives each port its own id, so two ports of one game clear the + igdb_id check and, sharing no title with the source, both took a slot. + A section of six then spent two of them naming the same game. + """ + other_platform = db_platform_handler.add_platform( + Platform(name="other", slug="other_slug", fs_slug="other_slug") + ) + source = make_rom( + platform, + "100 Classic Games", + igdb_id=4001, + genres=["Card & Board Game"], + ) + port_a = make_rom(platform, "Monopoly", igdb_id=4002, genres=["Card & Board Game"]) + port_b = make_rom( + other_platform, "Monopoly", igdb_id=4003, genres=["Card & Board Game"] + ) + + SimilarityBuilder().build() + + recommended = { + edge.rom_id + for edge in db_recommendation_handler.get_similar_rom_edges(source.id) + } + assert len({port_a.id, port_b.id} & recommended) == 1 + + +def test_igdb_similar_games_link_owned_roms(platform: Platform): + """IGDB's prior should create an edge even without shared metadata.""" + source = make_rom( + platform, + "Source Game", + igdb_id=3001, + genres=["Shooter"], + similar_igdb_ids=[3002], + ) + target = make_rom(platform, "Target Game", igdb_id=3002, genres=["Racing"]) + make_rom(platform, "Unrelated Game", igdb_id=3003, genres=["Racing"]) + + SimilarityBuilder().build() + + neighbours = db_recommendation_handler.get_similar_rom_edges(source.id) + assert target.id in {edge.rom_id for edge in neighbours} + + +def test_a_port_relation_is_not_a_recommendation(platform: Platform): + """A port is the same product on other hardware, not a suggestion. + + IGDB's other related buckets feed the prior; `ports` must not, or a game + scores its own port more highly for being a port of itself. Measured on a + 14,952-game library, that prior was live on 243 pairs whose titles differ + enough that the duplicate check cannot collapse them, e.g. Robocod: James + Pond II and James Pond: Codename - Robocod. + """ + source = make_rom( + platform, + "Source Game", + igdb_id=5001, + genres=["Shooter"], + port_igdb_ids=[5002], + ) + port = make_rom(platform, "Handheld Rework", igdb_id=5002, genres=["Puzzle"]) + + SimilarityBuilder().build() + + neighbours = db_recommendation_handler.get_similar_rom_edges(source.id) + assert port.id not in {edge.rom_id for edge in neighbours} + + +def test_rebuild_replaces_edges_rather_than_duplicating(library: dict[str, Rom]): + first = SimilarityBuilder().build() + second = SimilarityBuilder().build() + + assert first.edges_written == second.edges_written + assert db_recommendation_handler.count_similarity_edges() == second.edges_written + + +def test_unidentified_roms_are_not_related_to_each_other(platform: Platform): + """Sharing only a platform is not similarity. + + Two files that never matched a provider carry nothing but platform (and + maybe decade), which normalise to identical vectors -- they would score a + perfect match against each other if they were indexed at all. + """ + first = make_rom(platform, "Unknown Game A") + second = make_rom(platform, "Unknown Game B") + + stats = SimilarityBuilder().build() + + assert stats.roms_without_metadata >= 2 + assert db_recommendation_handler.get_similar_rom_edges(first.id) == [] + assert db_recommendation_handler.get_similar_rom_edges(second.id) == [] + + +def test_an_identified_rom_is_not_related_to_an_unidentified_one(platform: Platform): + identified = make_rom( + platform, "Known Game", igdb_id=4001, genres=["RPG"], franchises=["Saga"] + ) + unidentified = make_rom(platform, "Mystery File") + + SimilarityBuilder().build() + + neighbours = db_recommendation_handler.get_similar_rom_edges(identified.id) + assert unidentified.id not in {edge.rom_id for edge in neighbours} + + +def test_deleting_a_rom_clears_its_edges_in_both_directions(library: dict[str, Rom]): + """Edges are cleaned up by the foreign keys, not by an ORM cascade. + + ROMs are removed with a bulk delete, which never triggers an ORM-level + cascade, so inbound edges would be left dangling without ON DELETE CASCADE + on both columns. + """ + SimilarityBuilder().build() + deleted_id = library["metroid"].id + + assert db_recommendation_handler.get_similar_rom_edges(deleted_id) + + db_rom_handler.delete_rom(deleted_id) + + assert db_recommendation_handler.get_similar_rom_edges(deleted_id) == [] + # The surviving ROM must not still point at the deleted one. + survivors = db_recommendation_handler.get_similar_rom_edges(library["metroid_2"].id) + assert deleted_id not in {edge.rom_id for edge in survivors} + + +def test_build_on_an_empty_library_is_a_no_op(): + stats = SimilarityBuilder().build() + + assert stats.roms_indexed == 0 + assert stats.edges_written == 0 + + +def test_cold_start_prefers_a_well_voted_rating_over_a_lone_perfect_one( + platform: Platform, +): + """Regression: the cold-start feed used to be topped by obscure games. + + A single provider scoring something 100 is not evidence it is a great + game. On a real library exactly fourteen games hit a perfect 100, every + one a lone ScreenScraper score, and the feed recommended all of them + ahead of the classics. + """ + # Shrinkage is toward the library mean, so the library needs a realistic + # one. With only the two games below, the mean sits between them and the + # estimator has nothing to pull the outlier down to. + for index in range(12): + make_rom( + platform, + f"Ordinary Game {index}", + igdb_id=7100 + index, + genres=["Action"], + average_rating=60.0 + index, + rating_votes=40, + ) + + lone_perfect = make_rom( + platform, + "Obscure Sports Title", + igdb_id=7001, + genres=["Sport"], + average_rating=100.0, + rating_votes=1, + ) + broadly_loved = make_rom( + platform, + "Beloved Classic", + igdb_id=7002, + genres=["Adventure"], + average_rating=94.0, + rating_votes=1800, + ) + + ranked = db_recommendation_handler.get_fallback_rom_ids(limit=10) + + assert broadly_loved.id in ranked + assert ranked.index(broadly_loved.id) < ranked.index(lone_perfect.id) + + +def test_cold_start_still_returns_games_with_no_vote_count(platform: Platform): + """Most providers report no count at all; those games must not vanish.""" + unvoted = make_rom( + platform, "Unvoted Game", igdb_id=7003, genres=["RPG"], average_rating=88.0 + ) + + assert unvoted.id in db_recommendation_handler.get_fallback_rom_ids(limit=25) diff --git a/backend/tests/handler/recommendation/test_diversity.py b/backend/tests/handler/recommendation/test_diversity.py new file mode 100644 index 0000000000..ae776480cb --- /dev/null +++ b/backend/tests/handler/recommendation/test_diversity.py @@ -0,0 +1,165 @@ +"""Unit tests for the diversity caps applied when serving recommendations. + +`cap_by_series` only reads `rom.metadatum` and `rom.platform_id`, so these use +lightweight stand-ins rather than database rows. +""" + +from dataclasses import dataclass, field + +from handler.recommendation.diversity import ( + cap_by_series, + series_keys, +) + + +@dataclass +class FakeMetadata: + franchises: list[str] = field(default_factory=list) + collections: list[str] = field(default_factory=list) + + +@dataclass +class FakeRom: + id: int + metadatum: FakeMetadata | None = None + platform_id: int = 1 + + +def rom( + rom_id: int, + franchise: str = "", + collection: str = "", + franchises: list[str] | None = None, + platform_id: int = 1, +) -> FakeRom: + return FakeRom( + id=rom_id, + platform_id=platform_id, + metadatum=FakeMetadata( + franchises=( + franchises + if franchises is not None + else ([franchise] if franchise else []) + ), + collections=[collection] if collection else [], + ), + ) + + +def resolve(roms: dict[int, FakeRom]): + return lambda rom_id: roms.get(rom_id) + + +def test_caps_each_series_at_the_limit(): + """Two per series, with enough other candidates to fill the list. + + `limit` is set to exactly what the cap yields; asking for more would pull + the capped-out entries back in via the backfill below. + """ + roms = {i: rom(i, franchise="Metroid") for i in range(1, 4)} + roms.update({i: rom(i, franchise="Castlevania") for i in range(4, 7)}) + + selected = cap_by_series(list(roms), resolve(roms), limit=4, max_per_series=2) + + assert selected == [1, 2, 4, 5] + + +def test_games_without_a_series_are_never_capped(): + """Otherwise an unmatched shelf collapses to two results.""" + roms = {i: rom(i) for i in range(1, 6)} + + selected = cap_by_series(list(roms), resolve(roms), limit=5, max_per_series=2) + + assert len(selected) == 5 + + +def test_capped_out_entries_backfill_a_short_list(): + """A shelf deep in one franchise must not return a near-empty section.""" + roms = {i: rom(i, franchise="Sonic") for i in range(1, 7)} + + selected = cap_by_series(list(roms), resolve(roms), limit=4, max_per_series=2) + + assert len(selected) == 4 + + +def test_backfilled_entries_keep_their_original_order(): + """Backfill appends by rank, so the list must not end up out of order.""" + roms = {1: rom(1, franchise="Sonic"), 2: rom(2, franchise="Sonic")} + roms[3] = rom(3, franchise="Sonic") + roms[4] = rom(4, franchise="Mario") + + selected = cap_by_series([1, 2, 3, 4], resolve(roms), limit=4, max_per_series=2) + + # 3 is capped out then backfilled; it must land before 4, not after. + assert selected == [1, 2, 3, 4] + + +def test_stops_at_the_limit(): + roms = {i: rom(i, franchise=f"F{i}") for i in range(1, 10)} + + assert len(cap_by_series(list(roms), resolve(roms), limit=3)) == 3 + + +def test_unresolvable_items_are_dropped(): + """Permission filtering removes ROMs, leaving edges that resolve to nothing.""" + roms = {1: rom(1, franchise="Metroid")} + + assert cap_by_series([1, 2, 3], resolve(roms), limit=5) == [1] + + +def test_series_keys_returns_every_franchise_and_collection(): + entry = rom(1, collection="Madden NFL", franchises=["Madden", "NFL"]) + + assert series_keys(entry) == {"Madden", "NFL", "Madden NFL"} + + +def test_a_series_listed_under_several_names_shares_one_allowance(): + """Regression: four Madden games cleared a cap of two. + + IGDB lists franchises in no stable order, so some Madden titles resolved + to "Madden" and others to "NFL". Keying on one entry gave each spelling + its own allowance, and the section filled with a single series. + """ + roms = { + 1: rom(1, franchises=["Madden", "NFL"]), + 2: rom(2, franchises=["Madden", "NFL"]), + 3: rom(3, franchises=["NFL", "Madden"]), + 4: rom(4, franchises=["NFL"]), + 5: rom(5, franchise="Tecmo Bowl"), + } + + selected = cap_by_series([1, 2, 3, 4, 5], resolve(roms), limit=3, max_per_series=2) + + # Two from the shared series, then the unrelated game -- not a third Madden. + assert selected == [1, 2, 5] + + +def test_overlapping_series_still_backfills_when_nothing_else_exists(): + roms = {i: rom(i, franchises=["Madden", "NFL"]) for i in range(1, 5)} + + assert len(cap_by_series(list(roms), resolve(roms), limit=4, max_per_series=2)) == 4 + + +def test_a_platform_can_be_capped_too(): + """The feed shares this path and does not want one console owning the row.""" + roms = {i: rom(i, platform_id=1) for i in range(1, 4)} + roms.update({i: rom(i, platform_id=2) for i in range(4, 7)}) + + selected = cap_by_series(list(roms), resolve(roms), limit=4, max_per_platform=2) + + assert selected == [1, 2, 4, 5] + + +def test_the_platform_cap_is_off_unless_asked_for(): + """A single game's "Similar games" is happy to be all one platform.""" + roms = {i: rom(i, platform_id=1) for i in range(1, 5)} + + assert len(cap_by_series(list(roms), resolve(roms), limit=4)) == 4 + + +def test_a_platform_capped_entry_backfills_a_short_list(): + roms = {i: rom(i, platform_id=1) for i in range(1, 6)} + + selected = cap_by_series(list(roms), resolve(roms), limit=4, max_per_platform=2) + + assert len(selected) == 4 diff --git a/backend/tests/handler/recommendation/test_feed.py b/backend/tests/handler/recommendation/test_feed.py new file mode 100644 index 0000000000..f7a68beec5 --- /dev/null +++ b/backend/tests/handler/recommendation/test_feed.py @@ -0,0 +1,169 @@ +from datetime import datetime, timedelta, timezone + +import pytest + +from handler.database import db_rom_handler +from handler.database.recommendations_handler import UserAffinityRow +from handler.recommendation.feed import ( + MIN_RECENCY_FACTOR, + RECENCY_HALFLIFE_DAYS, + _cache_key, + seed_affinity, +) +from handler.redis_handler import sync_cache +from models.rom import RomUserStatus + +NOW = datetime(2026, 8, 7, tzinfo=timezone.utc) + + +def affinity_row(**overrides) -> UserAffinityRow: + defaults = { + "rom_id": 1, + "rating": None, + "difficulty": None, + "completion": None, + "status": None, + "last_played": NOW, + "now_playing": False, + "backlogged": False, + "hidden": False, + "playtime_ms": 0, + } + return UserAffinityRow(**{**defaults, **overrides}) + + +def test_high_rating_produces_a_positive_seed(): + assert seed_affinity(affinity_row(rating=10), now=NOW) > 0 + + +def test_low_rating_produces_a_negative_seed(): + # The point of a negative seed: steer the feed away from this kind of game. + assert seed_affinity(affinity_row(rating=1), now=NOW) < 0 + + +def test_midpoint_rating_is_roughly_neutral(): + assert seed_affinity(affinity_row(rating=5), now=NOW) == pytest.approx(0, abs=0.15) + + +def test_negative_seeds_are_damped_relative_to_positive_ones(): + liked = seed_affinity(affinity_row(rating=10), now=NOW) + disliked = seed_affinity(affinity_row(rating=1), now=NOW) + + assert abs(disliked) < abs(liked) + + +def test_playtime_increases_affinity_with_diminishing_returns(): + hour = 3_600_000 + short = seed_affinity(affinity_row(playtime_ms=hour), now=NOW) + medium = seed_affinity(affinity_row(playtime_ms=10 * hour), now=NOW) + marathon = seed_affinity(affinity_row(playtime_ms=200 * hour), now=NOW) + + assert short < medium <= marathon + # Saturation: 20h to 200h must matter far less than 1h to 10h. + assert (marathon - medium) < (medium - short) + + +def test_completion_status_outweighs_merely_starting_a_game(): + completed = seed_affinity( + affinity_row(status=RomUserStatus.COMPLETED_100.value), now=NOW + ) + incomplete = seed_affinity( + affinity_row(status=RomUserStatus.INCOMPLETE.value), now=NOW + ) + + assert completed > incomplete > 0 + + +def test_now_playing_is_a_strong_signal(): + assert seed_affinity(affinity_row(now_playing=True), now=NOW) > seed_affinity( + affinity_row(status=RomUserStatus.INCOMPLETE.value), now=NOW + ) + + +def test_recent_play_outweighs_an_old_one_of_equal_rating(): + recent = seed_affinity(affinity_row(rating=10, last_played=NOW), now=NOW) + old = seed_affinity( + affinity_row(rating=10, last_played=NOW - timedelta(days=365)), now=NOW + ) + + assert recent > old > 0 + + +def test_recency_decay_halves_at_the_configured_halflife(): + fresh = seed_affinity(affinity_row(rating=10, last_played=NOW), now=NOW) + halflife = seed_affinity( + affinity_row( + rating=10, last_played=NOW - timedelta(days=RECENCY_HALFLIFE_DAYS) + ), + now=NOW, + ) + + assert halflife == pytest.approx(fresh * 0.5, rel=0.01) + + +def test_old_favourites_never_decay_to_nothing(): + ancient = seed_affinity( + affinity_row(rating=10, last_played=NOW - timedelta(days=3650)), now=NOW + ) + + assert ancient == pytest.approx(MIN_RECENCY_FACTOR, rel=0.01) + + +def test_naive_timestamps_are_treated_as_utc(): + # MariaDB hands back naive datetimes; a crash here would break the feed. + naive = seed_affinity( + affinity_row(rating=10, last_played=NOW.replace(tzinfo=None)), now=NOW + ) + aware = seed_affinity(affinity_row(rating=10, last_played=NOW), now=NOW) + + assert naive == pytest.approx(aware) + + +def test_a_row_with_no_signals_at_all_contributes_nothing(): + assert seed_affinity(affinity_row(last_played=None), now=NOW) == 0.0 + + +def test_a_played_but_unrated_game_still_seeds_weakly(): + weak = seed_affinity(affinity_row(last_played=NOW), now=NOW) + rated = seed_affinity(affinity_row(rating=9, last_played=NOW), now=NOW) + + assert 0 < weak < rated + + +class TestFeedCacheInvalidation: + """The cached ranking has to drop when the signals under it move. + + Every writer of `rom_user` goes through `update_rom_user`: play-session + ingestion, save and state uploads, and the RetroAchievements sync all + move `last_played` or `status` without touching the ROM endpoints, so + invalidating at the endpoint left the feed stale for up to the cache TTL. + """ + + @staticmethod + def _rom_user(rom, user): + return db_rom_handler.get_rom_user( + rom_id=rom.id, user_id=user.id + ) or db_rom_handler.add_rom_user(rom_id=rom.id, user_id=user.id) + + @staticmethod + def _seed_cache(user_id: int) -> None: + sync_cache.set(_cache_key(user_id, 10), "[]") + + def test_a_play_updates_drops_the_cached_feed(self, rom, admin_user): + rom_user = self._rom_user(rom, admin_user) + self._seed_cache(admin_user.id) + + db_rom_handler.update_rom_user( + rom_user.id, {"last_played": datetime.now(timezone.utc)} + ) + + assert sync_cache.get(_cache_key(admin_user.id, 10)) is None + + def test_an_unrelated_field_leaves_the_cache_alone(self, rom, admin_user): + """Rebuilding a feed is not free, so only the scored fields drop it.""" + rom_user = self._rom_user(rom, admin_user) + self._seed_cache(admin_user.id) + + db_rom_handler.update_rom_user(rom_user.id, {"difficulty": 3}) + + assert sync_cache.get(_cache_key(admin_user.id, 10)) is not None diff --git a/backend/tests/handler/recommendation/test_scoring.py b/backend/tests/handler/recommendation/test_scoring.py new file mode 100644 index 0000000000..602b24d617 --- /dev/null +++ b/backend/tests/handler/recommendation/test_scoring.py @@ -0,0 +1,504 @@ +import pytest + +from handler.recommendation.scoring import ( + FACET_WEIGHTS, + MAX_QUALITY_BONUS, + RomFeatures, + blend, + build_inverted_index, + build_normalised_vectors, + build_vector, + candidate_ids, + compute_idf, + content_similarity, + extract_tokens, + has_taste_signal, + make_token, + normalise, + normalise_co_occurrence, + pivot_length, + quality_bonus, + release_year_from_epoch, + shared_reasons, + token_facet, + vector_norm, +) + +# 1991-08-13, the sort of epoch value roms_metadata carries. +SUPER_NES_ERA_EPOCH = 682_041_600 + + +def test_extract_tokens_namespaces_every_facet(): + tokens = extract_tokens( + platform_id=7, + genres=["Platform", "Adventure"], + franchises=["Metroid"], + collections=["Super Metroid"], + companies=["Nintendo"], + game_modes=["Single player"], + first_release_date=SUPER_NES_ERA_EPOCH, + ) + + assert make_token("genre", "Platform") in tokens + assert make_token("franchise", "Metroid") in tokens + assert make_token("collection", "Super Metroid") in tokens + assert make_token("company", "Nintendo") in tokens + assert make_token("game_mode", "Single player") in tokens + assert make_token("platform", "7") in tokens + assert make_token("decade", "1990") in tokens + + +def test_extract_tokens_namespaces_the_igdb_tag_facets(): + tokens = extract_tokens( + platform_id=1, + genres=["Platform"], + keywords=["metroidvania", "interconnected-world"], + themes=["Action", "Horror"], + player_perspectives=["Side view"], + ) + + assert make_token("keyword", "metroidvania") in tokens + assert make_token("theme", "Horror") in tokens + assert make_token("perspective", "Side view") in tokens + + +def test_igdb_tags_count_as_a_taste_signal(): + """A game with only keywords is still worth indexing.""" + assert has_taste_signal(extract_tokens(platform_id=1, keywords=["roguelike"])) + assert has_taste_signal(extract_tokens(platform_id=1, themes=["Horror"])) + assert has_taste_signal( + extract_tokens(platform_id=1, player_perspectives=["First person"]) + ) + + +def test_curated_facets_explain_a_match_before_keywords(): + """Keywords carry the highest IDF, so they would otherwise own every slot. + + Real data explained a Castlevania match with "frankenstein's monster" and a + Zelda match with "drawbridge", both of which read as nonsense next to the + shared franchise or genre that actually drove the score. + """ + idf = { + "collection:Castlevania": 2.0, + "genre:Platform": 1.0, + # Rare keywords dominate on IDF alone. + "keyword:frankenstein's monster": 9.0, + } + tokens = ( + "collection:Castlevania", + "genre:Platform", + "keyword:frankenstein's monster", + ) + vectors = build_normalised_vectors({1: tokens, 2: tokens}, idf) + + reasons = shared_reasons(vectors[1], vectors[2]) + + assert reasons[0]["facet"] == "collection" + assert reasons[-1]["facet"] == "keyword" + + +def test_role_split_companies_replace_the_merged_list(): + """Both would double-count a studio for IGDB-matched games only.""" + tokens = extract_tokens( + platform_id=1, + genres=["Platform"], + companies=["Nintendo R&D1", "Playtronic"], + developers=["Nintendo R&D1"], + publishers=["Playtronic"], + ) + + assert make_token("developer", "Nintendo R&D1") in tokens + assert make_token("publisher", "Playtronic") in tokens + assert not any(token_facet(token) == "company" for token in tokens) + + +def test_the_merged_company_list_is_the_fallback_without_roles(): + """Providers other than IGDB report no roles, so those games keep the old + behaviour rather than losing the signal entirely.""" + tokens = extract_tokens( + platform_id=1, genres=["Platform"], companies=["Some Studio"] + ) + + assert make_token("company", "Some Studio") in tokens + assert not any(token_facet(token) == "developer" for token in tokens) + + +def test_a_shared_developer_outweighs_a_shared_publisher(): + """Regression: matches were being explained by regional distributors. + + Tec Toy and Playtronic distributed hundreds of titles apiece, which put + creatively unrelated games together under a shared "company". + """ + idf = { + "developer:Treasure": 2.0, + "publisher:Sega": 2.0, + "genre:Action": 1.0, + } + vectors = build_normalised_vectors( + { + 1: ("genre:Action", "developer:Treasure", "publisher:Sega"), + 2: ("genre:Action", "developer:Treasure"), # same studio + 3: ("genre:Action", "publisher:Sega"), # same label only + }, + idf, + ) + + assert content_similarity(vectors[1], vectors[2]) > content_similarity( + vectors[1], vectors[3] + ) + + +def test_company_roles_count_as_a_taste_signal(): + assert has_taste_signal(extract_tokens(platform_id=1, developers=["Treasure"])) + assert has_taste_signal(extract_tokens(platform_id=1, publishers=["Sega"])) + + +def test_extract_tokens_skips_blanks_and_deduplicates(): + tokens = extract_tokens( + platform_id=1, + genres=["Action", " ", "", "Action"], + franchises=None, + ) + + assert tokens.count(make_token("genre", "Action")) == 1 + assert not any(token.endswith(":") for token in tokens) + + +def test_release_year_handles_seconds_and_millisecond_epochs(): + assert release_year_from_epoch(SUPER_NES_ERA_EPOCH) == 1991 + # Some provider rows arrive in milliseconds; both must land in the same decade. + assert release_year_from_epoch(SUPER_NES_ERA_EPOCH * 1000) == 1991 + assert release_year_from_epoch(None) is None + assert release_year_from_epoch(0) is None + + +def test_idf_penalises_ubiquitous_tokens(): + documents = [ + ("genre:Action", "franchise:Metroid"), + ("genre:Action",), + ("genre:Action",), + ("genre:Action",), + ] + + idf = compute_idf(documents, total_documents=4) + + # "Action" is on every game here and says nothing; the franchise is rare. + assert idf["genre:Action"] < idf["franchise:Metroid"] + + +def test_a_token_on_every_game_is_worth_almost_nothing(): + """The property the original smoothed IDF failed. + + "Single player" sits on nearly every game in a real library. If it keeps + meaningful weight it drags unrelated titles up the rankings on nothing but + a shared game mode. + """ + documents = [("game_mode:Single player", f"franchise:F{i}") for i in range(50)] + + idf = compute_idf(documents, total_documents=50) + + assert idf["game_mode:Single player"] < 0.05 + # ...while a genuinely rare token stays strong. + assert idf["franchise:F0"] > 3.0 + + +def test_ubiquitous_facets_cannot_outrank_a_shared_genre(): + """Regression: Tetris used to outrank Super Mario World for Super Metroid. + + Both shared "Nintendo" and "Single player" with the source game, but only + the platformer shared its genre -- which must dominate. + """ + library = [ + ("genre:Platform", "company:Nintendo", "game_mode:Single player"), + ("genre:Platform", "company:Nintendo", "game_mode:Single player"), + ("genre:Puzzle", "company:Nintendo", "game_mode:Single player"), + ("genre:RPG", "company:Square", "game_mode:Single player"), + ("genre:Racing", "company:Sega", "game_mode:Single player"), + ("genre:RPG", "company:Square", "game_mode:Single player"), + ] + idf = compute_idf(library, total_documents=len(library)) + + source = build_vector(library[0], idf) + same_genre = build_vector(library[1], idf) + same_company_only = build_vector(library[2], idf) + + assert content_similarity(source, same_genre) > content_similarity( + source, same_company_only + ) + + +def test_build_vector_is_raw_facet_weight_times_idf(): + idf = {"genre:Action": 1.2, "franchise:Metroid": 2.4} + vector = build_vector(("genre:Action", "franchise:Metroid"), idf) + + assert vector["genre:Action"] == pytest.approx(FACET_WEIGHTS["genre"] * 1.2) + assert vector["franchise:Metroid"] == pytest.approx( + FACET_WEIGHTS["franchise"] * 2.4 + ) + + +def test_pivoting_stops_a_sparse_game_outranking_a_rich_one(): + """The defect real data exposed: "Golf" above "Super Mario Sunshine". + + Both candidates share the source's franchise, and the richer one *also* + shares its genre -- so it is plainly the better match. Under plain L2 the + richer game is divided by its own longer vector and loses anyway, which is + exactly how a one-tag entry outranked the real Mario platformers. + + One value per facet throughout, so build_vector's per-facet split is + neutral here and the test isolates length normalisation. + """ + idf = { + token: 1.0 + for token in ( + "genre:a", + "franchise:b", + "company:c", + "theme:d", + "keyword:x", + "game_mode:z", + "perspective:q", + ) + } + token_sets = { + 1: ("genre:a", "franchise:b", "company:c", "theme:d"), # source + 2: ("franchise:b",), # sparse: shares the franchise only + 3: ( # rich: shares franchise *and* genre, but carries more besides + "genre:a", + "franchise:b", + "keyword:x", + "game_mode:z", + "perspective:q", + ), + } + + l2 = {} + for key, tokens in token_sets.items(): + raw = build_vector(tokens, idf) + l2[key] = normalise(raw, vector_norm(raw)) + + # Plain L2 ranks the sparse game first, which is the bug. + assert content_similarity(l2[1], l2[2]) > content_similarity(l2[1], l2[3]) + + pivoted = build_normalised_vectors(token_sets, idf) + assert content_similarity(pivoted[1], pivoted[3]) > content_similarity( + pivoted[1], pivoted[2] + ) + + +def test_pivot_length_blends_towards_the_library_average(): + average = 10.0 + + # Partial normalisation pulls both extremes towards the average. + assert 2.0 < pivot_length(2.0, average, b=0.75) < average + assert average < pivot_length(20.0, average, b=0.75) < 20.0 + assert pivot_length(average, average, b=0.75) == pytest.approx(average) + + # The shipped default ignores a vector's own length entirely. + assert pivot_length(2.0, average, b=0.0) == pytest.approx(average) + assert pivot_length(20.0, average, b=0.0) == pytest.approx(average) + + +def test_well_documented_games_are_not_penalised_by_default(): + """The shipped default must not let a one-tag game beat a five-tag one. + + Real data: at full L2 normalisation a Mario compilation's nearest matches + were all 6-8 token entries, with the 12-16 token Mario platformers nowhere + in the list. + """ + facets = ("genre", "franchise", "company", "theme", "perspective", "keyword") + idf = {f"{facet}:v": 1.0 for facet in facets} + vectors = build_normalised_vectors( + { + 1: ("genre:v", "franchise:v", "company:v", "theme:v"), + 2: ("genre:v",), + 3: ("genre:v", "franchise:v", "perspective:v", "keyword:v"), + }, + idf, + ) + + assert content_similarity(vectors[1], vectors[3]) > content_similarity( + vectors[1], vectors[2] + ) + + +def test_build_vector_drops_zero_weight_tokens(): + vector = build_vector(("genre:Action", "genre:Unknown"), {"genre:Action": 1.0}) + + assert "genre:Unknown" not in vector + + +def test_build_vector_of_unknown_tokens_is_empty(): + assert build_vector(("genre:Action",), {}) == {} + + +def test_content_similarity_ranks_shared_franchise_above_shared_genre(): + idf = compute_idf( + [ + ("genre:Action", "franchise:Metroid"), + ("genre:Action", "franchise:Metroid"), + ("genre:Action", "franchise:Mario"), + ("genre:Action", "franchise:Zelda"), + ], + total_documents=4, + ) + + source = build_vector(("genre:Action", "franchise:Metroid"), idf) + same_franchise = build_vector(("genre:Action", "franchise:Metroid"), idf) + same_genre_only = build_vector(("genre:Action", "franchise:Zelda"), idf) + + assert content_similarity(source, same_franchise) > content_similarity( + source, same_genre_only + ) + + +def test_content_similarity_is_symmetric(): + idf = {"genre:Action": 1.0, "franchise:Metroid": 2.0} + vectors = build_normalised_vectors( + {1: ("genre:Action", "franchise:Metroid"), 2: ("genre:Action",)}, idf + ) + + assert content_similarity(vectors[1], vectors[2]) == pytest.approx( + content_similarity(vectors[2], vectors[1]) + ) + assert content_similarity(vectors[1], vectors[2]) > 0 + + +def test_content_similarity_of_disjoint_vectors_is_zero(): + idf = {"genre:Action": 1.0, "genre:Puzzle": 1.0} + assert ( + content_similarity( + build_vector(("genre:Action",), idf), build_vector(("genre:Puzzle",), idf) + ) + == 0.0 + ) + + +def test_shared_reasons_reports_strongest_facet_first(): + idf = compute_idf( + [ + ("genre:Action", "franchise:Metroid", "company:Nintendo"), + ("genre:Action", "franchise:Mario", "company:Nintendo"), + ("genre:Action", "franchise:Zelda", "company:Sega"), + ("genre:Action", "franchise:Sonic", "company:Sega"), + ], + total_documents=4, + ) + tokens = ("genre:Action", "franchise:Metroid", "company:Nintendo") + vector = build_vector(tokens, idf) + + reasons = shared_reasons(vector, vector) + + assert reasons[0] == {"facet": "franchise", "value": "Metroid"} + # One reason per facet, so the list reads as distinct explanations. + assert len({reason["facet"] for reason in reasons}) == len(reasons) + + +def test_shared_reasons_never_explains_a_match_by_platform(): + idf = {"platform:7": 5.0, "genre:Action": 1.0} + vector = build_vector(("platform:7", "genre:Action"), idf) + + assert all( + reason["facet"] != "platform" for reason in shared_reasons(vector, vector) + ) + + +def test_quality_bonus_is_bounded_and_monotonic(): + assert quality_bonus(None) == 0.0 + assert quality_bonus(0) == 0.0 + assert quality_bonus(100) == pytest.approx(MAX_QUALITY_BONUS) + assert quality_bonus(50) < quality_bonus(90) + # Providers occasionally emit out-of-range scores. + assert quality_bonus(140) == pytest.approx(MAX_QUALITY_BONUS) + + +def test_blend_rewards_every_signal_independently(): + baseline = blend(content=0.5) + + assert blend(content=0.5, igdb_prior=1.0) > baseline + assert blend(content=0.5, co_play=1.0) > baseline + assert blend(content=0.5, co_collection=1.0) > baseline + assert blend(content=0.5, average_rating=95) > baseline + + +def test_blend_weights_igdb_above_collection_co_membership(): + assert blend(content=0.0, igdb_prior=1.0) > blend(content=0.0, co_collection=1.0) + + +def test_blend_clamps_out_of_range_signals(): + # A malformed signal must not let a score run away past the maximum. + assert blend(content=1.0, igdb_prior=50.0, average_rating=100) <= 1.0 + 1e-9 + + +def test_candidate_ids_skips_tokens_that_cover_the_library(): + features = { + rom_id: RomFeatures( + rom_id=rom_id, + platform_id=1, + tokens=("genre:Action",) + (("franchise:Metroid",) if rom_id < 3 else ()), + ) + for rom_id in range(1, 301) + } + postings = build_inverted_index(features) + + candidates = candidate_ids(features[1], postings, total_documents=len(features)) + + # "genre:Action" is on all 300 ROMs, well past the df cap, so only the ROM + # sharing the rare franchise is considered. + assert candidates == {2} + + +def test_candidate_ids_expands_every_token_on_a_small_library(): + # The df ratio must not starve a small shelf of candidates entirely. + features = { + rom_id: RomFeatures(rom_id=rom_id, platform_id=1, tokens=("genre:Action",)) + for rom_id in range(1, 5) + } + postings = build_inverted_index(features) + + assert candidate_ids(features[1], postings, total_documents=4) == {2, 3, 4} + + +def test_candidate_ids_excludes_the_source_rom(): + features = { + 1: RomFeatures(rom_id=1, platform_id=1, tokens=("franchise:Metroid",)), + 2: RomFeatures(rom_id=2, platform_id=1, tokens=("franchise:Metroid",)), + } + postings = build_inverted_index(features) + + assert 1 not in candidate_ids(features[1], postings, total_documents=2) + + +def test_has_taste_signal_rejects_context_only_tokens(): + # Platform and decade describe the shelf, not the game. + assert not has_taste_signal(("platform:7", "decade:1990")) + assert not has_taste_signal(()) + assert has_taste_signal(("platform:7", "genre:RPG")) + assert has_taste_signal(("franchise:Metroid",)) + + +def test_context_only_vectors_would_otherwise_match_each_other(): + """Documents the reason context-only ROMs are excluded from the index. + + Two games carrying nothing but platform and decade produce identical + vectors, so they match each other as strongly as anything possibly can. + """ + idf = {"platform:7": 2.0, "decade:1990": 1.5} + tokens = ("platform:7", "decade:1990") + + vectors = build_normalised_vectors({1: tokens, 2: tokens}, idf) + identical = content_similarity(vectors[1], vectors[2]) + self_match = content_similarity(vectors[1], vectors[1]) + + assert identical == pytest.approx(self_match) + + +def test_normalise_co_occurrence_damps_ubiquitous_items(): + # Two games seen together twice, where one appears in everything. + focused = normalise_co_occurrence(2, left_total=2, right_total=2) + ubiquitous = normalise_co_occurrence(2, left_total=2, right_total=500) + + assert focused > ubiquitous + assert focused <= 1.0 + assert normalise_co_occurrence(0, 5, 5) == 0.0 + assert normalise_co_occurrence(3, 0, 5) == 0.0 diff --git a/backend/tools/backfill_igdb_tags.py b/backend/tools/backfill_igdb_tags.py new file mode 100644 index 0000000000..d174a6713f --- /dev/null +++ b/backend/tools/backfill_igdb_tags.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Backfill IGDB keywords, themes and player perspectives into existing ROMs. + +These three fields were never requested by the scanner, so a library matched +before they were added carries none of them. Re-scanning to pick them up would +re-fetch every field for every game one at a time; this asks only for what is +missing and batches it, so a 12k-game library is a few dozen requests rather +than twelve thousand. + +Run from the backend directory, with IGDB credentials in the environment: + + uv run tools/backfill_igdb_tags.py --dry-run # report coverage only + uv run tools/backfill_igdb_tags.py # write to roms.igdb_metadata + uv run tools/backfill_igdb_tags.py --limit 500 # sample, for evaluating + +Reads the database configured by the usual env vars. Point it at a copy of +your library, not production: it rewrites `roms.igdb_metadata`. +""" + +from __future__ import annotations + +import argparse +import asyncio +import os +import sys +from typing import Any + +# Allow running as `python3 tools/backfill_igdb_tags.py` from backend/. +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from sqlalchemy import select # noqa: E402 + +from handler.database.base_handler import sync_session # noqa: E402 +from handler.metadata import meta_igdb_handler # noqa: E402 +from models.rom import Rom # noqa: E402 +from utils.context import initialize_context # noqa: E402 + +# IGDB caps a single response at 500 rows. +BATCH_SIZE = 500 + +TAG_FIELDS = ( + "id", + "keywords.name", + "themes.name", + "player_perspectives.name", + "total_rating_count", + "involved_companies.company.name", + "involved_companies.developer", + "involved_companies.publisher", +) +TAG_KEYS = ("keywords", "themes", "player_perspectives") +# Scalar rather than a list of named entities, so it is merged separately. +SCALAR_KEYS = ("total_rating_count",) +# Derived from involved_companies rather than returned directly: +# {facet key: the IGDB involvement flag that fills it}. +ROLE_KEYS = {"developers": "developer", "publishers": "publisher"} + + +def load_igdb_ids(limit: int | None) -> dict[int, list[int]]: + """IGDB id -> every ROM carrying it. + + A list rather than a single id: region and revision variants of one game + share an IGDB id, so keying one-to-one silently skips all but one of them. + On a 15k library 3,943 ROMs share an id with another, and keying this + way leaves most of them unbackfilled. + """ + stmt = select(Rom.igdb_id, Rom.id).where(Rom.igdb_id.is_not(None)) + if limit: + stmt = stmt.limit(limit) + + grouped: dict[int, list[int]] = {} + with sync_session.begin() as session: + for igdb_id, rom_id in session.execute(stmt).all(): + grouped.setdefault(igdb_id, []).append(rom_id) + + return grouped + + +def report_coverage() -> None: + with sync_session.begin() as session: + rows = session.execute( + select(Rom.igdb_metadata).where(Rom.igdb_id.is_not(None)) + ).all() + + total = len(rows) + have = {key: 0 for key in TAG_KEYS + SCALAR_KEYS + tuple(ROLE_KEYS)} + for (metadata,) in rows: + for key in TAG_KEYS + SCALAR_KEYS + tuple(ROLE_KEYS): + if metadata and metadata.get(key): + have[key] += 1 + + print(f"IGDB-matched roms: {total:,}") + for key in TAG_KEYS + SCALAR_KEYS + tuple(ROLE_KEYS): + share = (have[key] / total * 100) if total else 0 + print(f" with {key:<20} {have[key]:>7,} ({share:.1f}%)") + + +async def fetch_tags(igdb_ids: list[int]) -> dict[int, dict[str, Any]]: + """Ask IGDB for just the three tag fields, in batches of BATCH_SIZE.""" + results: dict[int, dict[str, list[str]]] = {} + + for start in range(0, len(igdb_ids), BATCH_SIZE): + chunk = igdb_ids[start : start + BATCH_SIZE] + where = f"id = ({','.join(str(i) for i in chunk)})" + + games = await meta_igdb_handler.igdb_service.list_games( + fields=TAG_FIELDS, where=where, limit=BATCH_SIZE + ) + + for game in games: + game_id = game.get("id") + if game_id is None: + continue + values: dict[str, Any] = { + key: [ + entry.get("name", "") + for entry in (game.get(key) or []) + if isinstance(entry, dict) and entry.get("name") + ] + for key in TAG_KEYS + } + values.update({key: game.get(key, 0) for key in SCALAR_KEYS}) + involved = game.get("involved_companies") or [] + for key, role in ROLE_KEYS.items(): + values[key] = [ + entry["company"]["name"] + for entry in involved + if isinstance(entry, dict) + and entry.get(role) + and entry.get("company") + ] + results[game_id] = values + + done = min(start + BATCH_SIZE, len(igdb_ids)) + print(f" fetched {done:,}/{len(igdb_ids):,}") + + return results + + +def merge_tags( + roms_by_igdb_id: dict[int, list[int]], tags: dict[int, dict[str, Any]] +) -> int: + """Merge the fetched tags into each ROM's existing igdb_metadata blob.""" + written = 0 + + with sync_session.begin() as session: + for igdb_id, values in tags.items(): + for rom_id in roms_by_igdb_id.get(igdb_id, []): + rom = session.get(Rom, rom_id) + if rom is None: + continue + + # Replace rather than merge: IGDB is the authority for these + # keys, and an empty list means "this game has no keywords". + metadata = dict(rom.igdb_metadata or {}) + metadata.update(values) + rom.igdb_metadata = metadata + written += 1 + + return written + + +@initialize_context() +async def main_async(args: argparse.Namespace) -> int: + print("Coverage before:") + report_coverage() + + # Coverage is a pure database read, so --dry-run works without credentials. + if args.dry_run: + return 0 + + if not meta_igdb_handler.is_enabled(): + print("\nIGDB is not enabled: set IGDB_CLIENT_ID and IGDB_CLIENT_SECRET.") + return 1 + + roms_by_igdb_id = load_igdb_ids(args.limit) + total_roms = sum(len(v) for v in roms_by_igdb_id.values()) + print( + f"\nFetching tags for {len(roms_by_igdb_id):,} IGDB ids " + f"covering {total_roms:,} roms..." + ) + + tags = await fetch_tags(sorted(roms_by_igdb_id)) + written = merge_tags(roms_by_igdb_id, tags) + print(f"\nUpdated {written:,} roms.") + + print("\nCoverage after:") + report_coverage() + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--dry-run", action="store_true", help="Report current coverage and stop" + ) + parser.add_argument( + "--limit", type=int, default=None, help="Only backfill this many roms" + ) + args = parser.parse_args() + + return asyncio.run(main_async(args)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/backend/tools/inspect_recommendations.py b/backend/tools/inspect_recommendations.py new file mode 100644 index 0000000000..aff5c1b963 --- /dev/null +++ b/backend/tools/inspect_recommendations.py @@ -0,0 +1,556 @@ +#!/usr/bin/env python3 +"""Print what the recommendations engine actually produces, for eyeballing. + +Recommendation quality is a judgement call that tests cannot make, so this +dumps the index in a readable form: for each sampled game, its own metadata +facets followed by its top neighbours, their scores and the reasons behind +them. Reading twenty of these tells you whether the weights are sane far +faster than clicking through the UI. + +Run from the backend directory: + + uv run tools/inspect_recommendations.py --build # build, then sample + uv run tools/inspect_recommendations.py --sample 20 + uv run tools/inspect_recommendations.py --name "Super Metroid" + uv run tools/inspect_recommendations.py --platform snes + uv run tools/inspect_recommendations.py --feed myusername + uv run tools/inspect_recommendations.py --stats + uv run tools/inspect_recommendations.py --sample 40 --seed 7 --report out.html + +Reads the database configured by the usual env vars (DB_HOST, DB_NAME, ...). +Point it at a copy of your library, not production: --build rewrites the +`rom_similarity` table. +""" + +from __future__ import annotations + +import argparse +import html +import os +import random +import shlex +import sys +from collections import Counter +from typing import Any + +# Allow running as `python3 tools/inspect_recommendations.py` from backend/. +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from sqlalchemy import distinct, func, select # noqa: E402 + +from handler.database import ( # noqa: E402 + db_recommendation_handler, + db_rom_handler, + db_user_handler, +) +from handler.database.base_handler import sync_session # noqa: E402 +from handler.recommendation import ( # noqa: E402 + FeedBuilder, + SimilarityBuilder, + cap_by_series, +) +from models.platform import Platform # noqa: E402 +from models.recommendation import RomSimilarity # noqa: E402 +from models.rom import Rom, RomMetadata # noqa: E402 + +NAME_WIDTH = 42 +FACET_ORDER = ("collection", "franchise", "company", "genre", "game_mode", "decade") + + +class RomInfo: + """The display fields for one ROM, resolved in a single query.""" + + def __init__(self, rom_id: int, name: str, platform: str) -> None: + self.rom_id = rom_id + self.name = name + self.platform = platform + + def label(self) -> str: + return f"{truncate(self.name, NAME_WIDTH):<{NAME_WIDTH}} ({self.platform})" + + +def truncate(value: str, width: int) -> str: + value = value or "?" + return value if len(value) <= width else value[: width - 1] + "…" + + +def load_rom_info(rom_ids: list[int]) -> dict[int, RomInfo]: + if not rom_ids: + return {} + + stmt = ( + select(Rom.id, Rom.name, Rom.fs_name_no_tags, Platform.slug) + .join(Platform, Platform.id == Rom.platform_id) + .where(Rom.id.in_(rom_ids)) + ) + with sync_session.begin() as session: + return { + row[0]: RomInfo(row[0], row[1] or row[2], row[3]) + for row in session.execute(stmt).all() + } + + +def load_facets(rom_id: int) -> dict[str, list[str]]: + stmt = select( + RomMetadata.genres, + RomMetadata.franchises, + RomMetadata.collections, + RomMetadata.companies, + RomMetadata.game_modes, + RomMetadata.keywords, + RomMetadata.themes, + RomMetadata.player_perspectives, + ).where(RomMetadata.rom_id == rom_id) + + with sync_session.begin() as session: + row = session.execute(stmt).first() + + if row is None: + return {} + + return { + "genres": row[0] or [], + "franchises": row[1] or [], + "collections": row[2] or [], + "companies": row[3] or [], + "game_modes": row[4] or [], + "keywords": row[5] or [], + "themes": row[6] or [], + "perspectives": row[7] or [], + } + + +def format_reasons(reasons: list[dict[str, Any]]) -> str: + if not reasons: + return "-" + return " · ".join( + ( + f"{reason.get('facet')}:{reason.get('value')}" + if reason.get("value") + else str(reason.get("facet")) + ) + for reason in reasons + ) + + +def indexed_rom_ids(platform_slug: str | None) -> list[int]: + """Every ROM that has at least one outgoing edge.""" + stmt = select(distinct(RomSimilarity.rom_id)) + if platform_slug: + stmt = ( + stmt.join(Rom, Rom.id == RomSimilarity.rom_id) + .join(Platform, Platform.id == Rom.platform_id) + .where(Platform.slug == platform_slug) + ) + + with sync_session.begin() as session: + return [row[0] for row in session.execute(stmt).all()] + + +def find_rom_ids_by_name(needle: str, limit: int) -> list[int]: + stmt = ( + select(Rom.id) + .where(Rom.name.ilike(f"%{needle}%")) + .order_by(Rom.name_sort_key.asc()) + .limit(limit) + ) + with sync_session.begin() as session: + return [row[0] for row in session.execute(stmt).all()] + + +def neighbours_of(rom_id: int, limit: int, capped: bool) -> list[Any]: + """The edges a surface would render, over-fetched and capped like the endpoint.""" + edges = db_recommendation_handler.get_similar_rom_edges(rom_id, limit=limit * 4) + if not edges: + return [] + + if not capped: + return edges[:limit] + + hydrated = { + rom.id: rom + for rom in db_rom_handler.get_roms_simple_by_ids( + [edge.rom_id for edge in edges] + ) + } + return cap_by_series(edges, lambda edge: hydrated.get(edge.rom_id), limit=limit) + + +def print_rom( + rom_id: int, info: dict[int, RomInfo], limit: int, capped: bool = True +) -> None: + source = info.get(rom_id) + print() + print("=" * 78) + print(source.label() if source else f"rom {rom_id}") + + facets = load_facets(rom_id) + summary = " · ".join( + f"{key}={', '.join(values)}" for key, values in facets.items() if values + ) + print(f" {summary or 'no metadata facets'}") + print("-" * 78) + + edges = neighbours_of(rom_id, limit, capped) + if not edges: + print(" (no neighbours -- unmatched metadata, or the index is not built)") + return + + neighbour_info = load_rom_info([edge.rom_id for edge in edges]) + for edge in edges: + neighbour = neighbour_info.get(edge.rom_id) + label = neighbour.label() if neighbour else f"rom {edge.rom_id}" + print(f" {edge.score:>6.3f} {label} {format_reasons(edge.reasons)}") + + +def print_feed(username: str, limit: int) -> int: + user = db_user_handler.get_user_by_username(username) + if user is None: + print(f"No user named {username!r}") + return 1 + + print() + print("=" * 78) + print(f"Personalised feed for {username}") + print("-" * 78) + + feed = FeedBuilder(user.id).build(limit=limit) + if not feed: + print(" (empty -- no play history and no rated games in the library)") + return 0 + + info = load_rom_info([item.rom.id for item in feed]) + for item in feed: + entry = info.get(item.rom.id) + label = entry.label() if entry else f"rom {item.rom.id}" + why = ( + f"because you played {item.seed_rom_name}" + if item.seed_rom_name + else format_reasons(item.reasons) + ) + print(f" {item.score:>6.3f} {label} {why}") + + return 0 + + +def gather_stats() -> dict[str, Any]: + with sync_session.begin() as session: + total_roms = session.scalar(select(func.count()).select_from(Rom)) or 0 + edges = session.scalar(select(func.count()).select_from(RomSimilarity)) or 0 + covered = ( + session.scalar(select(func.count(distinct(RomSimilarity.rom_id)))) or 0 + ) + score_bounds = session.execute( + select( + func.min(RomSimilarity.score), + func.avg(RomSimilarity.score), + func.max(RomSimilarity.score), + ) + ).first() + reason_rows = session.execute(select(RomSimilarity.reasons).limit(20_000)).all() + + facet_counts: Counter[str] = Counter() + for (reasons,) in reason_rows: + for reason in reasons or (): + facet_counts[str(reason.get("facet"))] += 1 + + return { + "total_roms": total_roms, + "edges": edges, + "covered": covered, + "coverage": (covered / total_roms * 100) if total_roms else 0.0, + "avg_edges": (edges / covered) if covered else 0.0, + "score_bounds": score_bounds, + "facet_counts": facet_counts, + } + + +def print_stats() -> None: + stats = gather_stats() + total_roms = stats["total_roms"] + edges = stats["edges"] + covered = stats["covered"] + score_bounds = stats["score_bounds"] + facet_counts = stats["facet_counts"] + + print() + print("=" * 78) + print("Index health") + print("-" * 78) + print(f" roms in library {total_roms:>10,}") + print( + f" roms with neighbours {covered:>10,}" + f" ({stats['coverage']:.1f}% coverage)" + ) + print(f" edges {edges:>10,}") + if covered: + print(f" avg edges per rom {stats['avg_edges']:>10.1f}") + if score_bounds and score_bounds[0] is not None: + print( + f" score min/avg/max " + f"{score_bounds[0]:.3f} / {float(score_bounds[1]):.3f} / {score_bounds[2]:.3f}" + ) + + if facet_counts: + print() + print(" Why games were matched (sampled):") + total_reasons = sum(facet_counts.values()) + for facet, count in facet_counts.most_common(): + share = count / total_reasons * 100 + bar = "█" * max(1, round(share / 2)) + print(f" {facet:<12} {share:>5.1f}% {bar}") + + # A library whose matches are nearly all one weak facet is the signal that + # the metadata is too thin for the content signal to say anything. + if facet_counts: + top_facet, top_count = facet_counts.most_common(1)[0] + if top_count / sum(facet_counts.values()) > 0.8: + print() + print( + f" NOTE: {top_facet} accounts for over 80% of matches. Expect weak " + "recommendations until more metadata is scraped." + ) + + +REPORT_CSS = """ +:root { color-scheme: light dark; --bg: #fff; --fg: #1a1a1a; --dim: #666; + --line: #e3e3e3; --card: #fafafa; --accent: #2f6f4f; } +@media (prefers-color-scheme: dark) { + :root { --bg: #131316; --fg: #e8e8ea; --dim: #9a9aa2; --line: #2a2a30; + --card: #1b1b20; --accent: #6fcf97; } +} +* { box-sizing: border-box; } +body { margin: 0 auto; padding: 2rem 1.25rem 4rem; max-width: 60rem; background: var(--bg); + color: var(--fg); font: 15px/1.55 ui-sans-serif, system-ui, sans-serif; } +h1 { font-size: 1.5rem; margin: 0 0 .25rem; } +h2 { font-size: 1.05rem; margin: 0 0 .1rem; } +.sub { color: var(--dim); font-size: .85rem; margin: 0 0 2rem; } +.health { display: grid; grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr)); + gap: .75rem; margin: 0 0 1rem; } +.health div { background: var(--card); border: 1px solid var(--line); + border-radius: .5rem; padding: .6rem .75rem; } +.health b { display: block; font-size: 1.2rem; font-variant-numeric: tabular-nums; } +.health span { color: var(--dim); font-size: .75rem; text-transform: uppercase; + letter-spacing: .04em; } +.game { border: 1px solid var(--line); border-radius: .5rem; margin: 0 0 1rem; + overflow: hidden; } +.game > header { background: var(--card); padding: .7rem .9rem; + border-bottom: 1px solid var(--line); } +.plat { color: var(--dim); font-weight: 400; font-size: .85rem; } +.facets { color: var(--dim); font-size: .8rem; margin: .2rem 0 0; } +table { width: 100%; border-collapse: collapse; } +td { padding: .4rem .9rem; border-top: 1px solid var(--line); vertical-align: top; } +tr:first-child td { border-top: 0; } +.score { font-variant-numeric: tabular-nums; color: var(--accent); width: 4.5rem; + font-weight: 600; } +.why { color: var(--dim); font-size: .8rem; } +.none { padding: .7rem .9rem; color: var(--dim); font-size: .85rem; } +.bars { margin: 0 0 2rem; } +.bars tr td { border: 0; padding: .15rem .5rem .15rem 0; } +.bar { background: var(--accent); height: .55rem; border-radius: .3rem; display: block; } +footer { color: var(--dim); font-size: .8rem; margin-top: 2.5rem; + border-top: 1px solid var(--line); padding-top: 1rem; } +""" + + +def esc(value: Any) -> str: + return html.escape(str(value if value is not None else ""), quote=True) + + +def write_report( + path: str, + rom_ids: list[int], + info: dict[int, RomInfo], + limit: int, + capped: bool, + command: str, +) -> None: + """Write the sampled games to a self-contained HTML page. + + The terminal output is for the person running the tool; this is for + sending to someone who has not got the library. + """ + stats = gather_stats() + out: list[str] = [ + "", + '', + "RomM recommendations sample", + f"", + "

RomM recommendations sample

", + f"

{len(rom_ids)} games, " + f"{'capped as the UI renders them' if capped else 'raw index, uncapped'}. " + f"Reproduce with {esc(command)}

", + "
", + f"
Library{stats['total_roms']:,}
", + f"
Coverage{stats['coverage']:.1f}%
", + f"
Edges{stats['edges']:,}
", + f"
Edges per game{stats['avg_edges']:.1f}
", + "
", + ] + + facet_counts = stats["facet_counts"] + if facet_counts: + total = sum(facet_counts.values()) + out.append("") + for facet, count in facet_counts.most_common(): + share = count / total * 100 + out.append( + f"" + f"" + f"" + ) + out.append("
{esc(facet)}{share:.1f}%
") + + for rom_id in rom_ids: + source = info.get(rom_id) + name = esc(source.name if source else f"rom {rom_id}") + platform = esc(source.platform if source else "?") + facets = load_facets(rom_id) + summary = " · ".join( + f"{key}={', '.join(values)}" for key, values in facets.items() if values + ) + out.append( + f"

{name} " + f"({platform})

" + f"

{esc(summary) or 'no metadata facets'}

" + ) + + edges = neighbours_of(rom_id, limit, capped) + if not edges: + out.append("

No neighbours.

") + continue + + neighbour_info = load_rom_info([edge.rom_id for edge in edges]) + out.append("") + for edge in edges: + neighbour = neighbour_info.get(edge.rom_id) + label = esc(neighbour.name if neighbour else f"rom {edge.rom_id}") + plat = esc(neighbour.platform if neighbour else "?") + out.append( + f"" + f"" + f"" + ) + out.append("
{edge.score:.3f}{label} ({plat}){esc(format_reasons(edge.reasons))}
") + + out.append( + "" + ) + out.append("") + + with open(path, "w", encoding="utf-8") as handle: + handle.write("\n".join(out)) + + print(f"\nWrote {path}") + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + parser.add_argument( + "--build", + action="store_true", + help="Rebuild the similarity index before inspecting", + ) + parser.add_argument( + "--sample", type=int, default=10, help="Random games to show (default: 10)" + ) + parser.add_argument( + "--limit", type=int, default=8, help="Neighbours per game (default: 8)" + ) + parser.add_argument("--rom", type=int, action="append", help="Inspect this rom id") + parser.add_argument("--name", help="Inspect games whose name matches this text") + parser.add_argument("--platform", help="Restrict the random sample to this slug") + parser.add_argument("--feed", help="Show the personalised feed for this username") + parser.add_argument("--stats", action="store_true", help="Show index health only") + parser.add_argument("--seed", type=int, default=None, help="RNG seed for sampling") + parser.add_argument( + "--raw", + action="store_true", + help="Show the uncapped index instead of what the UI would render", + ) + parser.add_argument( + "--report", + metavar="PATH", + help="Also write the sample to a self-contained HTML page", + ) + args = parser.parse_args() + + if args.build: + print("Building similarity index...") + stats = SimilarityBuilder().build() + print( + f" indexed {stats.roms_indexed:,} roms, wrote {stats.edges_written:,} edges, " + f"skipped {stats.roms_without_metadata:,} without usable metadata" + ) + + if args.stats: + print_stats() + return 0 + + if args.feed: + return print_feed(args.feed, args.limit) + + rom_ids: list[int] = list(args.rom or []) + if args.name: + found = find_rom_ids_by_name(args.name, args.sample) + if not found: + print(f"No games matching {args.name!r}") + return 1 + rom_ids.extend(found) + + if not rom_ids: + candidates = indexed_rom_ids(args.platform) + if not candidates: + print( + "No indexed games found. Run with --build first " + "(or check --platform is a real slug)." + ) + return 1 + rng = random.Random(args.seed) # nosec B311 - sampling for display only + rom_ids = rng.sample(candidates, min(args.sample, len(candidates))) + + info = load_rom_info(rom_ids) + for rom_id in rom_ids: + print_rom(rom_id, info, args.limit, capped=not args.raw) + + print() + print_stats() + + if args.report: + # The report path is whatever the reader chooses, so it is replaced + # rather than echoed: the point of the line is the sampling flags. + flags: list[str] = [] + skip_next = False + for token in sys.argv[1:]: + if skip_next: + skip_next = False + continue + if token == "--report": + skip_next = True + continue + if token.startswith("--report="): + continue + flags.append(token) + + write_report( + args.report, + rom_ids, + info, + args.limit, + capped=not args.raw, + command=shlex.join( + ["uv", "run", "tools/inspect_recommendations.py", *flags] + + ["--report", "out.html"] + ), + ) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/frontend/src/__generated__/index.ts b/frontend/src/__generated__/index.ts index 12762d7482..4c80ef1538 100644 --- a/frontend/src/__generated__/index.ts +++ b/frontend/src/__generated__/index.ts @@ -134,6 +134,7 @@ export type { PlaySessionSchema } from './models/PlaySessionSchema'; export type { RAGameRomAchievement } from './models/RAGameRomAchievement'; export type { RAProgression } from './models/RAProgression'; export type { RAUserGameProgression } from './models/RAUserGameProgression'; +export type { RecommendedRomSchema } from './models/RecommendedRomSchema'; export type { RegionBreakdownItem } from './models/RegionBreakdownItem'; export type { Role } from './models/Role'; export type { RomArchiveMember } from './models/RomArchiveMember'; @@ -167,6 +168,8 @@ export type { SearchCoverSchema } from './models/SearchCoverSchema'; export type { SearchRomSchema } from './models/SearchRomSchema'; export type { SGDBResource } from './models/SGDBResource'; export type { SiblingRomSchema } from './models/SiblingRomSchema'; +export type { SimilarityReasonSchema } from './models/SimilarityReasonSchema'; +export type { SimilarRomSchema } from './models/SimilarRomSchema'; export type { SimpleRomSchema } from './models/SimpleRomSchema'; export type { SlotSummarySchema } from './models/SlotSummarySchema'; export type { SmartCollectionSchema } from './models/SmartCollectionSchema'; diff --git a/frontend/src/__generated__/models/RecommendedRomSchema.ts b/frontend/src/__generated__/models/RecommendedRomSchema.ts new file mode 100644 index 0000000000..121eed60b3 --- /dev/null +++ b/frontend/src/__generated__/models/RecommendedRomSchema.ts @@ -0,0 +1,14 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { SimilarityReasonSchema } from './SimilarityReasonSchema'; +import type { SimpleRomSchema } from './SimpleRomSchema'; +export type RecommendedRomSchema = { + rom: SimpleRomSchema; + score: number; + reasons: Array; + seed_rom_id?: (number | null); + seed_rom_name?: (string | null); +}; + diff --git a/frontend/src/__generated__/models/SimilarRomSchema.ts b/frontend/src/__generated__/models/SimilarRomSchema.ts new file mode 100644 index 0000000000..35ff77a1aa --- /dev/null +++ b/frontend/src/__generated__/models/SimilarRomSchema.ts @@ -0,0 +1,12 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +import type { SimilarityReasonSchema } from './SimilarityReasonSchema'; +import type { SimpleRomSchema } from './SimpleRomSchema'; +export type SimilarRomSchema = { + rom: SimpleRomSchema; + score: number; + reasons: Array; +}; + diff --git a/frontend/src/__generated__/models/SimilarityReasonSchema.ts b/frontend/src/__generated__/models/SimilarityReasonSchema.ts new file mode 100644 index 0000000000..31320313c2 --- /dev/null +++ b/frontend/src/__generated__/models/SimilarityReasonSchema.ts @@ -0,0 +1,17 @@ +/* generated using openapi-typescript-codegen -- do not edit */ +/* istanbul ignore file */ +/* tslint:disable */ +/* eslint-disable */ +/** + * Why two games were linked, e.g. {"facet": "franchise", "value": "Metroid"}. + * + * `facet` is one of the metadata facets the engine scores on (genre, + * franchise, collection, company, game_mode, decade), or "igdb" when the + * link came from IGDB's own related-games list, or "top_rated" for the + * cold-start feed. The frontend maps it to a translated label. + */ +export type SimilarityReasonSchema = { + facet: string; + value: string; +}; + diff --git a/frontend/src/composables/useUISettings.ts b/frontend/src/composables/useUISettings.ts index ad8db6b05d..f013cd2c35 100644 --- a/frontend/src/composables/useUISettings.ts +++ b/frontend/src/composables/useUISettings.ts @@ -21,6 +21,10 @@ export const UI_SETTINGS_KEYS = { showStats: { key: "settings.showStats", default: true }, showRecentRoms: { key: "settings.showRecentRoms", default: true }, showContinuePlaying: { key: "settings.showContinuePlaying", default: true }, + showRecommendations: { + key: "settings.showRecommendations", + default: true, + }, showPlatforms: { key: "settings.showPlatforms", default: true }, showCollections: { key: "settings.showCollections", default: true }, showSmartCollections: { diff --git a/frontend/src/locales/bg_BG/recommendations.json b/frontend/src/locales/bg_BG/recommendations.json new file mode 100644 index 0000000000..b1d1035b9e --- /dev/null +++ b/frontend/src/locales/bg_BG/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Защото играхте {0}", + "for-you": "Препоръчано за вас", + "reason-igdb": "Подобни в IGDB", + "reason-top-rated": "Високо оценена", + "similar-games": "Подобни игри", + "why": "Защо е препоръчано това" +} diff --git a/frontend/src/locales/bg_BG/settings.json b/frontend/src/locales/bg_BG/settings.json index b7fd0bfd74..39b291008f 100644 --- a/frontend/src/locales/bg_BG/settings.json +++ b/frontend/src/locales/bg_BG/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Показвай секцията с платформи на началната страница", "show-recently-added": "Покажи наскоро добавените ROM-ове", "show-recently-added-desc": "Показвай секцията с наскоро добавени ROM-ове на началната страница", + "show-recommendations": "Показване на препоръки", + "show-recommendations-desc": "Показване на препоръките на началната страница и на страницата с детайли за игра", "show-regions": "Покажи региони", "show-regions-desc": "Показвай флаговете на регионите в галерията", "show-siblings": "Покажи версии", diff --git a/frontend/src/locales/cs_CZ/recommendations.json b/frontend/src/locales/cs_CZ/recommendations.json new file mode 100644 index 0000000000..bd786f9a26 --- /dev/null +++ b/frontend/src/locales/cs_CZ/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Protože jste hráli {0}", + "for-you": "Doporučeno pro vás", + "reason-igdb": "Podobné na IGDB", + "reason-top-rated": "Vysoce hodnocené", + "similar-games": "Podobné hry", + "why": "Proč bylo toto doporučeno" +} diff --git a/frontend/src/locales/cs_CZ/settings.json b/frontend/src/locales/cs_CZ/settings.json index a494853038..ada6378e58 100644 --- a/frontend/src/locales/cs_CZ/settings.json +++ b/frontend/src/locales/cs_CZ/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Zobrazit sekci platforem na hlavní stránce", "show-recently-added": "Zobrazit nedávno přidané ROM", "show-recently-added-desc": "Zobrazit sekci nedávno přidaných ROM na hlavní stránce", + "show-recommendations": "Zobrazit doporučení", + "show-recommendations-desc": "Zobrazit doporučení na domovské stránce a na stránce s detaily hry", "show-regions": "Zobrazit regiony", "show-regions-desc": "Zobrazit vlajky regionů v galerii", "show-siblings": "Zobrazit verze", diff --git a/frontend/src/locales/de_DE/recommendations.json b/frontend/src/locales/de_DE/recommendations.json new file mode 100644 index 0000000000..34439333c6 --- /dev/null +++ b/frontend/src/locales/de_DE/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Weil du {0} gespielt hast", + "for-you": "Für dich empfohlen", + "reason-igdb": "Ähnlich laut IGDB", + "reason-top-rated": "Hoch bewertet", + "similar-games": "Ähnliche Spiele", + "why": "Warum dies empfohlen wurde" +} diff --git a/frontend/src/locales/de_DE/settings.json b/frontend/src/locales/de_DE/settings.json index 21172fa166..b967facf88 100644 --- a/frontend/src/locales/de_DE/settings.json +++ b/frontend/src/locales/de_DE/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Plattformen werden auf der Startseite dargestellt", "show-recently-added": "Zeige kürzlich hinzugefügte ROMs", "show-recently-added-desc": "Kürzlich hinzugefügte ROMs werden auf der Startseite dargestellt", + "show-recommendations": "Zeige Empfehlungen", + "show-recommendations-desc": "Empfehlungen auf der Startseite und auf der Detailseite eines Spiels anzeigen", "show-regions": "Zeige Regionen", "show-regions-desc": "Zeige die Regionen des ROMs als Flaggen in der Galerie", "show-siblings": "Zeige Versionen", diff --git a/frontend/src/locales/en_GB/recommendations.json b/frontend/src/locales/en_GB/recommendations.json new file mode 100644 index 0000000000..724b0910bb --- /dev/null +++ b/frontend/src/locales/en_GB/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Because you played {0}", + "for-you": "Recommended for you", + "reason-igdb": "Similar on IGDB", + "reason-top-rated": "Highly rated", + "similar-games": "Similar games", + "why": "Why this was recommended" +} diff --git a/frontend/src/locales/en_GB/settings.json b/frontend/src/locales/en_GB/settings.json index 6ca0347cd4..9008f4d5ae 100644 --- a/frontend/src/locales/en_GB/settings.json +++ b/frontend/src/locales/en_GB/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Show platforms section at the home page", "show-recently-added": "Show recently added ROMs", "show-recently-added-desc": "Show recently added ROMs section at the home page", + "show-recommendations": "Show recommendations", + "show-recommendations-desc": "Show recommendations on the home page and on a game's details page", "show-regions": "Show regions", "show-regions-desc": "Show region flags in the gallery", "show-siblings": "Show versions", diff --git a/frontend/src/locales/en_US/recommendations.json b/frontend/src/locales/en_US/recommendations.json new file mode 100644 index 0000000000..724b0910bb --- /dev/null +++ b/frontend/src/locales/en_US/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Because you played {0}", + "for-you": "Recommended for you", + "reason-igdb": "Similar on IGDB", + "reason-top-rated": "Highly rated", + "similar-games": "Similar games", + "why": "Why this was recommended" +} diff --git a/frontend/src/locales/en_US/settings.json b/frontend/src/locales/en_US/settings.json index fcf25c4e83..1370bb33e4 100644 --- a/frontend/src/locales/en_US/settings.json +++ b/frontend/src/locales/en_US/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Show platforms section at the home page", "show-recently-added": "Show recently added ROMs", "show-recently-added-desc": "Show recently added ROMs section at the home page", + "show-recommendations": "Show recommendations", + "show-recommendations-desc": "Show recommendations on the home page and on a game's details page", "show-regions": "Show regions", "show-regions-desc": "Show region flags in the gallery", "show-siblings": "Show versions", diff --git a/frontend/src/locales/es_ES/recommendations.json b/frontend/src/locales/es_ES/recommendations.json new file mode 100644 index 0000000000..45b0e96f31 --- /dev/null +++ b/frontend/src/locales/es_ES/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Porque jugaste a {0}", + "for-you": "Recomendado para ti", + "reason-igdb": "Similar en IGDB", + "reason-top-rated": "Muy bien valorado", + "similar-games": "Juegos similares", + "why": "Por qué se ha recomendado esto" +} diff --git a/frontend/src/locales/es_ES/settings.json b/frontend/src/locales/es_ES/settings.json index ebb59dd5f0..1d0932441c 100644 --- a/frontend/src/locales/es_ES/settings.json +++ b/frontend/src/locales/es_ES/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Mostrar la sección de plataformas en la página principal", "show-recently-added": "Mostrar ROMs añadidos recientemente", "show-recently-added-desc": "Mostrar la sección de ROMs añadidos recientemente en la página principal", + "show-recommendations": "Mostrar recomendaciones", + "show-recommendations-desc": "Mostrar las recomendaciones en la página de inicio y en la página de detalles de un juego", "show-regions": "Mostrar regiones", "show-regions-desc": "Mostrar banderas de region en la galería", "show-siblings": "Mostrar versiones", diff --git a/frontend/src/locales/fr_FR/recommendations.json b/frontend/src/locales/fr_FR/recommendations.json new file mode 100644 index 0000000000..b3781c48f4 --- /dev/null +++ b/frontend/src/locales/fr_FR/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Parce que vous avez joué à {0}", + "for-you": "Recommandé pour vous", + "reason-igdb": "Similaire sur IGDB", + "reason-top-rated": "Très bien noté", + "similar-games": "Jeux similaires", + "why": "Pourquoi cette recommandation" +} diff --git a/frontend/src/locales/fr_FR/settings.json b/frontend/src/locales/fr_FR/settings.json index edf6b2c7cb..a4e6111683 100644 --- a/frontend/src/locales/fr_FR/settings.json +++ b/frontend/src/locales/fr_FR/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Afficher la section des plateformes sur la page d'accueil", "show-recently-added": "Afficher les ROMs récemment ajoutés", "show-recently-added-desc": "Afficher la section des ROMs récemment ajoutés sur la page d'accueil", + "show-recommendations": "Afficher les recommandations", + "show-recommendations-desc": "Afficher les recommandations sur la page d'accueil et sur la page de détails d'un jeu", "show-regions": "Afficher les régions", "show-regions-desc": "Afficher les drapeaux des régions dans la galerie", "show-siblings": "Afficher les versions", diff --git a/frontend/src/locales/hu_HU/recommendations.json b/frontend/src/locales/hu_HU/recommendations.json new file mode 100644 index 0000000000..8e57928014 --- /dev/null +++ b/frontend/src/locales/hu_HU/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Mert játszottál ezzel: {0}", + "for-you": "Neked ajánljuk", + "reason-igdb": "Hasonló az IGDB szerint", + "reason-top-rated": "Magasra értékelt", + "similar-games": "Hasonló játékok", + "why": "Miért ajánlottuk ezt" +} diff --git a/frontend/src/locales/hu_HU/settings.json b/frontend/src/locales/hu_HU/settings.json index de79852501..587b0601f1 100644 --- a/frontend/src/locales/hu_HU/settings.json +++ b/frontend/src/locales/hu_HU/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Platformok szekció megjelenítése a kezdőlapon", "show-recently-added": "Nemrég hozzáadott ROM-ok megjelenítése", "show-recently-added-desc": "A nemrég hozzáadott ROM-ok szekció megjelenítése a kezdőlapon", + "show-recommendations": "Ajánlások megjelenítése", + "show-recommendations-desc": "Ajánlások megjelenítése a kezdőlapon és a játék részletei oldalon", "show-regions": "Régiók megjelenítése", "show-regions-desc": "Régiózászlók megjelenítése a galériában", "show-siblings": "Verziók megjelenítése", diff --git a/frontend/src/locales/it_IT/recommendations.json b/frontend/src/locales/it_IT/recommendations.json new file mode 100644 index 0000000000..0d2e24528b --- /dev/null +++ b/frontend/src/locales/it_IT/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Perché hai giocato a {0}", + "for-you": "Consigliati per te", + "reason-igdb": "Simile su IGDB", + "reason-top-rated": "Molto apprezzato", + "similar-games": "Giochi simili", + "why": "Perché è stato consigliato" +} diff --git a/frontend/src/locales/it_IT/settings.json b/frontend/src/locales/it_IT/settings.json index ca2941b552..234b1d1548 100644 --- a/frontend/src/locales/it_IT/settings.json +++ b/frontend/src/locales/it_IT/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Mostra la sezione delle piattaforme nella home", "show-recently-added": "Mostra le ROM aggiunte di recente", "show-recently-added-desc": "Mostra la sezione delle ROM aggiunte di recente nella home", + "show-recommendations": "Mostra consigli", + "show-recommendations-desc": "Mostra i consigli nella pagina iniziale e nella pagina dei dettagli di un gioco", "show-regions": "Mostra regioni", "show-regions-desc": "Mostra le bandiere delle regioni nella galleria", "show-siblings": "Mostra versioni", diff --git a/frontend/src/locales/ja_JP/recommendations.json b/frontend/src/locales/ja_JP/recommendations.json new file mode 100644 index 0000000000..53c84c234a --- /dev/null +++ b/frontend/src/locales/ja_JP/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "{0} をプレイしたため", + "for-you": "あなたへのおすすめ", + "reason-igdb": "IGDB で類似", + "reason-top-rated": "高評価", + "similar-games": "類似のゲーム", + "why": "おすすめの理由" +} diff --git a/frontend/src/locales/ja_JP/settings.json b/frontend/src/locales/ja_JP/settings.json index 289588da9d..b1e5704773 100644 --- a/frontend/src/locales/ja_JP/settings.json +++ b/frontend/src/locales/ja_JP/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "ホームにプラットフォームを表示", "show-recently-added": "最近追加されたromを表示", "show-recently-added-desc": "ホームに最近追加されたromセクションを表示", + "show-recommendations": "おすすめを表示", + "show-recommendations-desc": "ホーム画面とゲームの詳細ページにおすすめを表示します", "show-regions": "リージョンを表示", "show-regions-desc": "ギャラリーにリージョンを表示", "show-siblings": "バージョンを表示", diff --git a/frontend/src/locales/ko_KR/recommendations.json b/frontend/src/locales/ko_KR/recommendations.json new file mode 100644 index 0000000000..5df54aa815 --- /dev/null +++ b/frontend/src/locales/ko_KR/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "{0}을(를) 플레이했기 때문", + "for-you": "추천 게임", + "reason-igdb": "IGDB 기준 유사", + "reason-top-rated": "높은 평점", + "similar-games": "비슷한 게임", + "why": "추천된 이유" +} diff --git a/frontend/src/locales/ko_KR/settings.json b/frontend/src/locales/ko_KR/settings.json index e956611f80..2efb71951e 100644 --- a/frontend/src/locales/ko_KR/settings.json +++ b/frontend/src/locales/ko_KR/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "홈페이지에 플랫폼 페이지를 보여줍니다", "show-recently-added": "최근 추가된 롬 보이기", "show-recently-added-desc": "홈페이지에 최근에 추가됨 페이지를 보여줍니다", + "show-recommendations": "추천 표시", + "show-recommendations-desc": "홈 화면과 게임 상세 페이지에 추천을 표시합니다", "show-regions": "지역 보이기", "show-regions-desc": "갤러리에서 지역을 보여줍니다", "show-siblings": "버전 보이기", diff --git a/frontend/src/locales/pl_PL/recommendations.json b/frontend/src/locales/pl_PL/recommendations.json new file mode 100644 index 0000000000..ce2bd3e779 --- /dev/null +++ b/frontend/src/locales/pl_PL/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Ponieważ grałeś w {0}", + "for-you": "Polecane dla Ciebie", + "reason-igdb": "Podobne w IGDB", + "reason-top-rated": "Wysoko oceniane", + "similar-games": "Podobne gry", + "why": "Dlaczego to polecono" +} diff --git a/frontend/src/locales/pl_PL/settings.json b/frontend/src/locales/pl_PL/settings.json index ae4f42803d..673476b9c3 100644 --- a/frontend/src/locales/pl_PL/settings.json +++ b/frontend/src/locales/pl_PL/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Pokaż sekcję platform na stronie głównej", "show-recently-added": "Pokaż ostatnio dodane ROM-y", "show-recently-added-desc": "Pokaż sekcję ostatnio dodanych ROM-ów na stronie głównej", + "show-recommendations": "Pokaż polecane", + "show-recommendations-desc": "Pokaż rekomendacje na stronie głównej i na stronie szczegółów gry", "show-regions": "Pokaż regiony", "show-regions-desc": "Pokaż flagi regionów w galerii", "show-siblings": "Pokaż wersje", diff --git a/frontend/src/locales/pt_BR/recommendations.json b/frontend/src/locales/pt_BR/recommendations.json new file mode 100644 index 0000000000..77c974e27a --- /dev/null +++ b/frontend/src/locales/pt_BR/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Porque você jogou {0}", + "for-you": "Recomendado para você", + "reason-igdb": "Semelhante no IGDB", + "reason-top-rated": "Muito bem avaliado", + "similar-games": "Jogos semelhantes", + "why": "Por que isto foi recomendado" +} diff --git a/frontend/src/locales/pt_BR/settings.json b/frontend/src/locales/pt_BR/settings.json index ce7cc132a8..13219235fe 100644 --- a/frontend/src/locales/pt_BR/settings.json +++ b/frontend/src/locales/pt_BR/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Mostrar seção de plataformas na página inicial", "show-recently-added": "Mostrar ROMs adicionados recentemente", "show-recently-added-desc": "Mostrar seção de ROMs adicionados recentemente na página inicial", + "show-recommendations": "Mostrar recomendações", + "show-recommendations-desc": "Mostrar as recomendações na página inicial e na página de detalhes de um jogo", "show-regions": "Mostrar regiões", "show-regions-desc": "Mostrar bandeiras de regiões na galeria", "show-siblings": "Mostrar versões", diff --git a/frontend/src/locales/ro_RO/recommendations.json b/frontend/src/locales/ro_RO/recommendations.json new file mode 100644 index 0000000000..75317a89c5 --- /dev/null +++ b/frontend/src/locales/ro_RO/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Pentru că ai jucat {0}", + "for-you": "Recomandat pentru tine", + "reason-igdb": "Similar pe IGDB", + "reason-top-rated": "Foarte bine cotat", + "similar-games": "Jocuri similare", + "why": "De ce a fost recomandat" +} diff --git a/frontend/src/locales/ro_RO/settings.json b/frontend/src/locales/ro_RO/settings.json index ccf6a02c6f..243bee9cb6 100644 --- a/frontend/src/locales/ro_RO/settings.json +++ b/frontend/src/locales/ro_RO/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Afișează secțiunea platformelor pe pagina principală", "show-recently-added": "Afișează rom-urile adăugate recent", "show-recently-added-desc": "Afișează secțiunea cu rom-urile adăugate recent pe pagina principală", + "show-recommendations": "Afișează recomandările", + "show-recommendations-desc": "Afișează recomandările pe pagina principală și pe pagina de detalii a unui joc", "show-regions": "Afișează regiunile", "show-regions-desc": "Afișează steagurile regiunilor în galerie", "show-siblings": "Afișează versiunile", diff --git a/frontend/src/locales/ru_RU/recommendations.json b/frontend/src/locales/ru_RU/recommendations.json new file mode 100644 index 0000000000..b17189d4ab --- /dev/null +++ b/frontend/src/locales/ru_RU/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "Потому что вы играли в {0}", + "for-you": "Рекомендуем вам", + "reason-igdb": "Похоже по IGDB", + "reason-top-rated": "Высокий рейтинг", + "similar-games": "Похожие игры", + "why": "Почему это рекомендовано" +} diff --git a/frontend/src/locales/ru_RU/settings.json b/frontend/src/locales/ru_RU/settings.json index 36e141e26a..711d7fd426 100644 --- a/frontend/src/locales/ru_RU/settings.json +++ b/frontend/src/locales/ru_RU/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Показать раздел платформ на главной странице", "show-recently-added": "Показать недавно добавленные ромы", "show-recently-added-desc": "Показать раздел недавно добавленных ромов на главной странице", + "show-recommendations": "Показывать рекомендации", + "show-recommendations-desc": "Показывать рекомендации на главной странице и на странице сведений об игре", "show-regions": "Показать регионы", "show-regions-desc": "Показать флаги регионов в галерее", "show-siblings": "Показать версии", diff --git a/frontend/src/locales/tr_TR/recommendations.json b/frontend/src/locales/tr_TR/recommendations.json new file mode 100644 index 0000000000..6886774b26 --- /dev/null +++ b/frontend/src/locales/tr_TR/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "{0} oyununu oynadığınız için", + "for-you": "Size özel öneriler", + "reason-igdb": "IGDB'de benzer", + "reason-top-rated": "Yüksek puanlı", + "similar-games": "Benzer oyunlar", + "why": "Bu neden önerildi" +} diff --git a/frontend/src/locales/tr_TR/settings.json b/frontend/src/locales/tr_TR/settings.json index 2b25b25826..40c7695af7 100644 --- a/frontend/src/locales/tr_TR/settings.json +++ b/frontend/src/locales/tr_TR/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "Ana sayfada platformlar bölümünü göster", "show-recently-added": "Son eklenen ROM'ları göster", "show-recently-added-desc": "Ana sayfada son eklenen ROM'lar bölümünü göster", + "show-recommendations": "Önerileri göster", + "show-recommendations-desc": "Önerileri ana sayfada ve bir oyunun ayrıntılar sayfasında göster", "show-regions": "Bölgeleri göster", "show-regions-desc": "Galeride bölge bayraklarını göster", "show-siblings": "Kardeşleri göster", diff --git a/frontend/src/locales/zh_CN/recommendations.json b/frontend/src/locales/zh_CN/recommendations.json new file mode 100644 index 0000000000..865f64b2a3 --- /dev/null +++ b/frontend/src/locales/zh_CN/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "因为你玩过 {0}", + "for-you": "为你推荐", + "reason-igdb": "IGDB 上的相似作品", + "reason-top-rated": "高评分", + "similar-games": "相似游戏", + "why": "推荐理由" +} diff --git a/frontend/src/locales/zh_CN/settings.json b/frontend/src/locales/zh_CN/settings.json index fe064c806b..45ba163e68 100644 --- a/frontend/src/locales/zh_CN/settings.json +++ b/frontend/src/locales/zh_CN/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "在主页上显示平台部分", "show-recently-added": "显示最近添加的 Rom 文件", "show-recently-added-desc": "在主页上显示最近添加的 Rom 文件部分", + "show-recommendations": "显示推荐", + "show-recommendations-desc": "在主页和游戏详情页显示推荐", "show-regions": "显示区域", "show-regions-desc": "在游戏库中显示区域标识", "show-siblings": "显示版本", diff --git a/frontend/src/locales/zh_TW/recommendations.json b/frontend/src/locales/zh_TW/recommendations.json new file mode 100644 index 0000000000..c749a6bc58 --- /dev/null +++ b/frontend/src/locales/zh_TW/recommendations.json @@ -0,0 +1,8 @@ +{ + "because-you-played": "因為你玩過 {0}", + "for-you": "為你推薦", + "reason-igdb": "IGDB 上的相似作品", + "reason-top-rated": "高評分", + "similar-games": "相似遊戲", + "why": "推薦理由" +} diff --git a/frontend/src/locales/zh_TW/settings.json b/frontend/src/locales/zh_TW/settings.json index 028f453775..0260c542e2 100644 --- a/frontend/src/locales/zh_TW/settings.json +++ b/frontend/src/locales/zh_TW/settings.json @@ -417,6 +417,8 @@ "show-platforms-desc": "在首頁上顯示平台", "show-recently-added": "顯示最近新增的 Rom", "show-recently-added-desc": "在首頁上顯示最近新增的 Rom", + "show-recommendations": "顯示推薦", + "show-recommendations-desc": "在首頁和遊戲詳細資料頁顯示推薦", "show-regions": "顯示地區", "show-regions-desc": "在遊戲庫中顯示地區標示", "show-siblings": "顯示版本", diff --git a/frontend/src/services/api/rom.ts b/frontend/src/services/api/rom.ts index 895f9765f8..d7da3362cd 100644 --- a/frontend/src/services/api/rom.ts +++ b/frontend/src/services/api/rom.ts @@ -6,9 +6,11 @@ import type { BulkOperationResponse, DetailedRomSchema, ManualMetadata, + RecommendedRomSchema, RomUserData, RomUserSchema, SearchRomSchema, + SimilarRomSchema, SimpleRomSchema, SoundtrackTrackMetaSchema, UserNoteSchema, @@ -393,6 +395,41 @@ async function getRecentPlayedRoms() { }); } +export const SIMILAR_ROMS_LIMIT = 12; +export const RECOMMENDED_ROMS_LIMIT = 15; + +/** Library games similar to this one, from the precomputed similarity index. */ +async function getSimilarRoms({ + romId, + limit = SIMILAR_ROMS_LIMIT, + signal, +}: { + romId: number; + limit?: number; + signal?: AbortSignal; +}) { + return api.get(`/roms/${romId}/similar`, { + params: { limit }, + signal, + }); +} + +/** Personalised recommendations for the signed-in user. */ +async function getRecommendedRoms({ + limit = RECOMMENDED_ROMS_LIMIT, + refresh = false, + signal, +}: { + limit?: number; + refresh?: boolean; + signal?: AbortSignal; +} = {}) { + return api.get("/recommendations", { + params: { limit, ...(refresh ? { refresh: true } : {}) }, + signal, + }); +} + async function getRom({ romId, signal, @@ -946,6 +983,8 @@ export default { getRoms, getRecentRoms, getRecentPlayedRoms, + getSimilarRoms, + getRecommendedRoms, getRom, getRomSimple, getRandomRom, diff --git a/frontend/src/v2/components/GameDetails/OverviewTab.test.ts b/frontend/src/v2/components/GameDetails/OverviewTab.test.ts new file mode 100644 index 0000000000..e865d6c065 --- /dev/null +++ b/frontend/src/v2/components/GameDetails/OverviewTab.test.ts @@ -0,0 +1,99 @@ +import { shallowMount } from "@vue/test-utils"; +import { describe, expect, it, vi } from "vitest"; +import { ref } from "vue"; +import type { SimilarRomSchema } from "@/__generated__"; +import type { DetailedRom } from "@/stores/roms"; +import OverviewTab from "./OverviewTab.vue"; + +vi.mock("vue-i18n", () => ({ + useI18n: () => ({ t: (key: string) => key }), +})); + +const showRecommendations = ref(true); + +vi.mock("@/composables/useUISettings", () => ({ + useUISettings: () => ({ showRecommendations }), +})); + +vi.mock("@/stores/collections", () => ({ + default: () => ({ allCollections: [], smartCollections: [] }), +})); + +vi.mock("@/v2/composables/useWebpSupport", () => ({ + useWebpSupport: () => ({ toWebp: (url: string) => url }), +})); + +// Artwork and collection mosaics belong to other sections; stubbed so this +// stays a test of the recommendations gate. +vi.mock("@/v2/utils/romArtwork", () => ({ resolveRomArtwork: () => [] })); + +vi.mock("@/v2/utils/collectionCovers", () => ({ + collectionCoverList: () => [], +})); + +function similar(id: number): SimilarRomSchema { + return { + rom: { id, name: `Game ${id}` } as SimilarRomSchema["rom"], + score: 0.5, + reasons: [{ facet: "franchise", value: "Metroid" }], + }; +} + +function mount(props: Record = {}) { + return shallowMount(OverviewTab, { + props: { + rom: { id: 1, metadatum: {} } as DetailedRom, + summary: null, + sections: [], + playerCount: null, + userCollections: [], + hltb: null, + lastPlayed: null, + revision: null, + screenshots: [], + expansions: [], + dlcs: [], + remakes: [], + remasters: [], + similarRoms: [similar(2), similar(3)], + ...props, + }, + global: { stubs: { RIcon: true } }, + }); +} + +describe("OverviewTab similar games", () => { + it("shows the section when recommendations are enabled", () => { + showRecommendations.value = true; + + expect(mount().findComponent({ name: "SimilarGamesGrid" }).exists()).toBe( + true, + ); + }); + + it("hides the section when recommendations are turned off", () => { + // The same preference hides the Home row; #3794 asked for the section to + // be switchable off wherever it appears, not only on the home page. + showRecommendations.value = false; + + expect(mount().findComponent({ name: "SimilarGamesGrid" }).exists()).toBe( + false, + ); + }); + + it("leaves the other related sections alone when it is off", () => { + // The preference gates recommendations, not expansions and remakes, and + // those share the panel whose visibility the gate feeds into. + showRecommendations.value = false; + const wrapper = mount({ + expansions: [{ id: 9, name: "Eagle Watch" }], + }); + + expect(wrapper.findComponent({ name: "SimilarGamesGrid" }).exists()).toBe( + false, + ); + expect(wrapper.findComponent({ name: "RelatedGamesGrid" }).exists()).toBe( + true, + ); + }); +}); diff --git a/frontend/src/v2/components/GameDetails/OverviewTab.vue b/frontend/src/v2/components/GameDetails/OverviewTab.vue index 97eb30aae5..08eced48d6 100644 --- a/frontend/src/v2/components/GameDetails/OverviewTab.vue +++ b/frontend/src/v2/components/GameDetails/OverviewTab.vue @@ -24,8 +24,10 @@ import { useI18n } from "vue-i18n"; import type { IGDBRelatedGame, RomHLTBMetadata, + SimilarRomSchema, UserCollectionSchema, } from "@/__generated__"; +import { useUISettings } from "@/composables/useUISettings"; import storeCollections from "@/stores/collections"; import type { DetailedRom } from "@/stores/roms"; import CollectionTile, { @@ -38,6 +40,7 @@ import InfoGrid from "@/v2/components/GameDetails/InfoGrid.vue"; import PlayerCountBadge from "@/v2/components/GameDetails/PlayerCountBadge.vue"; import RelatedGamesGrid from "@/v2/components/GameDetails/RelatedGamesGrid.vue"; import ScreenshotsTab from "@/v2/components/GameDetails/ScreenshotsTab.vue"; +import SimilarGamesGrid from "@/v2/components/GameDetails/SimilarGamesGrid.vue"; import { PROVIDERS, providerId } from "@/v2/components/GameDetails/providers"; import { useWebpSupport } from "@/v2/composables/useWebpSupport"; import { collectionCoverList } from "@/v2/utils/collectionCovers"; @@ -59,9 +62,18 @@ const props = defineProps<{ dlcs: IGDBRelatedGame[]; remakes: IGDBRelatedGame[]; remasters: IGDBRelatedGame[]; - similarGames: IGDBRelatedGame[]; + similarRoms: SimilarRomSchema[]; + webp?: boolean; }>(); +// The same preference hides the "Recommended for you" row on Home, so the +// feature can be switched off wherever it appears rather than per surface. +const { showRecommendations } = useUISettings(); + +const visibleSimilarRoms = computed(() => + showRecommendations.value ? props.similarRoms : [], +); + const hasAgeRatings = computed( () => (props.rom.metadatum?.age_ratings?.length ?? 0) > 0, ); @@ -132,7 +144,7 @@ const hasRelated = computed( props.dlcs.length + props.remakes.length + props.remasters.length + - props.similarGames.length > + visibleSimilarRoms.value.length > 0, ); @@ -314,12 +326,12 @@ const coverSource = computed(() => { -
+

- Similar games + {{ t("recommendations.similar-games") }}

- +
diff --git a/frontend/src/v2/components/GameDetails/SimilarGamesGrid.vue b/frontend/src/v2/components/GameDetails/SimilarGamesGrid.vue new file mode 100644 index 0000000000..1b2f6f9f8e --- /dev/null +++ b/frontend/src/v2/components/GameDetails/SimilarGamesGrid.vue @@ -0,0 +1,80 @@ + + + + + diff --git a/frontend/src/v2/utils/similarityReasons.test.ts b/frontend/src/v2/utils/similarityReasons.test.ts new file mode 100644 index 0000000000..72c914e1cb --- /dev/null +++ b/frontend/src/v2/utils/similarityReasons.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; +import type { SimilarityReasonSchema } from "@/__generated__"; +import { reasonIcon, reasonLabel } from "@/v2/utils/similarityReasons"; + +function reason(facet: string, value = "x"): SimilarityReasonSchema { + return { facet, value } as SimilarityReasonSchema; +} + +const DEFAULT_ICON = "mdi-tag-outline"; + +describe("reasonIcon", () => { + it.each([ + "collection", + "franchise", + "developer", + "publisher", + "company", + "genre", + "theme", + "perspective", + "keyword", + "game_mode", + "platform", + "decade", + "igdb", + "top_rated", + ])("maps %s to a dedicated icon", (facet) => { + expect(reasonIcon(reason(facet))).not.toBe(DEFAULT_ICON); + }); + + it("falls back for a facet it does not know", () => { + expect(reasonIcon(reason("something_new"))).toBe(DEFAULT_ICON); + }); +}); + +describe("reasonLabel", () => { + const t = (key: string) => key; + + it("shows the value for facets that are already proper nouns", () => { + expect(reasonLabel(reason("developer", "Treasure"), t)).toBe("Treasure"); + }); + + it("pluralises a decade", () => { + expect(reasonLabel(reason("decade", "1990"), t)).toBe("1990s"); + }); + + it("translates facets with no meaningful value", () => { + expect(reasonLabel(reason("igdb", ""), t)).toBe( + "recommendations.reason-igdb", + ); + }); +}); diff --git a/frontend/src/v2/utils/similarityReasons.ts b/frontend/src/v2/utils/similarityReasons.ts new file mode 100644 index 0000000000..22222ec925 --- /dev/null +++ b/frontend/src/v2/utils/similarityReasons.ts @@ -0,0 +1,64 @@ +import type { SimilarityReasonSchema } from "@/__generated__"; + +/** + * Maps a recommendation reason onto an icon and a display label. + * + * Facets whose value is already a proper noun (a franchise, a company) + * display that value directly, since "Metroid" explains the match better + * than "Same franchise" does. Facets with no meaningful value of their own + * fall back to a translated phrase. + */ + +const FACET_ICONS: Record = { + collection: "mdi-bookmark-multiple-outline", + franchise: "mdi-star-outline", + developer: "mdi-domain", + publisher: "mdi-domain", + company: "mdi-domain", + genre: "mdi-shape-outline", + theme: "mdi-palette-outline", + perspective: "mdi-camera-outline", + keyword: "mdi-tag-multiple-outline", + game_mode: "mdi-account-group-outline", + platform: "mdi-controller-classic-outline", + decade: "mdi-calendar-outline", + igdb: "mdi-link-variant", + top_rated: "mdi-trophy-outline", +}; + +const DEFAULT_ICON = "mdi-tag-outline"; + +/** Facets rendered as a translated phrase rather than their raw value. */ +const TRANSLATED_FACETS: Record = { + igdb: "recommendations.reason-igdb", + top_rated: "recommendations.reason-top-rated", +}; + +export function reasonIcon(reason: SimilarityReasonSchema): string { + return FACET_ICONS[reason.facet] ?? DEFAULT_ICON; +} + +export function reasonLabel( + reason: SimilarityReasonSchema, + t: (key: string) => string, +): string { + const translationKey = TRANSLATED_FACETS[reason.facet]; + if (translationKey) { + return t(translationKey); + } + + // Decades arrive as the starting year ("1990") and read better with the + // plural suffix the rest of the UI uses. + if (reason.facet === "decade") { + return `${reason.value}s`; + } + + return reason.value; +} + +/** The single most explanatory reason, used where only one chip fits. */ +export function primaryReason( + reasons: SimilarityReasonSchema[], +): SimilarityReasonSchema | null { + return reasons[0] ?? null; +} diff --git a/frontend/src/v2/views/GameDetails.vue b/frontend/src/v2/views/GameDetails.vue index b56f8e9e93..3e66a12692 100644 --- a/frontend/src/v2/views/GameDetails.vue +++ b/frontend/src/v2/views/GameDetails.vue @@ -7,10 +7,10 @@ // sub-component under components/GameDetails/. import { RTabNav, type RTabNavItem } from "@v2/lib"; import { storeToRefs } from "pinia"; -import { computed, ref, watch } from "vue"; +import { computed, ref, watch, watchEffect } from "vue"; import { useI18n } from "vue-i18n"; import { onBeforeRouteUpdate, useRoute, useRouter } from "vue-router"; -import type { IGDBRelatedGame } from "@/__generated__"; +import type { IGDBRelatedGame, SimilarRomSchema } from "@/__generated__"; import romApi from "@/services/api/rom"; import storeAuth from "@/stores/auth"; import storeRoms from "@/stores/roms"; @@ -37,7 +37,7 @@ const router = useRouter(); const romsStore = storeRoms(); const authStore = storeAuth(); const { currentRom } = storeToRefs(romsStore); -const { toWebp } = useWebpSupport(); +const { supportsWebp, toWebp } = useWebpSupport(); const { locale, t } = useI18n(); const setBgArt = useBackgroundArt(); @@ -205,13 +205,40 @@ const earnedAchievementIds = computed>(() => { const achievementsEarned = computed(() => earnedAchievementIds.value.size); const igdb = computed(() => currentRom.value?.igdb_metadata ?? null); -// IGDB ships up to ~10 similar games per title; rendering all of them -// would dominate the overview and push HLTB/Achievements below the -// fold. Cap to keep the section to ~2 rows of cards at typical widths. + +// Similar games come from the server-side recommendations index, not from +// `igdb_metadata.similar_games`: the IGDB list is mostly titles the server +// doesn't hold, and it's missing entirely for anything IGDB never matched. +// Every entry here is a real ROM in this library. +// +// Capped so the section stays ~2 rows of cards at typical widths and +// doesn't push HLTB / Achievements below the fold. const SIMILAR_GAMES_MAX = 6; -const similarGames = computed(() => - (igdb.value?.similar_games ?? []).slice(0, SIMILAR_GAMES_MAX), -); +const similarRoms = ref([]); + +watchEffect((onCleanup) => { + const romId = currentRom.value?.id; + similarRoms.value = []; + if (!romId) return; + + const controller = new AbortController(); + onCleanup(() => controller.abort()); + + romApi + .getSimilarRoms({ + romId, + limit: SIMILAR_GAMES_MAX, + signal: controller.signal, + }) + .then(({ data }) => { + similarRoms.value = data; + }) + .catch(() => { + // The section simply stays hidden: an empty recommendations index + // (never built, or a library too small to relate anything) is a + // normal state, not an error worth interrupting the page for. + }); +}); const remakes = computed(() => igdb.value?.remakes ?? []); const remasters = computed( () => igdb.value?.remasters ?? [], @@ -285,7 +312,8 @@ const tabs = computed(() => [ :dlcs="dlcs" :remakes="remakes" :remasters="remasters" - :similar-games="similarGames" + :similar-roms="similarRoms" + :webp="supportsWebp" /> diff --git a/frontend/src/v2/views/Home.test.ts b/frontend/src/v2/views/Home.test.ts index 3f58ee8797..16009e67d0 100644 --- a/frontend/src/v2/views/Home.test.ts +++ b/frontend/src/v2/views/Home.test.ts @@ -12,14 +12,19 @@ vi.mock("vue-i18n", () => ({ useI18n: () => ({ t: (key: string) => key }), })); -const { getLibraryInfo } = vi.hoisted(() => ({ +const { getLibraryInfo, getRecommendedRoms } = vi.hoisted(() => ({ getLibraryInfo: vi.fn(), + getRecommendedRoms: vi.fn(), })); vi.mock("@/services/api/setup", () => ({ default: { getLibraryInfo }, })); +vi.mock("@/services/api/rom", () => ({ + default: { getRecommendedRoms }, +})); + vi.mock("@v2/lib", () => ({ RChip: defineComponent({ template: "" }), RDivider: defineComponent({ template: "
" }), @@ -64,6 +69,7 @@ vi.mock("@/composables/useUISettings", () => ({ showHomeWidgets: ref(true), showRecentRoms: ref(true), showContinuePlaying: ref(true), + showRecommendations: ref(true), showPlatforms: ref(true), showCollections: ref(true), showSmartCollections: ref(false), @@ -182,6 +188,8 @@ describe("Home", () => { getLibraryInfo.mockResolvedValue({ data: { detected_structure: "struct_a", existing_platforms: [] }, }); + getRecommendedRoms.mockReset(); + getRecommendedRoms.mockResolvedValue({ data: [] }); }); it("never walks the filesystem for a populated library", async () => { @@ -209,6 +217,49 @@ describe("Home", () => { expect(wrapper.text()).toContain("home.empty-headline"); }); + it("renders the recommendations row with its per-card reason", async () => { + stubHomeFetches(true); + getRecommendedRoms.mockResolvedValue({ + data: [ + { + rom: rom(7), + score: 0.8, + reasons: [{ facet: "franchise", value: "Metroid" }], + seed_rom_id: 1, + seed_rom_name: "Super Metroid", + }, + ], + }); + + const wrapper = mountHome(); + await flushPromises(); + + expect(getRecommendedRoms).toHaveBeenCalledTimes(1); + expect(wrapper.text()).toContain("recommendations.because-you-played"); + }); + + it("hides the recommendations row when the feed comes back empty", async () => { + stubHomeFetches(true); + + const wrapper = mountHome(); + await flushPromises(); + + expect(wrapper.text()).not.toContain("recommendations.for-you"); + }); + + it("keeps the home page usable when the feed request fails", async () => { + stubHomeFetches(true); + getRecommendedRoms.mockRejectedValue(new Error("index not built")); + + const wrapper = mountHome(); + await flushPromises(); + + // An unbuilt index must not surface as an error or block the rest of + // the dashboard from rendering. + expect(wrapper.text()).not.toContain("recommendations.for-you"); + expect(wrapper.text()).not.toContain("home.empty-headline"); + }); + it("does not render the empty state before the initial loads settle", async () => { stubHomeFetches(false); diff --git a/frontend/src/v2/views/Home.vue b/frontend/src/v2/views/Home.vue index 5f7f876161..9013c6ff44 100644 --- a/frontend/src/v2/views/Home.vue +++ b/frontend/src/v2/views/Home.vue @@ -11,8 +11,10 @@ import { RChip, RDivider, RIcon, RSkeletonBlock } from "@v2/lib"; import { storeToRefs } from "pinia"; import { computed, onMounted, ref, watch } from "vue"; import { useI18n } from "vue-i18n"; +import type { RecommendedRomSchema } from "@/__generated__"; import { useUISettings } from "@/composables/useUISettings"; import { ROUTES } from "@/plugins/router"; +import romApi from "@/services/api/rom"; import setupApi, { type SetupLibraryInfo } from "@/services/api/setup"; import storeCollections from "@/stores/collections"; import storePlatforms from "@/stores/platforms"; @@ -25,6 +27,11 @@ import PlatformTile from "@/v2/components/Platforms/PlatformTile.vue"; import { useGridNav } from "@/v2/composables/useGridNav"; import { useWebpSupport } from "@/v2/composables/useWebpSupport"; import { collectionCoverList } from "@/v2/utils/collectionCovers"; +import { + primaryReason, + reasonIcon, + reasonLabel, +} from "@/v2/utils/similarityReasons"; const { t } = useI18n(); @@ -36,6 +43,7 @@ const { showHomeWidgets, showRecentRoms, showContinuePlaying, + showRecommendations, showPlatforms, showCollections, showSmartCollections, @@ -58,6 +66,26 @@ const { const fetchingRecent = ref(false); const fetchingContinue = ref(false); +// Personalised recommendations. Ranked server-side from the similarity index +// plus this user's play history, so the row is fetched here rather than +// derived from the ROM store's existing rails. +const recommendedRoms = ref([]); +const fetchingRecommendations = ref(false); + +async function loadRecommendations() { + fetchingRecommendations.value = true; + try { + const { data } = await romApi.getRecommendedRoms(); + recommendedRoms.value = data; + } catch { + // An unbuilt index or a library too small to relate anything is a normal + // state, not an error: the row just stays hidden. + recommendedRoms.value = []; + } finally { + fetchingRecommendations.value = false; + } +} + const gridRoot = ref(null); useGridNav(gridRoot); @@ -97,6 +125,9 @@ onMounted(async () => { .finally(() => (fetchingContinue.value = false)), ); } + if (showRecommendations.value) { + initialLoads.push(loadRecommendations()); + } await Promise.allSettled(initialLoads); initialLoadDone.value = true; @@ -315,6 +346,54 @@ function collectionCovers(c: { + + + + + + + +