diff --git a/alpina.md b/alpina.md new file mode 100644 index 0000000..343fea4 --- /dev/null +++ b/alpina.md @@ -0,0 +1,153 @@ +# Alpina ISBN Issues And Next Task + +## Context + +В этой сессии разбирали два ISBN из книг издательства Альпина / Alpina non-fiction. + +### 1. ISBN `9785916717099` + +Проблема: +- книга находилась, но без автора +- позже выяснилось, что title тоже был шумный из marketplace-выдачи + +Ожидаемые данные: +- `title`: `От 0 до 5: Простые подсказки для умных родителей` +- `author`: `Трейси Катчлоу` + +Что уже сделано: +- в `home_library/providers/ddg.py` добавлен разбор автора из DDG snippets +- добавлена очистка шумных title для кейсов `АНФ`, `арт. `, возрастных меток, marketplace-префиксов и edition suffix +- добавлены тесты в `tests/test_home_library_providers.py`, включая отдельный тест на эту книгу + +Git: +- ветка: `fix/alpina-non-fiction` +- commit: `4d6c613` +- PR: `https://github.com/gman-dev-nov/Home-Library-Telegram-Bot/pull/1` + +### 2. ISBN `9785961449136` + +Проблема: +- книга не находится вообще +- ожидаемая книга: `Марк Мэнсон - Мужские правила: Отношения, секс, психология` + +Ожидаемые данные: +- `title`: `Мужские правила: Отношения, секс, психология` +- `author`: `Марк Мэнсон` + +## Reproducer + +Текущий репродьюсер для проблемного ISBN `9785961449136`: + +```bash +venv/bin/python - <<'PY' +import asyncio +import httpx +from home_library.providers.labirint import fetch_from_labirint +from home_library.providers.piter import fetch_from_piter +from home_library.providers.google_books import fetch_from_google_books +from home_library.providers.ddg import fetch_from_ddg +from home_library.providers.lookup import fetch_book_by_isbn + +ISBN = "9785961449136" + +async def main(): + async with httpx.AsyncClient(timeout=20) as client: + for name, fn in [ + ("labirint", fetch_from_labirint), + ("piter", fetch_from_piter), + ("google", fetch_from_google_books), + ("ddg", fetch_from_ddg), + ]: + try: + book = await fn(ISBN, client) + except Exception as exc: + print(name, "ERROR", type(exc).__name__, exc) + continue + print(name, book) + + book = await fetch_book_by_isbn(ISBN) + print("lookup", book) + +asyncio.run(main()) +PY +``` + +Наблюдаемое поведение на момент записи: +- `labirint -> None` +- `piter -> None` +- `google -> None` +- `ddg -> None` +- `lookup -> None` + +## Extra Investigation Notes + +Что проверено по `9785961449136`: + +- `Google Books` по ISBN возвращает `totalItems: 0` +- `OpenLibrary` по ISBN возвращает `{}` +- `Labirint` по ISBN не даёт результата +- `Piter` по ISBN возвращает пустой список +- `DuckDuckGo` по чистому ISBN почти пустой и не даёт данных для парсинга + +Важная находка: +- если в DDG добавить контекст, например + - `9785961449136 Мужские правила` + - `9785961449136 Марк Мэнсон` +- нужная книга сразу находится + +Но это не решает задачу, потому что на вход текущего pipeline приходит только ISBN, а фразы `Мужские правила` или `Марк Мэнсон` бот сам ниоткуда не знает. + +Что ещё смотрели: +- `Читай-город` по server-rendered HTML пишет `Поиск по запросу «9785961449136», не принес результатов` +- `site:litres.ru`, `site:alpinabook.ru`, `9785961449136 Альпина` через DDG тоже оказались ненадёжными для автоматического ISBN-only поиска + +## Root Problem + +Текущая цепочка провайдеров: + +- `Labirint` +- `Piter` +- `Google Books` +- `DuckDuckGo` fallback + +Для `9785961449136` этого недостаточно. + +Проблема не в Telegram formatter и не в выводе карточки. +Проблема в отсутствии провайдера, который умеет стабильно находить эту книгу по одному ISBN. + +## What Needs To Be Done + +Нужна отдельная задача на реализацию нового провайдера ISBN metadata. + +Цель: +- научить `fetch_book_by_isbn("9785961449136")` возвращать книгу + - `title = "Мужские правила: Отношения, секс, психология"` + - `author = "Марк Мэнсон"` + +Предпочтительный подход: +1. Найти новый источник, который реально работает по одному ISBN. +2. Добавить новый провайдер в `home_library/providers/`. +3. Подключить его в `home_library/providers/lookup.py` перед `DuckDuckGo` fallback. +4. Покрыть провайдер unit-тестами. +5. Добавить regression-тест на ISBN `9785961449136`. + +## Suggested Task For Next Session + +Реализовать новый провайдер поиска книги по ISBN для кейсов, которые не покрываются `Labirint`, `Piter`, `Google Books` и текущим `DuckDuckGo` fallback. + +Обязательные требования: +- сохранить текущую логику и не ломать уже исправленный кейс `9785916717099` +- добавить regression coverage для `9785961449136` +- не усложнять DDG эвристиками без нового источника, если можно добавить более надёжный провайдер + +Definition of done: +- `fetch_book_by_isbn("9785961449136")` возвращает не `None` +- title и author соответствуют ожидаемым значениям +- тесты провайдеров проходят + +## Candidate Checks For New Provider + +В следующей сессии стоит первым делом проверить: +- есть ли стабильный JSON/API или server-side HTML у `Альпина` +- есть ли пригодный источник у другого магазина, который ищет по ISBN без client-side-only выдачи +- можно ли использовать ещё один открытый книжный каталог вместо расширения DDG diff --git a/home_library/benchmark.py b/home_library/benchmark.py new file mode 100644 index 0000000..ff76bad --- /dev/null +++ b/home_library/benchmark.py @@ -0,0 +1,785 @@ +"""CLI-бенчмарк книжных провайдеров. + +Поддерживает два режима: +- ``search-isbn``: поиск ISBN по ``author`` + ``title``; +- ``resolve-metadata``: поиск метаданных по ISBN. + +Скрипт читает книги из ``library.db`` и печатает только таблицы, +без JSON-выгрузок. +""" + +from __future__ import annotations + +import argparse +import asyncio +import importlib.util +import logging +import os +import re +from collections import Counter +from dataclasses import dataclass +from typing import Callable + +import httpx + +from home_library import config +from home_library.domain.models import BookRecord +from home_library.providers.brave_search import fetch_from_brave_search +from home_library.providers import yandex +from home_library.providers.google_search import fetch_from_google_search +from home_library.providers.litres import fetch_from_litres +from home_library.providers.mybook import fetch_from_mybook +from home_library.providers.ozon import fetch_from_ozon +from home_library.providers._playwright_fetch import playwright_available +from home_library.providers._serp_parser import extract_book_from_serp +from home_library.providers.google_books import fetch_from_google_books +from home_library.providers.labirint import _parse_labirint_book_page +from home_library.providers.labirint import fetch_from_labirint +from home_library.providers.piter import fetch_from_piter +from home_library.providers.wildberries import fetch_from_wildberries +from home_library.providers.yandex_books import fetch_from_yandex_books +from home_library.storage.sqlite import get_all_books + +logger = logging.getLogger(__name__) + +PROVIDER_ERROR = "error" +PROVIDER_NOT_FOUND = "not_found" +PROVIDER_FOUND = "found" +PROVIDER_DISABLED = "disabled" +PROVIDER_MISSING_DEPS = "missing_deps" + +SEARCH_FETCH_LIMIT = 3 +BENCHMARK_CONCURRENCY = 8 +ISBN13_PATTERN = re.compile(r"\b97[89](?:[\-\s]?\d){10}\b") +ISBN10_PATTERN = re.compile(r"\b\d(?:[\-\s]?\d){8}[\dXx]\b") +NON_ALNUM_PATTERN = re.compile(r"[^\w\s]+", re.UNICODE) +SPACE_PATTERN = re.compile(r"\s+") + + +@dataclass(slots=True) +class ProviderRunResult: + """Результат одного провайдера для одной книги.""" + + provider_name: str + status: str + book: BookRecord | None = None + detail: str = "" + + +@dataclass(slots=True) +class BenchmarkProvider: + """Провайдер, участвующий в бенчмарке.""" + + name: str + search_isbn: Callable[[BookRecord, httpx.AsyncClient], asyncio.Future] + resolve_metadata: Callable[[str, httpx.AsyncClient], asyncio.Future] + supports_search_isbn: bool = True + supports_resolve_metadata: bool = True + + +@dataclass(slots=True) +class SearchIsbnStats: + """Агрегированная статистика поиска ISBN.""" + + provider_name: str + raw_found: int = 0 + verified: int = 0 + not_found: int = 0 + disabled: int = 0 + missing_deps: int = 0 + errors: int = 0 + + @property + def precision(self) -> float: + if self.raw_found == 0: + return 0.0 + return self.verified / self.raw_found + + +@dataclass(slots=True) +class ResolveMetadataStats: + """Агрегированная статистика обратного поиска метаданных.""" + + provider_name: str + coverage: int = 0 + title_match: int = 0 + author_match: int = 0 + full_match: int = 0 + not_found: int = 0 + disabled: int = 0 + missing_deps: int = 0 + errors: int = 0 + + +def _normalize_text(value: str) -> str: + """Нормализует строку для грубого сопоставления.""" + normalized = value.casefold().replace("ё", "е") + normalized = NON_ALNUM_PATTERN.sub(" ", normalized) + normalized = SPACE_PATTERN.sub(" ", normalized) + return normalized.strip() + + +def _title_matches(expected: str, actual: str) -> bool: + """Проверяет грубое совпадение названия книги.""" + expected_normalized = _normalize_text(expected) + actual_normalized = _normalize_text(actual) + if not expected_normalized or not actual_normalized: + return False + return ( + expected_normalized == actual_normalized + or expected_normalized in actual_normalized + or actual_normalized in expected_normalized + ) + + +def _author_matches(expected: str, actual: str) -> bool: + """Проверяет грубое совпадение автора по набору токенов.""" + expected_tokens = {token for token in _normalize_text(expected).split() if len(token) > 1} + actual_tokens = {token for token in _normalize_text(actual).split() if len(token) > 1} + if not expected_tokens or not actual_tokens: + return False + overlap = expected_tokens & actual_tokens + return len(overlap) >= min(2, len(expected_tokens), len(actual_tokens)) + + +def _book_match_score(expected: BookRecord, candidate: BookRecord) -> int: + """Возвращает грубую оценку совпадения книги с кандидатом.""" + score = 0 + if _title_matches(expected.title, candidate.title): + score += 2 + if expected.author and candidate.author and _author_matches(expected.author, candidate.author): + score += 2 + elif not expected.author and candidate.author: + score += 1 + return score + + +def _pick_isbn(text: str) -> str: + """Достаёт наиболее правдоподобный ISBN из текста.""" + candidates: list[str] = [] + for pattern in (ISBN13_PATTERN, ISBN10_PATTERN): + for match in pattern.findall(text): + normalized = re.sub(r"[^\dXx]", "", match) + if len(normalized) in {config.ISBN10_LENGTH, config.ISBN13_LENGTH}: + candidates.append(normalized.upper()) + + if not candidates: + return "" + + counts = Counter(candidates) + best, _count = max( + counts.items(), + key=lambda item: (item[1], len(item[0]) == config.ISBN13_LENGTH, len(item[0]), item[0]), + ) + return best + + +def _yandex_precheck() -> str | None: + """Возвращает причину, по которой Yandex недоступен, или ``None``.""" + if os.environ.get(yandex.YANDEX_ENABLED_ENV) != "1": + return PROVIDER_DISABLED + try: + playwright_spec = importlib.util.find_spec("playwright") + stealth_spec = importlib.util.find_spec("playwright_stealth") + except ModuleNotFoundError: + return PROVIDER_MISSING_DEPS + if playwright_spec is None: + return PROVIDER_MISSING_DEPS + if stealth_spec is None: + return PROVIDER_MISSING_DEPS + return None + + +def _playwright_precheck() -> str | None: + """Возвращает причину недоступности Playwright benchmark-провайдеров.""" + return None if playwright_available() else PROVIDER_MISSING_DEPS + + +async def _fetch_labirint_page_by_query(query: str, client: httpx.AsyncClient) -> str: + """Возвращает HTML поисковой страницы Labirint по текстовому запросу.""" + response = await client.get( + config.LABIRINT_SEARCH_URL.format(isbn=query), + headers=config.LABIRINT_HEADERS, + follow_redirects=True, + ) + response.raise_for_status() + return response.text + + +async def _search_isbn_via_labirint(book: BookRecord, client: httpx.AsyncClient) -> ProviderRunResult: + """Ищет ISBN по автору и названию через Labirint.""" + try: + query = " ".join(part for part in (book.author, book.title) if part) + search_html = await _fetch_labirint_page_by_query(query, client) + book_ids = re.findall(r'data-product-id="(\d+)"', search_html) + if not book_ids: + return ProviderRunResult("Labirint", PROVIDER_NOT_FOUND) + + best_match: BookRecord | None = None + best_score = -1 + for book_id in book_ids[:SEARCH_FETCH_LIMIT]: + book_url = f"https://www.labirint.ru/books/{book_id}/" + page = await client.get(book_url, headers=config.LABIRINT_HEADERS, follow_redirects=True) + if page.status_code != 200: + continue + parsed = _parse_labirint_book_page(page.text, book_url, "") + if parsed is None: + continue + isbn = _pick_isbn(page.text) + if not isbn: + continue + parsed.isbn = isbn + score = _book_match_score(book, parsed) + if score > best_score: + best_match = parsed + best_score = score + if score >= 4: + break + if best_match is not None: + return ProviderRunResult("Labirint", PROVIDER_FOUND, book=best_match) + return ProviderRunResult("Labirint", PROVIDER_NOT_FOUND) + except Exception as exc: # noqa: BLE001 + return ProviderRunResult("Labirint", PROVIDER_ERROR, detail=str(exc)) + + +async def _search_isbn_via_google(book: BookRecord, client: httpx.AsyncClient) -> ProviderRunResult: + """Ищет ISBN по автору и названию через Google Books.""" + try: + query = f"intitle:{book.title} inauthor:{book.author}" if book.author else f"intitle:{book.title}" + response = await client.get(config.GOOGLE_BOOKS_URL, params={"q": query}) + response.raise_for_status() + data = response.json() + items = data.get("items", []) + best_match: BookRecord | None = None + best_score = -1 + for item in items[:SEARCH_FETCH_LIMIT]: + info = item.get("volumeInfo", {}) + identifiers = info.get("industryIdentifiers", []) + isbn = "" + for identifier in identifiers: + candidate = str(identifier.get("identifier", "")).replace("-", "") + if len(candidate) == config.ISBN13_LENGTH: + isbn = candidate + break + if len(candidate) == config.ISBN10_LENGTH and not isbn: + isbn = candidate + if not isbn: + continue + candidate = BookRecord( + title=info.get("title", ""), + author=", ".join(info.get("authors", [])), + publisher=info.get("publisher", ""), + isbn=isbn, + ) + score = _book_match_score(book, candidate) + if score > best_score: + best_match = candidate + best_score = score + if score >= 4: + break + if best_match is not None: + return ProviderRunResult("Google Books", PROVIDER_FOUND, book=best_match) + return ProviderRunResult("Google Books", PROVIDER_NOT_FOUND) + except Exception as exc: # noqa: BLE001 + return ProviderRunResult("Google Books", PROVIDER_ERROR, detail=str(exc)) + + +async def _search_isbn_via_piter(book: BookRecord, client: httpx.AsyncClient) -> ProviderRunResult: + """Пытается найти ISBN по поиску Piter и странице продукта.""" + try: + query = " ".join(part for part in (book.author, book.title) if part) + response = await client.get(config.PITER_SEARCH_URL, params={"q": query}) + response.raise_for_status() + products = response.json() + if not isinstance(products, list) or not products: + return ProviderRunResult("Piter", PROVIDER_NOT_FOUND) + + best_match: BookRecord | None = None + best_score = -1 + for product in products[:SEARCH_FETCH_LIMIT]: + product_url = str(product.get("url", "")) + if not product_url: + continue + page = await client.get(config.PITER_BOOK_URL.format(url=product_url), follow_redirects=True) + if page.status_code != 200: + continue + isbn = _pick_isbn(page.text) + if not isbn: + continue + candidate = BookRecord( + title=str(product.get("title", "")), + author=book.author, + publisher="Питер", + isbn=isbn, + link=config.PITER_BOOK_URL.format(url=product_url), + ) + score = _book_match_score(book, candidate) + if score > best_score: + best_match = candidate + best_score = score + if score >= 4: + break + if best_match is not None: + return ProviderRunResult("Piter", PROVIDER_FOUND, book=best_match) + return ProviderRunResult("Piter", PROVIDER_NOT_FOUND) + except Exception as exc: # noqa: BLE001 + return ProviderRunResult("Piter", PROVIDER_ERROR, detail=str(exc)) + + +async def _search_isbn_via_ddg(book: BookRecord, client: httpx.AsyncClient) -> ProviderRunResult: + """Ищет ISBN через DuckDuckGo по запросу title/author.""" + try: + query = " ".join(part for part in (book.author, book.title) if part) + response = await client.get( + "https://html.duckduckgo.com/html/", + params={"q": query}, + headers={"User-Agent": "Mozilla/5.0"}, + follow_redirects=True, + ) + response.raise_for_status() + titles = re.findall(r'class="result__a"[^>]*>(.*?)]*>(.*?)', response.text, re.DOTALL) + isbn = _pick_isbn(response.text) + if not isbn: + return ProviderRunResult("DuckDuckGo", PROVIDER_NOT_FOUND) + parsed = extract_book_from_serp(titles[: config.SEARCH_LIMIT], isbn, snippets=snippets[: config.SEARCH_LIMIT]) + if parsed is None: + parsed = BookRecord(title=book.title, author=book.author, isbn=isbn) + return ProviderRunResult("DuckDuckGo", PROVIDER_FOUND, book=parsed) + except Exception as exc: # noqa: BLE001 + return ProviderRunResult("DuckDuckGo", PROVIDER_ERROR, detail=str(exc)) + + +async def _search_isbn_via_yandex(book: BookRecord, client: httpx.AsyncClient) -> ProviderRunResult: + """Ищет ISBN через Yandex по запросу title/author.""" + unavailable = _yandex_precheck() + if unavailable is not None: + return ProviderRunResult("Yandex", unavailable) + + try: + from playwright.async_api import async_playwright # noqa: PLC0415 + from playwright_stealth import Stealth # noqa: PLC0415 + except ImportError: + return ProviderRunResult("Yandex", PROVIDER_MISSING_DEPS) + + query = " ".join(part for part in (book.author, book.title) if part) + url = f"https://ya.ru/search/?text={httpx.QueryParams({'text': query})['text']}" + del client + + try: + async with async_playwright() as pw: + browser = await pw.chromium.launch( + headless=True, + args=["--disable-blink-features=AutomationControlled"], + ) + try: + context = await browser.new_context( + user_agent=yandex.YANDEX_USER_AGENT, + locale="ru-RU", + timezone_id="Europe/Moscow", + viewport={"width": 1280, "height": 900}, + ) + await Stealth().apply_stealth_async(context) + page = await context.new_page() + await page.goto(url, wait_until="domcontentloaded", timeout=yandex.YANDEX_NAV_TIMEOUT_MS) + page_title = await page.title() + page_html = await page.content() + if yandex._looks_like_captcha(page_title, page_html): + return ProviderRunResult("Yandex", PROVIDER_NOT_FOUND, detail="captcha") + titles = (await page.locator("h2").all_text_contents())[: yandex.YANDEX_H2_LIMIT] + snippets = ( + await page.locator( + "div.OrganicTextContentSpan, .organic__text, .VanillaReact", + ).all_text_contents() + )[: yandex.YANDEX_SNIPPET_LIMIT] + finally: + await browser.close() + isbn = _pick_isbn(page_html) + if not isbn: + return ProviderRunResult("Yandex", PROVIDER_NOT_FOUND) + parsed = yandex._parse_yandex_serp(titles, snippets, isbn) + if parsed is None: + parsed = BookRecord(title=book.title, author=book.author, isbn=isbn) + return ProviderRunResult("Yandex", PROVIDER_FOUND, book=parsed) + except Exception as exc: # noqa: BLE001 + return ProviderRunResult("Yandex", PROVIDER_ERROR, detail=str(exc)) + + +async def _resolve_via_labirint(isbn: str, client: httpx.AsyncClient) -> ProviderRunResult: + """Оборачивает Labirint в формат benchmark-результата.""" + try: + book = await fetch_from_labirint(isbn, client) + if book is None: + return ProviderRunResult("Labirint", PROVIDER_NOT_FOUND) + return ProviderRunResult("Labirint", PROVIDER_FOUND, book=book) + except Exception as exc: # noqa: BLE001 + return ProviderRunResult("Labirint", PROVIDER_ERROR, detail=str(exc)) + + +async def _resolve_via_piter(isbn: str, client: httpx.AsyncClient) -> ProviderRunResult: + """Оборачивает Piter в формат benchmark-результата.""" + try: + book = await fetch_from_piter(isbn, client) + if book is None: + return ProviderRunResult("Piter", PROVIDER_NOT_FOUND) + return ProviderRunResult("Piter", PROVIDER_FOUND, book=book) + except Exception as exc: # noqa: BLE001 + return ProviderRunResult("Piter", PROVIDER_ERROR, detail=str(exc)) + + +async def _resolve_via_google(isbn: str, client: httpx.AsyncClient) -> ProviderRunResult: + """Оборачивает Google Books в формат benchmark-результата.""" + try: + book = await fetch_from_google_books(isbn, client) + if book is None: + return ProviderRunResult("Google Books", PROVIDER_NOT_FOUND) + return ProviderRunResult("Google Books", PROVIDER_FOUND, book=book) + except Exception as exc: # noqa: BLE001 + return ProviderRunResult("Google Books", PROVIDER_ERROR, detail=str(exc)) + + +async def _resolve_via_ddg(isbn: str, client: httpx.AsyncClient) -> ProviderRunResult: + """Разрешает ISBN в метаданные через DuckDuckGo.""" + from home_library.providers.ddg import fetch_from_ddg # noqa: PLC0415 + + try: + book = await fetch_from_ddg(isbn, client) + if book is None: + return ProviderRunResult("DuckDuckGo", PROVIDER_NOT_FOUND) + return ProviderRunResult("DuckDuckGo", PROVIDER_FOUND, book=book) + except Exception as exc: # noqa: BLE001 + return ProviderRunResult("DuckDuckGo", PROVIDER_ERROR, detail=str(exc)) + + +async def _resolve_via_yandex(isbn: str, client: httpx.AsyncClient) -> ProviderRunResult: + """Разрешает ISBN в метаданные через Yandex.""" + unavailable = _yandex_precheck() + if unavailable is not None: + return ProviderRunResult("Yandex", unavailable) + try: + book = await yandex.fetch_from_yandex(isbn, client) + if book is None: + return ProviderRunResult("Yandex", PROVIDER_NOT_FOUND) + return ProviderRunResult("Yandex", PROVIDER_FOUND, book=book) + except Exception as exc: # noqa: BLE001 + return ProviderRunResult("Yandex", PROVIDER_ERROR, detail=str(exc)) + + +async def _search_isbn_unsupported(_book: BookRecord, _client: httpx.AsyncClient) -> ProviderRunResult: + """Заглушка для провайдеров, поддерживаемых только в resolve benchmark.""" + return ProviderRunResult("unsupported", PROVIDER_DISABLED) + + +async def _resolve_playwright_provider( + provider_name: str, + fetcher: Callable[[str, httpx.AsyncClient], asyncio.Future], + isbn: str, + client: httpx.AsyncClient, +) -> ProviderRunResult: + """Общий wrapper для Playwright-only benchmark providers.""" + unavailable = _playwright_precheck() + if unavailable is not None: + return ProviderRunResult(provider_name, unavailable) + try: + book = await fetcher(isbn, client) + if book is None: + return ProviderRunResult(provider_name, PROVIDER_NOT_FOUND) + return ProviderRunResult(provider_name, PROVIDER_FOUND, book=book) + except Exception as exc: # noqa: BLE001 + return ProviderRunResult(provider_name, PROVIDER_ERROR, detail=str(exc)) + + +async def _resolve_via_google_search(isbn: str, client: httpx.AsyncClient) -> ProviderRunResult: + return await _resolve_playwright_provider("Google Search", fetch_from_google_search, isbn, client) + + +async def _resolve_via_brave_search(isbn: str, client: httpx.AsyncClient) -> ProviderRunResult: + return await _resolve_playwright_provider("Brave Search", fetch_from_brave_search, isbn, client) + + +async def _resolve_via_litres(isbn: str, client: httpx.AsyncClient) -> ProviderRunResult: + return await _resolve_playwright_provider("Litres", fetch_from_litres, isbn, client) + + +async def _resolve_via_ozon(isbn: str, client: httpx.AsyncClient) -> ProviderRunResult: + return await _resolve_playwright_provider("Ozon", fetch_from_ozon, isbn, client) + + +async def _resolve_via_wildberries(isbn: str, client: httpx.AsyncClient) -> ProviderRunResult: + return await _resolve_playwright_provider("Wildberries", fetch_from_wildberries, isbn, client) + + +async def _resolve_via_yandex_books(isbn: str, client: httpx.AsyncClient) -> ProviderRunResult: + return await _resolve_playwright_provider("Yandex Books", fetch_from_yandex_books, isbn, client) + + +async def _resolve_via_mybook(isbn: str, client: httpx.AsyncClient) -> ProviderRunResult: + return await _resolve_playwright_provider("MyBook", fetch_from_mybook, isbn, client) + + +def get_benchmark_providers() -> list[BenchmarkProvider]: + """Возвращает реестр провайдеров для benchmark-CLI.""" + return [ + BenchmarkProvider("Labirint", _search_isbn_via_labirint, _resolve_via_labirint), + BenchmarkProvider("Piter", _search_isbn_via_piter, _resolve_via_piter), + BenchmarkProvider("Google Books", _search_isbn_via_google, _resolve_via_google), + BenchmarkProvider("Yandex", _search_isbn_via_yandex, _resolve_via_yandex), + BenchmarkProvider("DuckDuckGo", _search_isbn_via_ddg, _resolve_via_ddg), + BenchmarkProvider( + "Google Search", _search_isbn_unsupported, _resolve_via_google_search, supports_search_isbn=False + ), + BenchmarkProvider( + "Brave Search", _search_isbn_unsupported, _resolve_via_brave_search, supports_search_isbn=False + ), + BenchmarkProvider("Litres", _search_isbn_unsupported, _resolve_via_litres, supports_search_isbn=False), + BenchmarkProvider("Ozon", _search_isbn_unsupported, _resolve_via_ozon, supports_search_isbn=False), + BenchmarkProvider( + "Wildberries", _search_isbn_unsupported, _resolve_via_wildberries, supports_search_isbn=False + ), + BenchmarkProvider( + "Yandex Books", _search_isbn_unsupported, _resolve_via_yandex_books, supports_search_isbn=False + ), + BenchmarkProvider("MyBook", _search_isbn_unsupported, _resolve_via_mybook, supports_search_isbn=False), + ] + + +def _format_percent(value: float) -> str: + """Форматирует долю в проценты.""" + return f"{value * 100:.1f}%" + + +def _render_table(headers: list[str], rows: list[list[object]]) -> str: + """Рендерит ASCII-таблицу.""" + string_rows = [[str(cell) for cell in row] for row in rows] + widths = [len(header) for header in headers] + for row in string_rows: + for index, cell in enumerate(row): + widths[index] = max(widths[index], len(cell)) + + def render_row(cells: list[str]) -> str: + return "| " + " | ".join(cell.ljust(widths[index]) for index, cell in enumerate(cells)) + " |" + + divider = "+-" + "-+-".join("-" * width for width in widths) + "-+" + lines = [divider, render_row(headers), divider] + lines.extend(render_row(row) for row in string_rows) + lines.append(divider) + return "\n".join(lines) + + +def _build_search_rows(stats: list[SearchIsbnStats]) -> list[list[object]]: + """Собирает строки таблицы для режима search-isbn.""" + ordered = sorted(stats, key=lambda item: (-item.verified, -item.raw_found, item.provider_name)) + return [ + [ + item.provider_name, + item.raw_found, + item.verified, + _format_percent(item.precision), + item.not_found, + item.disabled, + item.missing_deps, + item.errors, + ] + for item in ordered + ] + + +def _build_resolve_rows(stats: list[ResolveMetadataStats]) -> list[list[object]]: + """Собирает строки таблицы для режима resolve-metadata.""" + ordered = sorted(stats, key=lambda item: (-item.full_match, -item.coverage, item.provider_name)) + return [ + [ + item.provider_name, + item.coverage, + item.title_match, + item.author_match, + item.full_match, + item.not_found, + item.disabled, + item.missing_deps, + item.errors, + ] + for item in ordered + ] + + +async def _run_search_isbn_benchmark( + books: list[BookRecord], providers: list[BenchmarkProvider] +) -> list[SearchIsbnStats]: + """Запускает benchmark поиска ISBN.""" + stats = {provider.name: SearchIsbnStats(provider_name=provider.name) for provider in providers} + semaphore = asyncio.Semaphore(BENCHMARK_CONCURRENCY) + + async def process( + provider: BenchmarkProvider, + book: BookRecord, + client: httpx.AsyncClient, + ) -> tuple[str, ProviderRunResult, BookRecord]: + async with semaphore: + return provider.name, await provider.search_isbn(book, client), book + + async with httpx.AsyncClient(timeout=20) as client: + tasks = [process(provider, book, client) for book in books for provider in providers] + for provider_name, result, book in await asyncio.gather(*tasks): + item = stats[provider_name] + if result.status == PROVIDER_FOUND: + item.raw_found += 1 + if ( + result.book + and _title_matches(book.title, result.book.title) + and _author_matches(book.author, result.book.author) + ): + item.verified += 1 + elif result.status == PROVIDER_NOT_FOUND: + item.not_found += 1 + elif result.status == PROVIDER_DISABLED: + item.disabled += 1 + elif result.status == PROVIDER_MISSING_DEPS: + item.missing_deps += 1 + else: + item.errors += 1 + return list(stats.values()) + + +async def _run_resolve_metadata_benchmark( + books: list[BookRecord], providers: list[BenchmarkProvider] +) -> list[ResolveMetadataStats]: + """Запускает benchmark обратного поиска по ISBN.""" + stats = {provider.name: ResolveMetadataStats(provider_name=provider.name) for provider in providers} + semaphore = asyncio.Semaphore(BENCHMARK_CONCURRENCY) + + async def process( + provider: BenchmarkProvider, + book: BookRecord, + client: httpx.AsyncClient, + ) -> tuple[str, ProviderRunResult, BookRecord]: + async with semaphore: + return provider.name, await provider.resolve_metadata(book.isbn, client), book + + async with httpx.AsyncClient(timeout=20) as client: + tasks = [process(provider, book, client) for book in books if book.isbn for provider in providers] + for provider_name, result, book in await asyncio.gather(*tasks): + item = stats[provider_name] + if result.status == PROVIDER_FOUND and result.book: + item.coverage += 1 + title_match = _title_matches(book.title, result.book.title) + author_match = _author_matches(book.author, result.book.author) + item.title_match += int(title_match) + item.author_match += int(author_match) + item.full_match += int(title_match and author_match) + elif result.status == PROVIDER_NOT_FOUND: + item.not_found += 1 + elif result.status == PROVIDER_DISABLED: + item.disabled += 1 + elif result.status == PROVIDER_MISSING_DEPS: + item.missing_deps += 1 + else: + item.errors += 1 + return list(stats.values()) + + +def _parse_args() -> argparse.Namespace: + """Парсит аргументы CLI.""" + parser = argparse.ArgumentParser(description="Provider benchmark for Home Library") + parser.add_argument( + "--mode", + choices=["search-isbn", "resolve-metadata", "all"], + default="all", + help="Какой benchmark запускать", + ) + parser.add_argument("--limit", type=int, default=0, help="Ограничить число книг для прогона") + parser.add_argument( + "--providers", + nargs="*", + default=[], + help="Список провайдеров для запуска. По умолчанию запускаются все.", + ) + return parser.parse_args() + + +def _filter_books(mode: str, limit: int) -> list[BookRecord]: + """Возвращает книги для выбранного режима.""" + books = get_all_books() + if mode == "search-isbn": + filtered = [book for book in books if not book.isbn] + elif mode == "resolve-metadata": + filtered = [book for book in books if book.isbn] + else: + filtered = books + if limit > 0: + return filtered[:limit] + return filtered + + +def _filter_providers(selected_names: list[str]) -> list[BenchmarkProvider]: + """Оставляет только выбранные провайдеры.""" + providers = get_benchmark_providers() + if not selected_names: + return providers + selected = {name.casefold() for name in selected_names} + return [provider for provider in providers if provider.name.casefold() in selected] + + +def _providers_for_mode(mode: str, providers: list[BenchmarkProvider]) -> list[BenchmarkProvider]: + """Фильтрует провайдеры по поддерживаемому режиму benchmark.""" + if mode == "search-isbn": + return [provider for provider in providers if provider.supports_search_isbn] + if mode == "resolve-metadata": + return [provider for provider in providers if provider.supports_resolve_metadata] + return providers + + +async def _main_async() -> int: + """Основной async-вход для CLI.""" + args = _parse_args() + providers = _filter_providers(args.providers) + if not providers: + raise SystemExit("Не выбрано ни одного провайдера") + + sections: list[str] = [] + if args.mode in {"search-isbn", "all"}: + search_books = _filter_books("search-isbn", args.limit) + search_providers = _providers_for_mode("search-isbn", providers) + search_stats = await _run_search_isbn_benchmark(search_books, search_providers) + sections.append(f"Search ISBN Benchmark ({len(search_books)} books)") + sections.append( + _render_table( + ["Provider", "Raw Found", "Verified", "Precision", "Not Found", "Disabled", "Missing Deps", "Errors"], + _build_search_rows(search_stats), + ), + ) + + if args.mode in {"resolve-metadata", "all"}: + resolve_books = _filter_books("resolve-metadata", args.limit) + resolve_providers = _providers_for_mode("resolve-metadata", providers) + resolve_stats = await _run_resolve_metadata_benchmark(resolve_books, resolve_providers) + sections.append(f"Resolve Metadata Benchmark ({len(resolve_books)} books)") + sections.append( + _render_table( + [ + "Provider", + "Coverage", + "Title Match", + "Author Match", + "Full Match", + "Not Found", + "Disabled", + "Missing Deps", + "Errors", + ], + _build_resolve_rows(resolve_stats), + ), + ) + + print("\n\n".join(sections)) + return 0 + + +def main() -> int: + """CLI entrypoint.""" + logging.basicConfig(level=logging.WARNING) + return asyncio.run(_main_async()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/home_library/providers/_playwright_fetch.py b/home_library/providers/_playwright_fetch.py new file mode 100644 index 0000000..e48a827 --- /dev/null +++ b/home_library/providers/_playwright_fetch.py @@ -0,0 +1,262 @@ +"""Общие helper-функции Playwright для benchmark-only провайдеров.""" + +from __future__ import annotations + +import asyncio +import contextlib +import html +import importlib.util +import json +import re +from dataclasses import dataclass +from typing import Any +from urllib.parse import quote, urljoin + +from home_library.domain.models import BookRecord + +PLAYWRIGHT_USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" +) +PLAYWRIGHT_NAV_TIMEOUT_MS = 30_000 +PLAYWRIGHT_NETWORK_IDLE_TIMEOUT_MS = 10_000 +PLAYWRIGHT_SETTLE_SECONDS = 1 + +JSON_LD_PATTERN = re.compile( + r']+type=["\']application/ld\+json["\'][^>]*>(?P.*?)', + re.DOTALL | re.IGNORECASE, +) +META_PATTERN = re.compile( + r']+(?:property|name)=["\'](?P[^"\']+)["\'][^>]+content=["\'](?P[^"\']*)["\'][^>]*>', + re.IGNORECASE, +) +TITLE_PATTERN = re.compile(r"(?P<title>.*?)", re.DOTALL | re.IGNORECASE) +HREF_PATTERN = re.compile(r'href=["\'](?P[^"\']+)["\']', re.IGNORECASE) +SITE_SUFFIX_PATTERN = re.compile( + r"\s*(?:\||-|:|—)\s*(?:Google Search|Brave Search|Литрес|ЛитРес|Ozon|Wildberries|Яндекс Книги|MyBook|Яндекс)\s*$", + re.IGNORECASE, +) +AUTHOR_DESCRIPTION_PATTERN = re.compile( + r"(?:автор|author)[:\s]+(?P[A-ZА-ЯЁ][^.;|]{2,120})", + re.IGNORECASE, +) + + +@dataclass(slots=True) +class PlaywrightPageSnapshot: + """Снимок страницы, достаточный для грубого парсинга книги.""" + + url: str + title: str + html: str + headings: list[str] + snippets: list[str] + description: str = "" + + +def playwright_available() -> bool: + """Проверяет наличие playwright и playwright-stealth.""" + try: + playwright_spec = importlib.util.find_spec("playwright") + stealth_spec = importlib.util.find_spec("playwright_stealth") + except ModuleNotFoundError: + return False + return playwright_spec is not None and stealth_spec is not None + + +async def fetch_page_snapshot( + url: str, + *, + heading_selectors: tuple[str, ...] = ("h1", "h2", "h3"), + snippet_selectors: tuple[str, ...] = (), + wait_selector: str | None = None, +) -> PlaywrightPageSnapshot | None: + """Открывает страницу в Playwright и возвращает её снимок.""" + if not playwright_available(): + return None + + from playwright.async_api import TimeoutError as PlaywrightTimeoutError # noqa: PLC0415 + from playwright.async_api import async_playwright # noqa: PLC0415 + from playwright_stealth import Stealth # noqa: PLC0415 + + async with async_playwright() as pw: + browser = await pw.chromium.launch( + headless=True, + args=["--disable-blink-features=AutomationControlled"], + ) + try: + context = await browser.new_context( + user_agent=PLAYWRIGHT_USER_AGENT, + locale="ru-RU", + timezone_id="Europe/Moscow", + viewport={"width": 1280, "height": 900}, + ) + await Stealth().apply_stealth_async(context) + page = await context.new_page() + await page.goto(url, wait_until="domcontentloaded", timeout=PLAYWRIGHT_NAV_TIMEOUT_MS) + with contextlib.suppress(PlaywrightTimeoutError): + await page.wait_for_load_state("networkidle", timeout=PLAYWRIGHT_NETWORK_IDLE_TIMEOUT_MS) + if wait_selector: + with contextlib.suppress(PlaywrightTimeoutError): + await page.locator(wait_selector).first.wait_for(timeout=5_000) + await asyncio.sleep(PLAYWRIGHT_SETTLE_SECONDS) + + title = await page.title() + html_text = await page.content() + headings: list[str] = [] + for selector in heading_selectors: + headings.extend(await page.locator(selector).all_text_contents()) + snippets: list[str] = [] + for selector in snippet_selectors: + snippets.extend(await page.locator(selector).all_text_contents()) + return PlaywrightPageSnapshot( + url=page.url, + title=title, + html=html_text, + headings=[item.strip() for item in headings if item.strip()], + snippets=[item.strip() for item in snippets if item.strip()], + description=_extract_meta_content(html_text, "description"), + ) + finally: + await browser.close() + + +def build_search_url(base: str, query: str) -> str: + """Собирает URL поиска с URL-encoded query.""" + return base.format(query=quote(query), query_plus=quote(query, safe="")) + + +def find_first_matching_link( + snapshot: PlaywrightPageSnapshot, *, base_url: str, patterns: tuple[str, ...] +) -> str | None: + """Возвращает первую ссылку, подходящую под паттерны.""" + for href in HREF_PATTERN.findall(snapshot.html): + normalized_href = html.unescape(href) + absolute = urljoin(base_url, normalized_href) + for pattern in patterns: + if re.search(pattern, absolute, re.IGNORECASE): + return absolute + return None + + +def _extract_meta_content(html_text: str, meta_name: str) -> str: + for match in META_PATTERN.finditer(html_text): + if match.group("name").casefold() == meta_name.casefold(): + return html.unescape(match.group("content")).strip() + return "" + + +def _normalize_author_value(value: Any) -> str: + if isinstance(value, str): + return value.strip() + if isinstance(value, dict): + for key in ("name", "title"): + if isinstance(value.get(key), str): + return value[key].strip() + if isinstance(value, list): + authors = [_normalize_author_value(item) for item in value] + return ", ".join(item for item in authors if item) + return "" + + +def _collect_json_ld_candidates(value: Any) -> list[BookRecord]: + candidates: list[BookRecord] = [] + if isinstance(value, list): + for item in value: + candidates.extend(_collect_json_ld_candidates(item)) + return candidates + + if not isinstance(value, dict): + return candidates + + title = "" + for key in ("name", "headline", "title"): + if isinstance(value.get(key), str): + title = value[key].strip() + break + + author = _normalize_author_value(value.get("author")) + url = value.get("url") if isinstance(value.get("url"), str) else "" + publisher = _normalize_author_value(value.get("publisher")) + if title: + candidates.append(BookRecord(title=title, author=author, publisher=publisher, link=url)) + + for nested in value.values(): + candidates.extend(_collect_json_ld_candidates(nested)) + return candidates + + +def extract_book_candidates_from_html(html_text: str) -> list[BookRecord]: + """Пытается извлечь кандидатов книги из JSON-LD и meta-тегов HTML.""" + candidates: list[BookRecord] = [] + for match in JSON_LD_PATTERN.finditer(html_text): + body = html.unescape(match.group("body")).strip() + if not body: + continue + try: + data = json.loads(body) + except json.JSONDecodeError: + continue + candidates.extend(_collect_json_ld_candidates(data)) + + og_title = _extract_meta_content(html_text, "og:title") + description = _extract_meta_content(html_text, "description") or _extract_meta_content(html_text, "og:description") + if og_title: + author_match = AUTHOR_DESCRIPTION_PATTERN.search(description) + candidates.append( + BookRecord( + title=og_title, + author=author_match.group("author").strip() if author_match else "", + ), + ) + return candidates + + +def clean_provider_title(title: str) -> str: + """Очищает title/meta title от хвостов сайта.""" + return SITE_SUFFIX_PATTERN.sub("", title).strip(" |:-—") + + +def choose_best_book_candidate( + candidates: list[BookRecord], + *, + isbn: str, + fallback_title: str = "", + fallback_author: str = "", + fallback_link: str = "", +) -> BookRecord | None: + """Выбирает лучший кандидат из HTML-метаданных.""" + normalized: list[BookRecord] = [] + for candidate in candidates: + title = clean_provider_title(candidate.title) + author = candidate.author.strip() + if title: + normalized.append( + BookRecord( + title=title, + author=author, + publisher=candidate.publisher, + link=candidate.link or fallback_link, + isbn=isbn, + ), + ) + if normalized: + normalized.sort(key=lambda item: (bool(item.author), len(item.title), len(item.author)), reverse=True) + return normalized[0] + if fallback_title: + return BookRecord( + title=clean_provider_title(fallback_title), author=fallback_author, link=fallback_link, isbn=isbn + ) + return None + + +def extract_book_from_snapshot(snapshot: PlaywrightPageSnapshot, *, isbn: str) -> BookRecord | None: + """Пытается извлечь книгу из snapshot страницы.""" + candidates = extract_book_candidates_from_html(snapshot.html) + return choose_best_book_candidate( + candidates, + isbn=isbn, + fallback_title=snapshot.title, + fallback_author="", + fallback_link=snapshot.url, + ) diff --git a/home_library/providers/brave_search.py b/home_library/providers/brave_search.py new file mode 100644 index 0000000..83bd039 --- /dev/null +++ b/home_library/providers/brave_search.py @@ -0,0 +1,24 @@ +"""Brave Search Playwright provider for benchmark-only ISBN lookup.""" + +from __future__ import annotations + +import httpx + +from home_library.providers._playwright_fetch import build_search_url +from home_library.providers._playwright_fetch import fetch_page_snapshot +from home_library.providers._serp_parser import extract_book_from_serp + +BRAVE_SEARCH_URL = "https://search.brave.com/search?q={query_plus}&source=web" + + +async def fetch_from_brave_search(isbn: str, _client: httpx.AsyncClient) -> object | None: + """Ищет книгу по ISBN через Brave Search SERP.""" + del _client + snapshot = await fetch_page_snapshot( + build_search_url(BRAVE_SEARCH_URL, f"ISBN {isbn}"), + heading_selectors=("h2", "h3"), + snippet_selectors=("p.snippet", "div.snippet", "div[slot=description]"), + ) + if snapshot is None: + return None + return extract_book_from_serp(snapshot.headings, isbn, snippets=snapshot.snippets) diff --git a/home_library/providers/google_search.py b/home_library/providers/google_search.py new file mode 100644 index 0000000..5be3a59 --- /dev/null +++ b/home_library/providers/google_search.py @@ -0,0 +1,24 @@ +"""Google Search Playwright provider for benchmark-only ISBN lookup.""" + +from __future__ import annotations + +import httpx + +from home_library.providers._playwright_fetch import build_search_url +from home_library.providers._playwright_fetch import fetch_page_snapshot +from home_library.providers._serp_parser import extract_book_from_serp + +GOOGLE_SEARCH_URL = "https://www.google.com/search?q={query_plus}&hl=ru" + + +async def fetch_from_google_search(isbn: str, _client: httpx.AsyncClient) -> object | None: + """Ищет книгу по ISBN через Google Search SERP.""" + del _client + snapshot = await fetch_page_snapshot( + build_search_url(GOOGLE_SEARCH_URL, f"ISBN {isbn}"), + heading_selectors=("h3",), + snippet_selectors=("div.VwiC3b", "div[data-sncf='1']"), + ) + if snapshot is None: + return None + return extract_book_from_serp(snapshot.headings, isbn, snippets=snapshot.snippets) diff --git a/home_library/providers/litres.py b/home_library/providers/litres.py new file mode 100644 index 0000000..e4615b6 --- /dev/null +++ b/home_library/providers/litres.py @@ -0,0 +1,24 @@ +"""Litres Playwright provider for benchmark-only ISBN lookup.""" + +from __future__ import annotations + +import httpx + +from home_library.providers._playwright_fetch import fetch_page_snapshot +from home_library.providers.yandex import _parse_yandex_serp + +LITRES_SEARCH_URL = "https://ya.ru/search/?text=site%3Alitres.ru%20ISBN%20{isbn}" + + +async def fetch_from_litres(isbn: str, _client: httpx.AsyncClient) -> object | None: + """Ищет книгу по ISBN через Литрес.""" + del _client + snapshot = await fetch_page_snapshot( + LITRES_SEARCH_URL.format(isbn=isbn), + heading_selectors=("h2",), + snippet_selectors=("div.OrganicTextContentSpan", ".organic__text", ".VanillaReact"), + wait_selector="body", + ) + if snapshot is None: + return None + return _parse_yandex_serp(snapshot.headings, snapshot.snippets, isbn) diff --git a/home_library/providers/mybook.py b/home_library/providers/mybook.py new file mode 100644 index 0000000..630b624 --- /dev/null +++ b/home_library/providers/mybook.py @@ -0,0 +1,24 @@ +"""MyBook Playwright provider for benchmark-only ISBN lookup.""" + +from __future__ import annotations + +import httpx + +from home_library.providers._playwright_fetch import fetch_page_snapshot +from home_library.providers.yandex import _parse_yandex_serp + +MYBOOK_SEARCH_URL = "https://ya.ru/search/?text=site%3Amybook.ru%20ISBN%20{isbn}" + + +async def fetch_from_mybook(isbn: str, _client: httpx.AsyncClient) -> object | None: + """Ищет книгу по ISBN через MyBook.""" + del _client + snapshot = await fetch_page_snapshot( + MYBOOK_SEARCH_URL.format(isbn=isbn), + heading_selectors=("h2",), + snippet_selectors=("div.OrganicTextContentSpan", ".organic__text", ".VanillaReact"), + wait_selector="body", + ) + if snapshot is None: + return None + return _parse_yandex_serp(snapshot.headings, snapshot.snippets, isbn) diff --git a/home_library/providers/ozon.py b/home_library/providers/ozon.py new file mode 100644 index 0000000..5218561 --- /dev/null +++ b/home_library/providers/ozon.py @@ -0,0 +1,24 @@ +"""Ozon Playwright provider for benchmark-only ISBN lookup.""" + +from __future__ import annotations + +import httpx + +from home_library.providers._playwright_fetch import fetch_page_snapshot +from home_library.providers.yandex import _parse_yandex_serp + +OZON_SEARCH_URL = "https://ya.ru/search/?text=site%3Aozon.ru%20ISBN%20{isbn}" + + +async def fetch_from_ozon(isbn: str, _client: httpx.AsyncClient) -> object | None: + """Ищет книгу по ISBN через Ozon.""" + del _client + snapshot = await fetch_page_snapshot( + OZON_SEARCH_URL.format(isbn=isbn), + heading_selectors=("h2",), + snippet_selectors=("div.OrganicTextContentSpan", ".organic__text", ".VanillaReact"), + wait_selector="body", + ) + if snapshot is None: + return None + return _parse_yandex_serp(snapshot.headings, snapshot.snippets, isbn) diff --git a/home_library/providers/wildberries.py b/home_library/providers/wildberries.py new file mode 100644 index 0000000..17bc4c0 --- /dev/null +++ b/home_library/providers/wildberries.py @@ -0,0 +1,24 @@ +"""Wildberries Playwright provider for benchmark-only ISBN lookup.""" + +from __future__ import annotations + +import httpx + +from home_library.providers._playwright_fetch import fetch_page_snapshot +from home_library.providers.yandex import _parse_yandex_serp + +WILDBERRIES_SEARCH_URL = "https://ya.ru/search/?text=site%3Awildberries.ru%20ISBN%20{isbn}" + + +async def fetch_from_wildberries(isbn: str, _client: httpx.AsyncClient) -> object | None: + """Ищет книгу по ISBN через Wildberries.""" + del _client + snapshot = await fetch_page_snapshot( + WILDBERRIES_SEARCH_URL.format(isbn=isbn), + heading_selectors=("h2",), + snippet_selectors=("div.OrganicTextContentSpan", ".organic__text", ".VanillaReact"), + wait_selector="body", + ) + if snapshot is None: + return None + return _parse_yandex_serp(snapshot.headings, snapshot.snippets, isbn) diff --git a/home_library/providers/yandex_books.py b/home_library/providers/yandex_books.py new file mode 100644 index 0000000..8f1f683 --- /dev/null +++ b/home_library/providers/yandex_books.py @@ -0,0 +1,23 @@ +"""Yandex Books benchmark-only provider via Yandex site search.""" + +from __future__ import annotations + +import httpx + +from home_library.providers._playwright_fetch import fetch_page_snapshot +from home_library.providers.yandex import _parse_yandex_serp + +YANDEX_BOOKS_SEARCH_URL = "https://ya.ru/search/?text=site%3Abooks.yandex.ru%20ISBN%20{isbn}" + + +async def fetch_from_yandex_books(isbn: str, _client: httpx.AsyncClient) -> object | None: + """Ищет книгу по ISBN по домену Яндекс Книг через Yandex Search.""" + del _client + snapshot = await fetch_page_snapshot( + YANDEX_BOOKS_SEARCH_URL.format(isbn=isbn), + heading_selectors=("h2",), + snippet_selectors=("div.OrganicTextContentSpan", ".organic__text", ".VanillaReact"), + ) + if snapshot is None: + return None + return _parse_yandex_serp(snapshot.headings, snapshot.snippets, isbn) diff --git a/tests/test_home_library_benchmark.py b/tests/test_home_library_benchmark.py new file mode 100644 index 0000000..88892a7 --- /dev/null +++ b/tests/test_home_library_benchmark.py @@ -0,0 +1,121 @@ +"""Tests for benchmark CLI helpers.""" + +from home_library.benchmark import ( + PROVIDER_DISABLED, + PROVIDER_MISSING_DEPS, + SearchIsbnStats, + _author_matches, + _build_resolve_rows, + _build_search_rows, + _normalize_text, + _pick_isbn, + _render_table, + _title_matches, + _yandex_precheck, + get_benchmark_providers, +) + + +def test_get_benchmark_providers_contains_yandex() -> None: + names = [provider.name for provider in get_benchmark_providers()] + assert names == [ + "Labirint", + "Piter", + "Google Books", + "Yandex", + "DuckDuckGo", + "Google Search", + "Brave Search", + "Litres", + "Ozon", + "Wildberries", + "Yandex Books", + "MyBook", + ] + + +def test_normalize_text_handles_yo_and_punctuation() -> None: + assert _normalize_text(" Ёжик, в тумане! ") == "ежик в тумане" + + +def test_title_matches_by_containment() -> None: + assert _title_matches( + "Когнитивно-поведенческая терапия", "Когнитивно-поведенческая терапия. От основ к направлениям" + ) + + +def test_author_matches_by_token_overlap() -> None: + assert _author_matches("Владимир Хориков", "Хориков В.") + + +def test_pick_isbn_prefers_13_digit_candidate() -> None: + text = "ISBN 5446116836 и ISBN-13 9785446116836" + assert _pick_isbn(text) == "9785446116836" + + +def test_render_table_renders_ascii_table() -> None: + table = _render_table(["A", "B"], [[1, 2], [10, 20]]) + assert "+" in table + assert "| A" in table + assert "| 10" in table + + +def test_build_search_rows_sorted_by_verified_then_raw() -> None: + rows = _build_search_rows( + [ + SearchIsbnStats(provider_name="B", raw_found=10, verified=4), + SearchIsbnStats(provider_name="A", raw_found=9, verified=5), + ], + ) + assert rows[0][0] == "A" + assert rows[1][0] == "B" + + +def test_build_resolve_rows_sorted_by_full_match_then_coverage() -> None: + rows = _build_resolve_rows( + [ + type( + "Stats", + (), + { + "provider_name": "B", + "coverage": 8, + "title_match": 8, + "author_match": 8, + "full_match": 6, + "not_found": 0, + "disabled": 0, + "missing_deps": 0, + "errors": 0, + }, + )(), + type( + "Stats", + (), + { + "provider_name": "A", + "coverage": 7, + "title_match": 7, + "author_match": 7, + "full_match": 7, + "not_found": 0, + "disabled": 0, + "missing_deps": 0, + "errors": 0, + }, + )(), + ], + ) + assert rows[0][0] == "A" + assert rows[1][0] == "B" + + +def test_yandex_precheck_disabled_without_env(monkeypatch) -> None: + monkeypatch.delenv("YANDEX_ENABLED", raising=False) + assert _yandex_precheck() == PROVIDER_DISABLED + + +def test_yandex_precheck_missing_deps(monkeypatch) -> None: + monkeypatch.setenv("YANDEX_ENABLED", "1") + monkeypatch.setattr("importlib.util.find_spec", lambda _name: None) + assert _yandex_precheck() == PROVIDER_MISSING_DEPS diff --git a/tests/test_home_library_playwright_providers.py b/tests/test_home_library_playwright_providers.py new file mode 100644 index 0000000..1770d0e --- /dev/null +++ b/tests/test_home_library_playwright_providers.py @@ -0,0 +1,39 @@ +"""Tests for Playwright benchmark provider helpers.""" + +from home_library.providers._playwright_fetch import choose_best_book_candidate +from home_library.providers._playwright_fetch import clean_provider_title +from home_library.providers._playwright_fetch import extract_book_candidates_from_html + + +def test_extract_book_candidates_from_html_json_ld() -> None: + html = """ + + + + + + """ + candidates = extract_book_candidates_from_html(html) + assert candidates + assert candidates[0].title == "Принципы юнит-тестирования" + assert candidates[0].author == "Владимир Хориков" + + +def test_clean_provider_title_strips_site_suffix() -> None: + assert clean_provider_title("Книга - Ozon") == "Книга" + + +def test_choose_best_book_candidate_prefers_author() -> None: + candidates = extract_book_candidates_from_html( + """ + + + + """, + ) + book = choose_best_book_candidate(candidates, isbn="9785446116836") + assert book is not None + assert book.title == "Книга" + assert book.author == "Иван Иванов"