diff --git a/scrapers/JavLibrary_python/JavLibrary_python.py b/scrapers/JavLibrary_python/JavLibrary_python.py index ce07ee6a1..5cf9a27ba 100644 --- a/scrapers/JavLibrary_python/JavLibrary_python.py +++ b/scrapers/JavLibrary_python/JavLibrary_python.py @@ -41,7 +41,7 @@ PROTECTION_CLOUDFLARE = False # Flaresolverr -FLARESOLVERR_ENABLED = False +FLARESOLVERR_ENABLED = True FLARESOLVERR_URL = "http://localhost:8191/v1" FLARESOLVERR_TIMEOUT_MAX = 60000 @@ -62,13 +62,15 @@ "Featured Actress", "VR Exclusive", "MOODYZ SALE 4" ] # Select preferable name order +# Site language: en, ja, tw, cn +LANGUAGE = "ja" NAME_ORDER_JAPANESE = False # Some performers don't need to be reversed IGNORE_PERF_REVERSE = ["Lily Heart"] # Keep the legacy field scheme: # Actual Code -> Title, actual Title -> Details, actual Details -> /dev/null -LEGACY_FIELDS = True +LEGACY_FIELDS = False # Studio Code now in a separate field, so it may (or may not) be stripped from title # Makes sense only if not LEGACY_FIELDS KEEP_CODE_IN_TITLE = True @@ -421,7 +423,7 @@ class ResponseHTML: def bypass_protection(url, retries=4): url_domain = re.sub(r"www\.|\.com", "", urlparse(url).netloc) log.debug("=== Checking Status of Javlib site ===") - response_html = ResponseHTML + response_html = ResponseHTML() site = "javlibrary" url_n = url.replace(url_domain, site) try: @@ -430,7 +432,7 @@ def bypass_protection(url, retries=4): headers = {"Content-Type": "application/json"} data = { "cmd": "request.get", - "url": url_n,"" + "url": url_n, "session": "2", "session_ttl_minutes": 120, "maxTimeout": FLARESOLVERR_TIMEOUT_MAX, @@ -443,18 +445,16 @@ def bypass_protection(url, retries=4): responseJson = requests.post(FLARESOLVERR_URL, cookies=cookies, headers=headers, json=data) json_input = responseJson.json() - response_html.content = json_input['solution']['response'] - response_html.html = json_input['solution']['response'] - response_html.status_code = json_input['solution']['status'] - response_html.url = json_input['solution']['url'] + if json_input.get('status') == 'ok' and 'solution' in json_input: + response_html.content = json_input['solution']['response'] + response_html.html = json_input['solution']['response'] + response_html.status_code = json_input['solution']['status'] + response_html.url = json_input['solution']['url'] + else: + raise Exception(f"FlareSolverr returned error: {json_input.get('message', 'Unknown error')}") #log.info(f"Flaresolverr response html: {response_html}") else: - log.info( - f"[{site}] Using this site for scraping ({response_html.status_code})" - ) - log.debug("======================================") - return site, response_html response = requests.get(url_n, headers=JAV_HEADERS, timeout=10) response_html.content = response.content response_html.html = response.text @@ -463,10 +463,10 @@ def bypass_protection(url, retries=4): except Exception as exc_req: log.warning(f"Exception error {exc_req} while checking protection for {site}") if retries == 4: - retries = retries -1 + retries = retries - 1 log.warning(f"Retrying once normally after 7s delay [retries left: {retries}] for site: {site}") time.sleep(7.2) - bypass_protection(url_n,retries) + return bypass_protection(url, retries) else: return None, None if response_html.url == "https://www.javlib.com/maintenance.html": @@ -499,6 +499,10 @@ def send_request(url, head, retries=0, delay=2.5): response = None if url_domain in SITE_JAVLIB: # Javlib + if FLARESOLVERR_ENABLED: + _, response = bypass_protection(url) + return response + if JAV_DOMAIN == "Check": JAV_DOMAIN, response = bypass_protection(url) if response: @@ -548,6 +552,8 @@ def cleanup_title(title): return title def regexreplace(input_replace): + if not LEGACY_FIELDS: + return "" word_pattern = re.compile(r'(\w|\*)+') output = word_pattern.sub(replace_banned_words, input_replace) return re.sub(r"[\[\]\"]", "", output) @@ -583,14 +589,15 @@ def getxpath(xpath, tree): def jav_search(html, xpath): - if "/en/jav" in html.url: + if f"/{LANGUAGE}/jav" in html.url or "?v=jav" in html.url: log.debug(f"Using the provided movie page ({html.url})") return html jav_search_tree = lxml.html.fromstring(html.content) + jav_url = getxpath(xpath['url'], jav_search_tree) # ./javme5it6a if jav_url: url_domain = urlparse(html.url).netloc - jav_url = re.sub(r"^\.", f"https://{url_domain}/en", jav_url[0]) + jav_url = re.sub(r"^\.", f"https://{url_domain}/{LANGUAGE}", jav_url[0]) log.debug(f"Using API URL: {jav_url}") main_html = send_request(jav_url, JAV_HEADERS) return main_html @@ -626,7 +633,7 @@ def jav_search_by_name(html, xpath): lst.append({ "title": jav_title[count], "url": - f"https://www.javlibrary.com/en/{jav_url[count].replace('./', '')}", + f"https://www.javlibrary.com/{LANGUAGE}/{jav_url[count].replace('./', '')}", "image": re.sub("^//","https://",jav_image[count]) }) log.debug(f"There is/are {len(lst)} scene(s)") @@ -686,7 +693,7 @@ def buildlist_tagperf(data, type_scrape=""): def th_request_perfpage(page_url, perf_url): # vl_star.php?s=afhvw #log.debug("[DEBUG] Aliases Thread: {}".format(threading.get_ident())) - javlibrary_ja_html = send_request(page_url.replace("/en/", "/ja/"), + javlibrary_ja_html = send_request(re.sub(r"/(en|ja|tw|cn)/", "/ja/", page_url), JAV_HEADERS) if javlibrary_ja_html: javlibrary_perf_ja = lxml.html.fromstring(javlibrary_ja_html.content) @@ -757,6 +764,8 @@ def th_imageto_base64(imageurl, typevar): SEARCH_TITLE = FRAGMENT.get("name") SEARCH_TITLE = cleanup_title(SEARCH_TITLE) SCENE_URL = FRAGMENT.get("url") +if SCENE_URL: + SCENE_URL = re.sub(r"/(en|ja|tw|cn)/", f"/{LANGUAGE}/", SCENE_URL) if FRAGMENT.get("title"): SCENE_TITLE = FRAGMENT["title"] @@ -770,7 +779,7 @@ def th_imageto_base64(imageurl, typevar): if "searchName" in sys.argv: log.debug(f"Using search with Title: {SEARCH_TITLE}") JAV_SEARCH_HTML = send_request( - f"https://www.javlibrary.com/en/vl_searchbyid.php?keyword={SEARCH_TITLE}", + f"https://www.javlibrary.com/{LANGUAGE}/vl_searchbyid.php?keyword={SEARCH_TITLE}", JAV_HEADERS) else: if SCENE_URL: @@ -784,7 +793,7 @@ def th_imageto_base64(imageurl, typevar): if JAV_MAIN_HTML is None and SCENE_TITLE: log.debug(f"Using search with Title: {SCENE_TITLE}") JAV_SEARCH_HTML = send_request( - f"https://www.javlibrary.com/en/vl_searchbyid.php?keyword={SCENE_TITLE}", + f"https://www.javlibrary.com/{LANGUAGE}/vl_searchbyid.php?keyword={SCENE_TITLE}", JAV_HEADERS) # XPATH @@ -798,7 +807,7 @@ def th_imageto_base64(imageurl, typevar): jav_xPath = {} jav_xPath[ - "code"] = '//td[@class="header" and text()="ID:"]/following-sibling::td/text()' + "code"] = '//div[@id="video_id"]//td[@class="text"]/text()' # or '//div[@id="video_id"]//td[2][@class="text"]/text()' jav_xPath[ "title"] = jav_xPath["code"] if LEGACY_FIELDS else '//div[@id="video_title"]/h3/a/text()' @@ -808,21 +817,17 @@ def th_imageto_base64(imageurl, typevar): "details"] = None if not LEGACY_FIELDS else '//div[@id="video_title"]/h3/a/text()' jav_xPath["url"] = '//meta[@property="og:url"]/@content' jav_xPath[ - "date"] = '//td[@class="header" and text()="Release Date:"]/following-sibling::td/text()' + "date"] = '//div[@id="video_date"]//td[@class="text"]/text()' jav_xPath[ "director"] = '//div[@id="video_director"]//td[@class="text"]/span[@class="director"]/a/text()' jav_xPath[ - "tags"] = '//td[@class="header" and text()="Genre(s):"]'\ - '/following::td/span[@class="genre"]/a/text()' + "tags"] = '//div[@id="video_genres"]//span[@class="genre"]/a/text()' jav_xPath[ - "performers"] = '//td[@class="header" and text()="Cast:"]'\ - '/following::td/span[@class="cast"]/span/a/text()' + "performers"] = '//div[@id="video_cast"]//span[@class="cast"]/span/a/text()' jav_xPath[ - "performers_url"] = '//td[@class="header" and text()="Cast:"]'\ - '/following::td/span[@class="cast"]/span/a/@href' + "performers_url"] = '//div[@id="video_cast"]//span[@class="cast"]/span/a/@href' jav_xPath[ - "studio"] = '//td[@class="header" and text()="Maker:"]'\ - '/following-sibling::td/span[@class="maker"]/a/text()' + "studio"] = '//div[@id="video_maker"]//span[@class="maker"]/a/text()' #jav_xPath[ # "label"] = '//td[@class="header" and text()="Label:"]'\ # '/following-sibling::td/span[@class="label"]/a/text()' @@ -933,15 +938,20 @@ def th_imageto_base64(imageurl, typevar): scrape = {} # DVD code -scrape['code'] = next(iter(jav_result.get('code', []))) -scrape['title'] = jav_result.get('title') -scrape['date'] = next(iter(jav_result.get('date', []))) -scrape['director'] = jav_result.get('director') or None -scrape['url'] = jav_result.get('url') -scrape['details'] = regexreplace(jav_result.get('details', "")) -scrape['studio'] = { - 'name': next(iter(jav_result.get('studio', []))), -} +try: + scrape['code'] = next(iter(jav_result.get('code', [])), None) + scrape['title'] = jav_result.get('title') + scrape['date'] = next(iter(jav_result.get('date', [])), None) + scrape['director'] = jav_result.get('director') or None + scrape['url'] = jav_result.get('url') + scrape['details'] = regexreplace(jav_result.get('details', "")) + scrape['studio'] = { + 'name': next(iter(jav_result.get('studio', [])), None), + } +except Exception as e: + log.error(f"Error mapping scraped data to fields: {e}") + log.error(f"Raw jav_result dump: {jav_result}") + raise e #scrape['label'] = { # 'name': jav_result.get('label'), #} diff --git a/scrapers/fc2db/fc2db.py b/scrapers/fc2db/fc2db.py new file mode 100644 index 000000000..c7db57b92 --- /dev/null +++ b/scrapers/fc2db/fc2db.py @@ -0,0 +1,430 @@ +"""FC2DB scraper for Stash — scrapes scene and performer data from fc2db.net + +Uses JSON-LD structured data as the primary extraction method, +with HTML XPath fallback for fields not in JSON-LD (tags, mosaic status). +No authentication required. +""" +import json +import os +import re +import sys + +try: + from py_common import log + from py_common.util import scraper_args +except ModuleNotFoundError: + print( + "You need to download the folder 'py_common' from the community repo! " + "(CommunityScrapers/tree/master/scrapers/py_common)", + file=sys.stderr, + ) + sys.exit(1) + +try: + import lxml.html +except ModuleNotFoundError: + print( + "You need to install the lxml module. (pip install lxml)", + file=sys.stderr, + ) + sys.exit(1) + +try: + import requests +except ModuleNotFoundError: + print( + "You need to install the requests module. (pip install requests)", + file=sys.stderr, + ) + sys.exit(1) + + +# Flaresolverr +FLARESOLVERR_URL = "http://localhost:8191/v1" +FLARESOLVERR_TIMEOUT_MAX = 60000 + +BASE_URL = "https://fc2db.net" +HEADERS = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0", +} + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def fetch_page(url: str) -> lxml.html.HtmlElement | None: + """Fetch a page and return the parsed HTML tree, or None on failure.""" + try: + resp = requests.get(url, headers=HEADERS, timeout=15) + + # Check for Cloudflare challenge + if resp.status_code in (403, 503) or "One moment, please..." in resp.text or "Just a moment..." in resp.text: + log.info("Cloudflare challenge detected, attempting to use FlareSolverr...") + headers = {"Content-Type": "application/json"} + data = { + "cmd": "request.get", + "url": url, + "maxTimeout": FLARESOLVERR_TIMEOUT_MAX + } + try: + fs_resp = requests.post(FLARESOLVERR_URL, headers=headers, json=data, timeout=65) + json_input = fs_resp.json() + if json_input.get("status") == "ok" and "solution" in json_input: + return lxml.html.fromstring(json_input["solution"]["response"]) + else: + log.error(f"FlareSolverr returned error: {json_input.get('message', 'Unknown error')}") + return None + except requests.RequestException as e: + log.error(f"Failed to connect to FlareSolverr at {FLARESOLVERR_URL}: {e}") + return None + + resp.raise_for_status() + return lxml.html.fromstring(resp.content) + except requests.RequestException as exc: + log.error(f"Failed to fetch {url}: {exc}") + return None + + +def extract_jsonld(tree: lxml.html.HtmlElement, expected_type: str) -> dict | None: + """Extract the first JSON-LD block matching the expected @type.""" + for script in tree.xpath('//script[@type="application/ld+json"]'): + try: + data = json.loads(script.text_content()) + if data.get("@type") == expected_type: + return data + except (json.JSONDecodeError, TypeError): + continue + return None + + +def duration_to_seconds(duration_str: str) -> int | None: + """Convert HH:MM:SS or MM:SS to total seconds.""" + if not duration_str: + return None + parts = duration_str.strip().split(":") + try: + if len(parts) == 3: + return int(parts[0]) * 3600 + int(parts[1]) * 60 + int(parts[2]) + if len(parts) == 2: + return int(parts[0]) * 60 + int(parts[1]) + except ValueError: + pass + return None + + +def extract_fc2_number(text: str) -> str | None: + """Extract an FC2 number (5+ digits) from a string.""" + match = re.search(r"(\d{5,})", text) + return match.group(1) if match else None + + +def work_url_from_number(number: str) -> str: + """Construct the fc2db work URL from an FC2 number.""" + return f"{BASE_URL}/work/{number}/" + + +# --------------------------------------------------------------------------- +# Scene scraping +# --------------------------------------------------------------------------- + +def scene_from_url(url: str) -> dict: + """Scrape a scene from a fc2db.net work page.""" + tree = fetch_page(url) + if tree is None: + return {} + + scene = {} + + # --- Primary: JSON-LD VideoObject --- + ld = extract_jsonld(tree, "VideoObject") + + if not ld: + raise Exception(f"Failed to find JSON-LD VideoObject for {url}. Cloudflare challenge may have failed, or site layout changed.") + + log.debug(f"Found JSON-LD VideoObject: {json.dumps(ld, ensure_ascii=False)[:200]}") + + # Title: JSON-LD "name" is typically "FC2PPV-NNNNNN" + # Use the full title from

instead, which includes the actual title text + ld_name = ld.get("name", "") + fc2_number = extract_fc2_number(ld_name) + + # Code + if fc2_number: + scene["code"] = f"FC2-PPV-{fc2_number}" + + # Date + upload_date = ld.get("uploadDate", "") + if upload_date: + scene["date"] = upload_date + + # Duration + duration_val = duration_to_seconds(ld.get("duration", "")) + if duration_val: + scene["duration"] = duration_val + + # Cover image + thumbnail = ld.get("thumbnailUrl", "") + if thumbnail: + scene["image"] = thumbnail + + # Performers from JSON-LD actors + actors = ld.get("actor", []) + if isinstance(actors, dict): + actors = [actors] + performers = [] + for actor in actors: + name = actor.get("name", "").strip() + if name: + actor_url = actor.get("url", "") + if actor_url: + # Fetch full performer details (images, aliases) from their profile page + perf = performer_from_url(actor_url) + else: + perf = {"name": name, "gender": "FEMALE"} + performers.append(perf) + if performers: + scene["performers"] = performers + + # Director (seller/publisher) + publisher = ld.get("publisher", {}) + if isinstance(publisher, dict): + pub_name = publisher.get("name", "").strip() + if pub_name: + scene["director"] = pub_name + + # URL from JSON-LD + ld_url = ld.get("url", "") + if ld_url: + scene["urls"] = [ld_url] + + # --- HTML fallback / supplemental --- + + # Full title from

(includes [FC2-PPV-NNNN] prefix + actual title) + h1_elements = tree.xpath('//h1[contains(@class, "text-2xl")]') + if h1_elements: + full_title = h1_elements[0].text_content().strip() + # Strip the [FC2-PPV-NNNN] prefix if present to get the clean title + clean_title = re.sub(r"^\[FC2-PPV-\d+\]\s*", "", full_title).strip() + if clean_title: + scene["title"] = clean_title + elif full_title: + scene["title"] = full_title + + # Extract code from title if not already set from JSON-LD + if "code" not in scene and h1_elements: + h1_text = h1_elements[0].text_content() + num = extract_fc2_number(h1_text) + if num: + scene["code"] = f"FC2-PPV-{num}" + + # Cover image fallback from og:image + if "image" not in scene: + og_images = tree.xpath('//meta[@property="og:image"]/@content') + if og_images: + scene["image"] = og_images[0] + + # Tags — extracted from work-tags links (NOT in JSON-LD) + tag_elements = tree.xpath('//a[contains(@href, "/work-tags/")]') + tags = [] + for tag_el in tag_elements: + tag_name = tag_el.text_content().strip() + if tag_name: + tags.append({"name": tag_name}) + if tags: + scene["tags"] = tags + + # Mosaic status — from the
metadata grid + dt_elements = tree.xpath('//dt[contains(@class, "text-text-sub")]') + for dt in dt_elements: + label = dt.text_content().strip() + dd = dt.getnext() + if dd is not None and label == "モザイク": + mosaic_val = dd.text_content().strip() + if mosaic_val == "なし": + # Add "無修正" (uncensored) tag if not already present + uncensored_tag = {"name": "無修正"} + if not any(t.get("name") == "無修正" for t in tags): + scene.setdefault("tags", []).append(uncensored_tag) + break + + # Performers fallback — from HTML links if not already set from JSON-LD + if "performers" not in scene: + perf_links = tree.xpath('//a[contains(@href, "/actress/")]') + performers = [] + for link in perf_links: + name = link.text_content().strip() + href = link.get("href", "") + # Filter out navigation links (only include actress profile links) + if name and "/actress/" in href and href != f"{BASE_URL}/actress/": + if href.startswith("http"): + perf = performer_from_url(href) + else: + perf = {"name": name, "gender": "FEMALE"} + performers.append(perf) + if performers: + scene["performers"] = performers + + # Director fallback — from seller link if not already set + if "director" not in scene: + seller_names = tree.xpath('//a[contains(@href, "/seller/")]//span[contains(@class, "font-medium")]/text()') + if seller_names: + scene["director"] = seller_names[0].strip() + + # Fixed studio + scene["studio"] = {"name": "FC2"} + + log.info(f"Scraped scene: {scene.get('code', 'unknown')} - {scene.get('title', '')[:50]}") + return scene + + +# --------------------------------------------------------------------------- +# Performer scraping +# --------------------------------------------------------------------------- + +def performer_from_url(url: str) -> dict: + """Scrape a performer from a fc2db.net actress page.""" + tree = fetch_page(url) + if tree is None: + return {} + + performer = {} + + # --- Primary: JSON-LD Person --- + ld = extract_jsonld(tree, "Person") + + if not ld: + raise Exception(f"Failed to find JSON-LD Person for {url}. Cloudflare challenge may have failed, or site layout changed.") + + log.debug(f"Found JSON-LD Person: {json.dumps(ld, ensure_ascii=False)[:200]}") + + # Name + name = ld.get("name", "").strip() + if name: + performer["name"] = name + + # Image + image = ld.get("image", "") + if image: + performer["images"] = [image] + + # Aliases from alternateName + alt_names = ld.get("alternateName", []) + if isinstance(alt_names, str): + alt_names = [alt_names] + if alt_names: + performer["aliases"] = ", ".join(alt_names) + + # URLs: fc2db URL + sameAs URLs + urls = [url] + same_as = ld.get("sameAs", []) + if isinstance(same_as, str): + same_as = [same_as] + urls.extend(same_as) + performer["urls"] = urls + + # Gender default + performer["gender"] = "FEMALE" + + # Country (FC2 is a Japanese platform) + performer["country"] = "Japan" + + log.info(f"Scraped performer: {performer.get('name', 'unknown')}") + return performer + + +# --------------------------------------------------------------------------- +# Filename → URL resolution +# --------------------------------------------------------------------------- + +def url_from_fragment(args: dict) -> str | None: + """Extract an FC2 number from scene fragment data and construct a URL.""" + # Try title/code first + for field in ("code", "title"): + value = args.get(field, "") + if value: + num = extract_fc2_number(value) + if num: + return work_url_from_number(num) + + # Try URLs + for u in args.get("urls", []): + if "fc2db.net/work/" in u: + return u + num = extract_fc2_number(u) + if num: + return work_url_from_number(num) + + # Try files (for scene-by-fragment via stdin) + files = args.get("files", []) + if files: + filepath = files[0].get("path") if isinstance(files[0], dict) else str(files[0]) + if filepath: + # Try basename first + basename = os.path.basename(filepath) + num = extract_fc2_number(basename) + if num: + return work_url_from_number(num) + # Fallback to parent directory + parent = os.path.basename(os.path.dirname(filepath)) + num = extract_fc2_number(parent) + if num: + return work_url_from_number(num) + + return None + + +# --------------------------------------------------------------------------- +# Main entry point +# --------------------------------------------------------------------------- + +if __name__ == "__main__": + op, args = scraper_args() + result = None + + if op == "scene-by-url": + url = args.get("url", "") + if url: + scene = scene_from_url(url) + result = scene if scene else None + else: + log.error("No URL provided for scene-by-url") + sys.exit(1) + + elif op == "scene-by-name": + name = args.get("name", "") + if not name: + log.error("No name provided for scene-by-name") + sys.exit(1) + num = extract_fc2_number(name) + if num: + url = work_url_from_number(num) + scene = scene_from_url(url) + result = [scene] if scene else [] + else: + log.error(f"Could not extract FC2 number from: {name}") + result = [] + + elif op in ("scene-by-fragment", "scene-by-query-fragment"): + url = url_from_fragment(args) + if url: + scene = scene_from_url(url) + result = scene if scene else None + else: + log.error("Could not extract FC2 number from fragment data") + sys.exit(1) + + elif op == "performer-by-url": + url = args.get("url", "") + if url: + performer = performer_from_url(url) + result = performer if performer else None + else: + log.error("No URL provided for performer-by-url") + sys.exit(1) + + else: + log.error(f"Unknown operation: {op}, arguments: {json.dumps(args)}") + sys.exit(1) + + print(json.dumps(result)) diff --git a/scrapers/fc2db/fc2db.yml b/scrapers/fc2db/fc2db.yml new file mode 100644 index 000000000..aac7cd985 --- /dev/null +++ b/scrapers/fc2db/fc2db.yml @@ -0,0 +1,43 @@ +name: fc2db +# requires: py_common + +sceneByURL: + - url: + - fc2db.net/work/ + action: script + script: + - python + - fc2db.py + - scene-by-url + +sceneByName: + action: script + script: + - python + - fc2db.py + - scene-by-name + +sceneByFragment: + action: script + script: + - python + - fc2db.py + - scene-by-fragment + +sceneByQueryFragment: + action: script + script: + - python + - fc2db.py + - scene-by-query-fragment + +performerByURL: + - url: + - fc2db.net/actress/ + action: script + script: + - python + - fc2db.py + - performer-by-url + +# Last Updated July 07, 2026