diff --git a/scrapers/Sheer.yml b/scrapers/Sheer.yml deleted file mode 100644 index b3099c551..000000000 --- a/scrapers/Sheer.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Sheer -sceneByURL: - - action: scrapeXPath - url: - - sheer.com - scraper: sheer - -xPathScrapers: - sheer: - scene: - Title: //h3[@class="post__title"] - Details: //div[@class="post__text"] - Date: - selector: //span[@class="post__date-text"] - postProcess: - - replace: - # Append the current year for partial dates: this will need to be updated every year - - regex: ^(\w{3}\s*\d{2})$ - with: $1, 2025 - - parseDate: Jan 02, 2006 - Image: //div[@class="post__sfw-preview"]/img/@src - Performers: - Name: //div[@class="post__featuring-models__list"]/a - Tags: - Name: //a[@class="post-tags__link"] - Studio: - Name: //div[contains(@class, "post__profile-name")]/a/span - URL: //div[contains(@class, "post__profile-name")]/a/@href -# Last Updated April 18, 2025 diff --git a/scrapers/Sheer/Sheer.py b/scrapers/Sheer/Sheer.py new file mode 100644 index 000000000..3e865ddf9 --- /dev/null +++ b/scrapers/Sheer/Sheer.py @@ -0,0 +1,204 @@ +import configparser +import json +import os +import re +import sys +from datetime import datetime, timedelta + +sys.path.insert(0, "..") +from py_common import log +import requests +from bs4 import BeautifulSoup + +USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +SHEER_BASE = "https://www.sheer.com" +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +CONFIG_FILE = os.path.join(SCRIPT_DIR, "config.ini") + +DEFAULT_CONFIG = """[Sheer] +# Paste your cookie string here. +# In Chrome/Firefox: F12 -> Network -> any sheer.com request -> Headers -> Cookie +cookie = +""" + + +def get_cookies(): + if not os.path.exists(CONFIG_FILE): + with open(CONFIG_FILE, "w") as f: + f.write(DEFAULT_CONFIG) + log.info(f"Created config.ini at {CONFIG_FILE} - add your cookie string to scrape authenticated content") + return {} + + config = configparser.RawConfigParser() + config.read(CONFIG_FILE) + cookie_str = config.get("Sheer", "cookie", fallback="").strip() + if not cookie_str: + log.warning("No cookie set in config.ini - scraping logged-out content only") + return {} + + cookies = {} + for part in cookie_str.split(";"): + part = part.strip() + if "=" in part: + name, _, value = part.partition("=") + cookies[name.strip()] = value.strip() + return cookies + + +def fetch(url): + cookies = get_cookies() + r = requests.get( + url, + headers={"User-Agent": USER_AGENT}, + cookies=cookies, + timeout=15, + ) + if r.status_code != 200: + log.error(f"HTTP {r.status_code} for {url}") + sys.exit(1) + return r.text + + +def parse_relative_date(text): + text = text.strip().lower() + now = datetime.now() + + m = re.match(r"(\d+)\s+(second|minute|hour|day|week|month|year)s?\s+ago", text) + if m: + n = int(m.group(1)) + unit = m.group(2) + deltas = { + "second": timedelta(seconds=n), + "minute": timedelta(minutes=n), + "hour": timedelta(hours=n), + "day": timedelta(days=n), + "week": timedelta(weeks=n), + "month": timedelta(days=n * 30), + "year": timedelta(days=n * 365), + } + return (now - deltas.get(unit, timedelta(0))).strftime("%Y-%m-%d") + + for fmt in ("%b %d, %Y", "%B %d, %Y", "%Y-%m-%d", "%b. %d, %Y", + "%d %b %Y", "%d %B %Y"): + try: + return datetime.strptime(text, fmt).strftime("%Y-%m-%d") + except ValueError: + pass + + for fmt in ("%b %d", "%B %d", "%d %b", "%d %B"): + try: + d = datetime.strptime(text, fmt) + return d.replace(year=now.year).strftime("%Y-%m-%d") + except ValueError: + pass + + log.warning(f"Could not parse date: {text!r}") + return None + + +def parse_post(html, post_url): + soup = BeautifulSoup(html, "html.parser") + scrape = {} + + # Scope to the specific post article + post_id = post_url.rstrip("/").split("/")[-1] + article = soup.find("article", {"data-post-id": post_id}) or soup.find("article") + + ctx = article if article else soup + + # Title + el = ctx.select_one("h3.post__title") + if el: + scrape["title"] = el.get_text(strip=True) + else: + og = soup.find("meta", property="og:title") + if og: + scrape["title"] = og.get("content", "").strip() + + # Date + time_el = soup.find("time") + if time_el and time_el.get("datetime"): + scrape["date"] = time_el["datetime"][:10] + else: + date_el = ctx.select_one(".post__date-text") + if date_el: + parsed = parse_relative_date(date_el.get_text(strip=True)) + if parsed: + scrape["date"] = parsed + + # Details + el = ctx.select_one(".post__text") + if el: + scrape["details"] = el.get_text(strip=True) + else: + og = soup.find("meta", property="og:description") + if og: + scrape["details"] = og.get("content", "").strip() + + # Image — data-poster on video tag (server-rendered) + video_el = ctx.select_one("video[data-poster]") + if video_el and video_el.get("data-poster"): + scrape["image"] = video_el["data-poster"] + else: + img_el = ctx.select_one("img.video-poster__img") + if img_el and img_el.get("src"): + scrape["image"] = img_el["src"] + else: + og_img = soup.find("meta", property="og:image") + if og_img: + scrape["image"] = og_img.get("content", "") + + # Tags — inside