From be5c9592325b8216297cee0c3c37bb6211850e5f Mon Sep 17 00:00:00 2001 From: LiuShen <3162475700@qq.com> Date: Sat, 26 Jul 2025 19:39:49 +0800 Subject: [PATCH 01/30] =?UTF-8?q?=F0=9F=98=83=E9=87=8D=E6=9E=84=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E7=BB=93=E6=9E=84=EF=BC=8C=E6=96=B9=E4=BE=BF=E5=90=8E?= =?UTF-8?q?=E7=BB=AD=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.example | 2 + .gitignore | 6 +- conf.yaml | 2 +- friend_circle_lite/__init__.py | 26 + friend_circle_lite/all_friends.py | 260 +++++++ friend_circle_lite/get_conf.py | 14 - friend_circle_lite/get_info.py | 651 ------------------ friend_circle_lite/single_friend.py | 310 +++++++++ .../utils}/__init__.py | 0 friend_circle_lite/utils/cache.py | 35 + friend_circle_lite/utils/config.py | 25 + friend_circle_lite/utils/github.py | 39 ++ friend_circle_lite/utils/json.py | 30 + .../utils/mail.py | 1 - friend_circle_lite/utils/time.py | 42 ++ friend_circle_lite/utils/url.py | 27 + main/fclite.js | 6 +- .../default.html | 0 rss_subscribe/__init__.py | 0 rss_subscribe/push_article_update.py | 105 --- run.py | 47 +- server.py | 3 - 22 files changed, 826 insertions(+), 805 deletions(-) create mode 100644 .env.example create mode 100644 friend_circle_lite/all_friends.py delete mode 100644 friend_circle_lite/get_conf.py delete mode 100644 friend_circle_lite/get_info.py create mode 100644 friend_circle_lite/single_friend.py rename {push_rss_update => friend_circle_lite/utils}/__init__.py (100%) create mode 100644 friend_circle_lite/utils/cache.py create mode 100644 friend_circle_lite/utils/config.py create mode 100644 friend_circle_lite/utils/github.py create mode 100644 friend_circle_lite/utils/json.py rename push_rss_update/send_email.py => friend_circle_lite/utils/mail.py (99%) create mode 100644 friend_circle_lite/utils/time.py create mode 100644 friend_circle_lite/utils/url.py rename rss_subscribe/email_template.html => push_templates/default.html (100%) delete mode 100644 rss_subscribe/__init__.py delete mode 100644 rss_subscribe/push_article_update.py diff --git a/.env.example b/.env.example new file mode 100644 index 00000000000..08299e579af --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +# SMTP 密码，请在github上配置，如果为服务器部署，不担心泄露，可以在这里直接写入 +SMTP_PWD=123432424 \ No newline at end of file diff --git a/.gitignore b/.gitignore index 7747aadb5c2..e52e7fb4080 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,8 @@ # 忽略数据文件 *.json -*.bat \ No newline at end of file +*.bat + +.env + +temp/ \ No newline at end of file diff --git a/conf.yaml b/conf.yaml index b26d072a945..b5a406e1768 100644 --- a/conf.yaml +++ b/conf.yaml @@ -39,7 +39,7 @@ rss_subscribe: github_username: willow-god github_repo: Friend-Circle-Lite your_blog_url: https://blog.liushen.fun/ - email_template: "./rss_subscribe/email_template.html" + email_template: "./push_templates/default.html" website_info: title: "清羽飞扬" diff --git a/friend_circle_lite/__init__.py b/friend_circle_lite/__init__.py index e69de29bb2d..0df0993ad48 100644 --- a/friend_circle_lite/__init__.py +++ b/friend_circle_lite/__init__.py @@ -0,0 +1,26 @@ +# 标准化的请求头 +HEADERS_JSON = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/123.0.0.0 Safari/537.36 " + "(Friend-Circle-Lite/1.0; +https://github.com/willow-god/Friend-Circle-Lite)" + ), + "X-Friend-Circle": "1.0" +} + +HEADERS_XML = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/123.0.0.0 Safari/537.36 " + "(Friend-Circle-Lite/1.0; +https://github.com/willow-god/Friend-Circle-Lite)" + ), + "Accept": "application/atom+xml, application/rss+xml, application/xml;q=0.9, */*;q=0.8", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "X-Friend-Circle": "1.0" +} + +timeout = (10, 15) \ No newline at end of file diff --git a/friend_circle_lite/all_friends.py b/friend_circle_lite/all_friends.py new file mode 100644 index 00000000000..956f1aa4708 --- /dev/null +++ b/friend_circle_lite/all_friends.py @@ -0,0 +1,260 @@ +import string +import requests +import logging +from datetime import datetime +from zoneinfo import ZoneInfo +import requests +from concurrent.futures import ThreadPoolExecutor, as_completed +from friend_circle_lite.utils.cache import load_cache, save_cache +from friend_circle_lite.single_friend import process_friend +from friend_circle_lite import HEADERS_JSON, timeout + +def fetch_and_process_data(json_url: str, specific_RSS: list = None, count: int = 5, cache_file: str = None): + """ + 读取 JSON 数据并处理订阅信息，返回统计数据和文章信息。 + + 参数： + json_url (str): 包含朋友信息的 JSON 文件的 URL。 + count (int): 获取每个博客的最大文章数。 + specific_RSS (list): 包含特定 RSS 源的字典列表 [{name, url}]（来自 YAML）。 + cache_file (str): 缓存文件路径。 + + 返回： + (result_dict, error_friends_info_list) + """ + if specific_RSS is None: + specific_RSS = [] + + # 1. 加载缓存 + cache_list = load_cache(cache_file) + + # 2. 标记 YAML 条目 + manual_list = [] + for item in specific_RSS: + if isinstance(item, dict) and 'name' in item and 'url' in item: + manual_list.append({'name': item['name'], 'url': item['url'], 'source': 'manual'}) + + # 3. 合并（缓存先，YAML 后覆盖） + combined_map = {e['name']: e for e in cache_list} + for e in manual_list: # 手动优先 + combined_map[e['name']] = e + specific_and_cache = list(combined_map.values()) + + # 4. 建立方便判断的集合：手动源名称集合 + manual_name_set = {e['name'] for e in manual_list} + + # 5. 获取朋友列表 + session = requests.Session() + try: + response = session.get(json_url, headers=HEADERS_JSON, timeout=timeout) + friends_data = response.json() + except Exception as e: + logging.error(f"无法获取链接：{json_url} ：{e}", exc_info=True) + return None + + friends = friends_data.get('friends', []) + total_friends = len(friends) + active_friends = 0 + error_friends = 0 + total_articles = 0 + article_data = [] + error_friends_info = [] + cache_updates = [] # 用于收集缓存更新（线程安全：用局部列表 + 合并） + + # 6. 并发处理 + with ThreadPoolExecutor(max_workers=10) as executor: + future_to_friend = { + executor.submit(process_friend, friend, session, count, specific_and_cache): friend + for friend in friends + } + + for future in as_completed(future_to_friend): + friend = future_to_friend[future] + try: + result = future.result() + + # 拿回缓存更新意图 + upd = result.get('cache_update', {}) + if upd and upd.get('action') != 'none': + cache_updates.append(upd) + + if result['status'] == 'active': + active_friends += 1 + article_data.extend(result['articles']) + total_articles += len(result['articles']) + else: + error_friends += 1 + error_friends_info.append(friend) + + except Exception as e: + logging.error(f"处理 {friend} 时发生错误: {e}", exc_info=True) + error_friends += 1 + error_friends_info.append(friend) + + # 7. 处理缓存更新 + cache_map = {e['name']: e for e in cache_list} + + # 去重 & 过滤无效条目 + unique_updates = {} + for upd in cache_updates: + name = upd.get('name') + action = upd.get('action') + url = upd.get('url') + if not name: + continue + + # 过滤手动 YAML 的条目（不允许覆盖） + if name in manual_name_set: + continue + + # 只缓存有效 RSS 地址 + if action == 'set': + if url and url != 'none' and url != '': + unique_updates[name] = {'action': 'set', 'url': url, 'reason': upd.get('reason', '')} + elif action == 'delete': + unique_updates[name] = {'action': 'delete', 'url': None, 'reason': upd.get('reason', '')} + + # 应用缓存更新 + for name, upd in unique_updates.items(): + if upd['action'] == 'set': + cache_map[name] = {'name': name, 'url': upd['url'], 'source': 'cache'} + logging.info(f"缓存更新：SET {name} -> {upd['url']} ({upd['reason']})") + elif upd['action'] == 'delete': + if name in cache_map: + cache_map.pop(name) + logging.info(f"缓存更新：DELETE {name} ({upd['reason']})") + + # 8. 保存缓存 + save_cache(cache_file, list(cache_map.values())) + + # 9. 汇总统计 + result = { + 'statistical_data': { + 'friends_num': total_friends, + 'active_num': active_friends, + 'error_num': error_friends, + 'article_num': total_articles, + 'last_updated_time': datetime.now(ZoneInfo("Asia/Shanghai")).strftime('%Y-%m-%d %H:%M:%S'), + }, + 'article_data': article_data, + } + + logging.info( + f"数据处理完成，总共有 {total_friends} 位朋友，其中 {active_friends} 位博客可访问，" + f"{error_friends} 位博客无法访问。缓存更新 {len(unique_updates)} 条。" + ) + + return result, error_friends_info + +def sort_articles_by_time(data): + """ + 对文章数据按时间排序 + + 参数： + data (dict): 包含文章信息的字典 + + 返回： + dict: 按时间排序后的文章信息字典 + """ + # 先确保每个元素存在时间 + for article in data['article_data']: + if article['created'] == '' or article['created'] == None: + article['created'] = '2024-01-01 00:00' + # 输出警告信息 + logging.warning(f"文章 {article['title']} 未包含时间信息，已设置为默认时间 2024-01-01 00:00") + + if 'article_data' in data: + sorted_articles = sorted( + data['article_data'], + key=lambda x: datetime.strptime(x['created'], '%Y-%m-%d %H:%M'), + reverse=True + ) + data['article_data'] = sorted_articles + return data + +def marge_data_from_json_url(data, marge_json_url): + """ + 从另一个 JSON 文件中获取数据并合并到原数据中。 + + 参数： + data (dict): 包含文章信息的字典 + marge_json_url (str): 包含另一个文章信息的 JSON 文件的 URL。 + + 返回： + dict: 合并后的文章信息字典，已去重处理 + """ + try: + response = requests.get(marge_json_url, headers=HEADERS_JSON, timeout=timeout) + marge_data = response.json() + except Exception as e: + logging.error(f"无法获取链接：{marge_json_url}，出现的问题为：{e}", exc_info=True) + return data + + if 'article_data' in marge_data: + logging.info(f"开始合并数据，原数据共有 {len(data['article_data'])} 篇文章，第三方数据共有 {len(marge_data['article_data'])} 篇文章") + data['article_data'].extend(marge_data['article_data']) + data['article_data'] = list({v['link']:v for v in data['article_data']}.values()) + logging.info(f"合并数据完成，现在共有 {len(data['article_data'])} 篇文章") + return data + +def marge_errors_from_json_url(errors, marge_json_url): + """ + 从另一个网络 JSON 文件中获取错误信息并遍历，删除在errors中， + 不存在于marge_errors中的友链信息。 + + 参数： + errors (list): 包含错误信息的列表 + marge_json_url (str): 包含另一个错误信息的 JSON 文件的 URL。 + + 返回： + list: 合并后的错误信息列表 + """ + try: + response = requests.get(marge_json_url, timeout=10) # 设置请求超时时间 + marge_errors = response.json() + except Exception as e: + logging.error(f"无法获取链接：{marge_json_url}，出现的问题为：{e}", exc_info=True) + return errors + + # 提取 marge_errors 中的 URL + marge_urls = {item[1] for item in marge_errors} + + # 使用过滤器保留 errors 中在 marge_errors 中出现的 URL + filtered_errors = [error for error in errors if error[1] in marge_urls] + + logging.info(f"合并错误信息完成，合并后共有 {len(filtered_errors)} 位朋友") + return filtered_errors + +def deal_with_large_data(result): + """ + 处理文章数据，保留前150篇及其作者在后续文章中的出现。 + + 参数： + result (dict): 包含统计数据和文章数据的字典。 + + 返回： + dict: 处理后的数据，只包含需要的文章。 + """ + result = sort_articles_by_time(result) + article_data = result.get("article_data", []) + + # 检查文章数量是否大于 150 + max_articles = 150 + if len(article_data) > max_articles: + logging.info("数据量较大，开始进行处理...") + # 获取前 max_articles 篇文章的作者集合 + top_authors = {article["author"] for article in article_data[:max_articles]} + + # 从第 {max_articles + 1} 篇开始过滤，只保留前 max_articles 篇出现过的作者的文章 + filtered_articles = article_data[:max_articles] + [ + article for article in article_data[max_articles:] + if article["author"] in top_authors + ] + + # 更新结果中的 article_data + result["article_data"] = filtered_articles + # 更新结果中的统计数据 + result["statistical_data"]["article_num"] = len(filtered_articles) + logging.info(f"数据处理完成，保留 {len(filtered_articles)} 篇文章") + + return result \ No newline at end of file diff --git a/friend_circle_lite/get_conf.py b/friend_circle_lite/get_conf.py deleted file mode 100644 index 46b487c1f9b..00000000000 --- a/friend_circle_lite/get_conf.py +++ /dev/null @@ -1,14 +0,0 @@ -import yaml - -def load_config(config_file): - """ - 加载配置文件。 - - 参数： - config_file (str): 配置文件的路径。 - - 返回： - dict: 加载的配置数据。 - """ - with open(config_file, 'r', encoding='utf-8') as file: - return yaml.safe_load(file) \ No newline at end of file diff --git a/friend_circle_lite/get_info.py b/friend_circle_lite/get_info.py deleted file mode 100644 index f2856364f09..00000000000 --- a/friend_circle_lite/get_info.py +++ /dev/null @@ -1,651 +0,0 @@ -import logging -from datetime import datetime, timedelta, timezone -import re -import os -import json -from urllib.parse import urljoin, urlparse -from dateutil import parser -from zoneinfo import ZoneInfo -import requests -import feedparser -from concurrent.futures import ThreadPoolExecutor, as_completed - -# 标准化的请求头 -HEADERS_JSON = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/123.0.0.0 Safari/537.36 " - "(Friend-Circle-Lite/1.0; +https://github.com/willow-god/Friend-Circle-Lite)" - ), - "X-Friend-Circle": "1.0" -} - -HEADERS_XML = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/123.0.0.0 Safari/537.36 " - "(Friend-Circle-Lite/1.0; +https://github.com/willow-god/Friend-Circle-Lite)" - ), - "Accept": "application/atom+xml, application/rss+xml, application/xml;q=0.9, */*;q=0.8", - "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", - "Accept-Encoding": "gzip, deflate", - "Connection": "keep-alive", - "X-Friend-Circle": "1.0" -} - -timeout = (10, 15) # 连接超时和读取超时，防止requests接受时间过长 - -def format_published_time(time_str): - """ - 格式化发布时间为统一格式 YYYY-MM-DD HH:MM - - 参数: - time_str (str): 输入的时间字符串，可能是多种格式。 - - 返回: - str: 格式化后的时间字符串，若解析失败返回空字符串。 - """ - # 尝试自动解析输入时间字符串 - try: - parsed_time = parser.parse(time_str, fuzzy=True) - except (ValueError, parser.ParserError): - # 定义支持的时间格式 - time_formats = [ - '%a, %d %b %Y %H:%M:%S %z', # Mon, 11 Mar 2024 14:08:32 +0000 - '%a, %d %b %Y %H:%M:%S GMT', # Wed, 19 Jun 2024 09:43:53 GMT - '%Y-%m-%dT%H:%M:%S%z', # 2024-03-11T14:08:32+00:00 - '%Y-%m-%dT%H:%M:%SZ', # 2024-03-11T14:08:32Z - '%Y-%m-%d %H:%M:%S', # 2024-03-11 14:08:32 - '%Y-%m-%d' # 2024-03-11 - ] - for fmt in time_formats: - try: - parsed_time = datetime.strptime(time_str, fmt) - break - except ValueError: - continue - else: - logging.warning(f"无法解析时间字符串：{time_str}") - return '' - - # 处理时区转换 - if parsed_time.tzinfo is None: - parsed_time = parsed_time.replace(tzinfo=timezone.utc) - shanghai_time = parsed_time.astimezone(timezone(timedelta(hours=8))) - return shanghai_time.strftime('%Y-%m-%d %H:%M') - -def check_feed(blog_url, session): - """ - 检查博客的 RSS 或 Atom 订阅链接。 - - 优化点： - - 检查 HTTP 状态码。 - - 检查 Content-Type 是否包含 xml / rss / atom。 - - 检查响应内容前几百字节内是否有 RSS/Atom 的特征标签。 - """ - possible_feeds = [ - ('atom', '/atom.xml'), - ('rss', '/rss.xml'), # 2024-07-26 添加 /rss.xml内容的支持 - ('rss2', '/rss2.xml'), - ('rss3', '/rss.php'), # 2024-12-07 添加 /rss.php内容的支持 - ('feed', '/feed'), - ('feed2', '/feed.xml'), # 2024-07-26 添加 /feed.xml内容的支持 - ('feed3', '/feed/'), - ('feed4', '/feed.php'), # 2025-07-22 添加 /feed.php内容的支持 - ('index', '/index.xml') # 2024-07-25 添加 /index.xml内容的支持 - ] - - for feed_type, path in possible_feeds: - feed_url = blog_url.rstrip('/') + path - try: - response = session.get(feed_url, headers=HEADERS_XML, timeout=timeout) - if response.status_code == 200: - # 检查 Content-Type - content_type = response.headers.get('Content-Type', '').lower() - if 'xml' in content_type or 'rss' in content_type or 'atom' in content_type: - return [feed_type, feed_url] - - # 如果 Content-Type 是 text/html 或未明确，但内容本身是 RSS - text_head = response.text[:1000].lower() # 读取前1000字符 - if (' str: - """ - 暂未实现 - 检测并替换字符串中的非正常域名部分（如 IP 地址或 localhost），替换为 blog_url。 - 替换后强制使用 https，且考虑 blog_url 尾部是否有斜杠。 - - :param link: 原始地址字符串 - :param blog_url: 替换为的博客地址 - :return: 替换后的地址字符串 - """ - - # 提取link中的路径部分，无需协议和域名 - # path = re.sub(r'^https?://[^/]+', '', link) - # print(path) - - try: - parsed = urlparse(link) - if 'localhost' in parsed.netloc or re.match(r'^\d{1,3}(\.\d{1,3}){3}$', parsed.netloc): # IP地址或localhost - # 提取 path + query - path = parsed.path or '/' - if parsed.query: - path += '?' + parsed.query - return urljoin(blog_url.rstrip('/') + '/', path.lstrip('/')) - else: - return link # 合法域名则返回原链接 - except Exception as e: - logging.warning(f"替换链接时出错：{link}, error: {e}") - return link - -def process_friend(friend, session, count, specific_and_cache=None): - """ - 处理单个朋友的博客信息。 - - 参数： - friend (list/tuple): [name, blog_url, avatar] - session (requests.Session): 请求会话 - count (int): 每个博客最大文章数 - specific_and_cache (list[dict]): [{name, url, source?}]，合并后的特殊 + 缓存列表 - - 返回： - { - 'name': name, - 'status': 'active' | 'error', - 'articles': [...], - 'feed_url': str | None, - 'feed_type': str, - 'cache_update': { - 'action': 'set' | 'delete' | 'none', - 'name': name, - 'url': feed_url_or_None, - 'reason': 'auto_discovered' | 'repair_cache' | 'remove_invalid', - }, - 'source_used': 'manual' | 'cache' | 'auto' | 'none' - } - """ - if specific_and_cache is None: - specific_and_cache = [] - - # 解包 friend - try: - name, blog_url, avatar = friend - except Exception: - logging.error(f"friend 数据格式不正确: {friend!r}") - return { - 'name': None, - 'status': 'error', - 'articles': [], - 'feed_url': None, - 'feed_type': 'none', - 'cache_update': {'action': 'none', 'name': None, 'url': None, 'reason': 'bad_friend_data'}, - 'source_used': 'none', - } - - rss_lookup = {e['name']: e for e in specific_and_cache if 'name' in e and 'url' in e} - cache_update = {'action': 'none', 'name': name, 'url': None, 'reason': ''} - feed_url, feed_type, source_used = None, 'none', 'none' - - # ---- 1. 优先使用 specific 或 cache ---- - entry = rss_lookup.get(name) - if entry: - feed_url = entry['url'] - feed_type = 'specific' - source_used = entry.get('source', 'unknown') - logging.info(f"“{name}” 使用预设 RSS 源：{feed_url} （source={source_used}）。") - else: - # ---- 2. 自动探测 ---- - feed_type, feed_url = check_feed(blog_url, session) - source_used = 'auto' - logging.info(f"“{name}” 自动探测 RSS：type：{feed_type}, url：{feed_url} 。") - - if feed_type != 'none' and feed_url: - cache_update = {'action': 'set', 'name': name, 'url': feed_url, 'reason': 'auto_discovered'} - - # ---- 3. 尝试解析 RSS ---- - articles, parse_error = [], False - if feed_type != 'none' and feed_url: - try: - feed_info = parse_feed(feed_url, session, count, blog_url) - if isinstance(feed_info, dict) and 'articles' in feed_info: - articles = [ - { - 'title': a['title'], - 'created': a['published'], - 'link': a['link'], - 'author': name, - 'avatar': avatar, - } - for a in feed_info['articles'] - ] - - for a in articles: - logging.info(f"{name} 发布了新文章：{a['title']}，时间：{a['created']}，链接：{a['link']}") - else: - parse_error = True - except Exception as e: - logging.warning(f"解析 RSS 失败（{name} -> {feed_url}）：{e}") - parse_error = True - - # ---- 4. 如果缓存 RSS 无效则重新探测 ---- - if parse_error and source_used in ('cache', 'unknown'): - logging.info(f"缓存 RSS 无效，重新探测：{name} ({blog_url})。") - new_type, new_url = check_feed(blog_url, session) - if new_type != 'none' and new_url: - try: - feed_info = parse_feed(new_url, session, count, blog_url) - if isinstance(feed_info, dict) and 'articles' in feed_info: - articles = [ - { - 'title': a['title'], - 'created': a['published'], - 'link': a['link'], - 'author': name, - 'avatar': avatar, - } - for a in feed_info['articles'] - ] - - for a in articles: - logging.info(f"{name} 发布了新文章：{a['title']}，时间：{a['created']}，链接：{a['link']}") - - feed_type, feed_url, source_used = new_type, new_url, 'auto' - cache_update = {'action': 'set', 'name': name, 'url': new_url, 'reason': 'repair_cache'} - parse_error = False - except Exception as e: - logging.warning(f"重新探测解析仍失败：{name} ({new_url})：{e}") - cache_update = {'action': 'delete', 'name': name, 'url': None, 'reason': 'remove_invalid'} - feed_type, feed_url = 'none', None - else: - cache_update = {'action': 'delete', 'name': name, 'url': None, 'reason': 'remove_invalid'} - feed_type, feed_url = 'none', None - - # ---- 5. 最终状态 ---- - status = 'active' if articles else 'error' - if not articles: - if feed_type == 'none': - logging.warning(f"{name} 的博客 {blog_url} 未找到有效 RSS。") - else: - logging.warning(f"{name} 的 RSS {feed_url} 未解析出文章。") - - return { - 'name': name, - 'status': status, - 'articles': articles, - 'feed_url': feed_url, - 'feed_type': feed_type, - 'cache_update': cache_update, - 'source_used': source_used, - } - -def _load_cache(cache_file): - if not cache_file: - return [] - if not os.path.exists(cache_file): - logging.info(f"缓存文件 {cache_file} 不存在，将自动创建。") - return [] - try: - with open(cache_file, 'r', encoding='utf-8') as f: - data = json.load(f) - if not isinstance(data, list): - logging.warning(f"缓存文件 {cache_file} 格式异常（应为列表）。将忽略。") - return [] - # 标准化 - norm = [] - for item in data: - if not isinstance(item, dict): - continue - name = item.get('name') - url = item.get('url') - if name and url: - norm.append({'name': name, 'url': url, 'source': 'cache'}) - return norm - except Exception as e: - logging.warning(f"读取缓存文件 {cache_file} 失败: {e}") - return [] - -def _atomic_write_json(path, data) -> None: - """原子写，减少写坏文件风险。""" - tmp = f"{path}.tmp" - with open(tmp, 'w', encoding='utf-8') as f: - json.dump(data, f, ensure_ascii=False, indent=2) - os.replace(tmp, path) - -def _save_cache(cache_file, cache_items): - if not cache_file: - return - try: - # 丢弃 source 字段以保持文件简洁 - out = [{'name': i['name'], 'url': i['url']} for i in cache_items] - _atomic_write_json(cache_file, out) - logging.info(f"缓存已保存到 {cache_file}（{len(out)} 条）。") - except Exception as e: - logging.error(f"保存缓存文件 {cache_file} 失败: {e}") - -def fetch_and_process_data(json_url, specific_RSS=None, count=5, cache_file=None): - """ - 读取 JSON 数据并处理订阅信息，返回统计数据和文章信息。 - - 参数： - json_url (str): 包含朋友信息的 JSON 文件的 URL。 - count (int): 获取每个博客的最大文章数。 - specific_RSS (list): 包含特定 RSS 源的字典列表 [{name, url}]（来自 YAML）。 - cache_file (str): 缓存文件路径。 - - 返回： - (result_dict, error_friends_info_list) - """ - if specific_RSS is None: - specific_RSS = [] - - # 1. 加载缓存 - cache_list = _load_cache(cache_file) - - # 2. 标记 YAML 条目 - manual_list = [] - for item in specific_RSS: - if isinstance(item, dict) and 'name' in item and 'url' in item: - manual_list.append({'name': item['name'], 'url': item['url'], 'source': 'manual'}) - - # 3. 合并（缓存先，YAML 后覆盖） - combined_map = {e['name']: e for e in cache_list} - for e in manual_list: # 手动优先 - combined_map[e['name']] = e - specific_and_cache = list(combined_map.values()) - - # 4. 建立方便判断的集合：手动源名称集合 - manual_name_set = {e['name'] for e in manual_list} - - # 5. 获取朋友列表 - session = requests.Session() - try: - response = session.get(json_url, headers=HEADERS_JSON, timeout=timeout) - friends_data = response.json() - except Exception as e: - logging.error(f"无法获取链接：{json_url} ：{e}", exc_info=True) - return None - - friends = friends_data.get('friends', []) - total_friends = len(friends) - active_friends = 0 - error_friends = 0 - total_articles = 0 - article_data = [] - error_friends_info = [] - cache_updates = [] # 用于收集缓存更新（线程安全：用局部列表 + 合并） - - # 6. 并发处理 - with ThreadPoolExecutor(max_workers=10) as executor: - future_to_friend = { - executor.submit(process_friend, friend, session, count, specific_and_cache): friend - for friend in friends - } - - for future in as_completed(future_to_friend): - friend = future_to_friend[future] - try: - result = future.result() - - # 拿回缓存更新意图 - upd = result.get('cache_update', {}) - if upd and upd.get('action') != 'none': - cache_updates.append(upd) - - if result['status'] == 'active': - active_friends += 1 - article_data.extend(result['articles']) - total_articles += len(result['articles']) - else: - error_friends += 1 - error_friends_info.append(friend) - - except Exception as e: - logging.error(f"处理 {friend} 时发生错误: {e}", exc_info=True) - error_friends += 1 - error_friends_info.append(friend) - - # 7. 处理缓存更新 - cache_map = {e['name']: e for e in cache_list} - - # 去重 & 过滤无效条目 - unique_updates = {} - for upd in cache_updates: - name = upd.get('name') - action = upd.get('action') - url = upd.get('url') - if not name: - continue - - # 过滤手动 YAML 的条目（不允许覆盖） - if name in manual_name_set: - continue - - # 只缓存有效 RSS 地址 - if action == 'set': - if url and url != 'none' and url != '': - unique_updates[name] = {'action': 'set', 'url': url, 'reason': upd.get('reason', '')} - elif action == 'delete': - unique_updates[name] = {'action': 'delete', 'url': None, 'reason': upd.get('reason', '')} - - # 应用缓存更新 - for name, upd in unique_updates.items(): - if upd['action'] == 'set': - cache_map[name] = {'name': name, 'url': upd['url'], 'source': 'cache'} - logging.info(f"缓存更新：SET {name} -> {upd['url']} ({upd['reason']})") - elif upd['action'] == 'delete': - if name in cache_map: - cache_map.pop(name) - logging.info(f"缓存更新：DELETE {name} ({upd['reason']})") - - # 8. 保存缓存 - _save_cache(cache_file, list(cache_map.values())) - - # 9. 汇总统计 - result = { - 'statistical_data': { - 'friends_num': total_friends, - 'active_num': active_friends, - 'error_num': error_friends, - 'article_num': total_articles, - 'last_updated_time': datetime.now(ZoneInfo("Asia/Shanghai")).strftime('%Y-%m-%d %H:%M:%S'), - }, - 'article_data': article_data, - } - - logging.info( - f"数据处理完成，总共有 {total_friends} 位朋友，其中 {active_friends} 位博客可访问，" - f"{error_friends} 位博客无法访问。缓存更新 {len(unique_updates)} 条。" - ) - - return result, error_friends_info - -def sort_articles_by_time(data): - """ - 对文章数据按时间排序 - - 参数： - data (dict): 包含文章信息的字典 - - 返回： - dict: 按时间排序后的文章信息字典 - """ - # 先确保每个元素存在时间 - for article in data['article_data']: - if article['created'] == '' or article['created'] == None: - article['created'] = '2024-01-01 00:00' - # 输出警告信息 - logging.warning(f"文章 {article['title']} 未包含时间信息，已设置为默认时间 2024-01-01 00:00") - - if 'article_data' in data: - sorted_articles = sorted( - data['article_data'], - key=lambda x: datetime.strptime(x['created'], '%Y-%m-%d %H:%M'), - reverse=True - ) - data['article_data'] = sorted_articles - return data - -def marge_data_from_json_url(data, marge_json_url): - """ - 从另一个 JSON 文件中获取数据并合并到原数据中。 - - 参数： - data (dict): 包含文章信息的字典 - marge_json_url (str): 包含另一个文章信息的 JSON 文件的 URL。 - - 返回： - dict: 合并后的文章信息字典，已去重处理 - """ - try: - response = requests.get(marge_json_url, headers=HEADERS_JSON, timeout=timeout) - marge_data = response.json() - except Exception as e: - logging.error(f"无法获取链接：{marge_json_url}，出现的问题为：{e}", exc_info=True) - return data - - if 'article_data' in marge_data: - logging.info(f"开始合并数据，原数据共有 {len(data['article_data'])} 篇文章，第三方数据共有 {len(marge_data['article_data'])} 篇文章") - data['article_data'].extend(marge_data['article_data']) - data['article_data'] = list({v['link']:v for v in data['article_data']}.values()) - logging.info(f"合并数据完成，现在共有 {len(data['article_data'])} 篇文章") - return data - -import requests - -def marge_errors_from_json_url(errors, marge_json_url): - """ - 从另一个网络 JSON 文件中获取错误信息并遍历，删除在errors中， - 不存在于marge_errors中的友链信息。 - - 参数： - errors (list): 包含错误信息的列表 - marge_json_url (str): 包含另一个错误信息的 JSON 文件的 URL。 - - 返回： - list: 合并后的错误信息列表 - """ - try: - response = requests.get(marge_json_url, timeout=10) # 设置请求超时时间 - marge_errors = response.json() - except Exception as e: - logging.error(f"无法获取链接：{marge_json_url}，出现的问题为：{e}", exc_info=True) - return errors - - # 提取 marge_errors 中的 URL - marge_urls = {item[1] for item in marge_errors} - - # 使用过滤器保留 errors 中在 marge_errors 中出现的 URL - filtered_errors = [error for error in errors if error[1] in marge_urls] - - logging.info(f"合并错误信息完成，合并后共有 {len(filtered_errors)} 位朋友") - return filtered_errors - -def deal_with_large_data(result): - """ - 处理文章数据，保留前150篇及其作者在后续文章中的出现。 - - 参数： - result (dict): 包含统计数据和文章数据的字典。 - - 返回： - dict: 处理后的数据，只包含需要的文章。 - """ - result = sort_articles_by_time(result) - article_data = result.get("article_data", []) - - # 检查文章数量是否大于 150 - max_articles = 150 - if len(article_data) > max_articles: - logging.info("数据量较大，开始进行处理...") - # 获取前 max_articles 篇文章的作者集合 - top_authors = {article["author"] for article in article_data[:max_articles]} - - # 从第 {max_articles + 1} 篇开始过滤，只保留前 max_articles 篇出现过的作者的文章 - filtered_articles = article_data[:max_articles] + [ - article for article in article_data[max_articles:] - if article["author"] in top_authors - ] - - # 更新结果中的 article_data - result["article_data"] = filtered_articles - # 更新结果中的统计数据 - result["statistical_data"]["article_num"] = len(filtered_articles) - logging.info(f"数据处理完成，保留 {len(filtered_articles)} 篇文章") - - return result \ No newline at end of file diff --git a/friend_circle_lite/single_friend.py b/friend_circle_lite/single_friend.py new file mode 100644 index 00000000000..d1ee1108b97 --- /dev/null +++ b/friend_circle_lite/single_friend.py @@ -0,0 +1,310 @@ +import logging +from datetime import datetime +import re +import os +import json +import requests +import feedparser +from friend_circle_lite import HEADERS_XML, timeout +from friend_circle_lite.utils.time import format_published_time +from friend_circle_lite.utils.url import replace_non_domain + +def check_feed(blog_url, session): + """ + 检查博客的 RSS 或 Atom 订阅链接。 + + 优化点： + - 检查 HTTP 状态码。 + - 检查 Content-Type 是否包含 xml / rss / atom。 + - 检查响应内容前几百字节内是否有 RSS/Atom 的特征标签。 + """ + possible_feeds = [ + ('atom', '/atom.xml'), + ('rss', '/rss.xml'), # 2024-07-26 添加 /rss.xml内容的支持 + ('rss2', '/rss2.xml'), + ('rss3', '/rss.php'), # 2024-12-07 添加 /rss.php内容的支持 + ('feed', '/feed'), + ('feed2', '/feed.xml'), # 2024-07-26 添加 /feed.xml内容的支持 + ('feed3', '/feed/'), + ('feed4', '/feed.php'), # 2025-07-22 添加 /feed.php内容的支持 + ('index', '/index.xml') # 2024-07-25 添加 /index.xml内容的支持 + ] + + for feed_type, path in possible_feeds: + feed_url = blog_url.rstrip('/') + path + try: + response = session.get(feed_url, headers=HEADERS_XML, timeout=timeout) + if response.status_code == 200: + # 检查 Content-Type + content_type = response.headers.get('Content-Type', '').lower() + if 'xml' in content_type or 'rss' in content_type or 'atom' in content_type: + return [feed_type, feed_url] + + # 如果 Content-Type 是 text/html 或未明确，但内容本身是 RSS + text_head = response.text[:1000].lower() # 读取前1000字符 + if (' {feed_url}）：{e}") + parse_error = True + + # ---- 4. 如果缓存 RSS 无效则重新探测 ---- + if parse_error and source_used in ('cache', 'unknown'): + logging.info(f"缓存 RSS 无效，重新探测：{name} ({blog_url})。") + new_type, new_url = check_feed(blog_url, session) + if new_type != 'none' and new_url: + try: + feed_info = parse_feed(new_url, session, count, blog_url) + if isinstance(feed_info, dict) and 'articles' in feed_info: + articles = [ + { + 'title': a['title'], + 'created': a['published'], + 'link': a['link'], + 'author': name, + 'avatar': avatar, + } + for a in feed_info['articles'] + ] + + for a in articles: + logging.info(f"{name} 发布了新文章：{a['title']}，时间：{a['created']}，链接：{a['link']}") + + feed_type, feed_url, source_used = new_type, new_url, 'auto' + cache_update = {'action': 'set', 'name': name, 'url': new_url, 'reason': 'repair_cache'} + parse_error = False + except Exception as e: + logging.warning(f"重新探测解析仍失败：{name} ({new_url})：{e}") + cache_update = {'action': 'delete', 'name': name, 'url': None, 'reason': 'remove_invalid'} + feed_type, feed_url = 'none', None + else: + cache_update = {'action': 'delete', 'name': name, 'url': None, 'reason': 'remove_invalid'} + feed_type, feed_url = 'none', None + + # ---- 5. 最终状态 ---- + status = 'active' if articles else 'error' + if not articles: + if feed_type == 'none': + logging.warning(f"{name} 的博客 {blog_url} 未找到有效 RSS。") + else: + logging.warning(f"{name} 的 RSS {feed_url} 未解析出文章。") + + return { + 'name': name, + 'status': status, + 'articles': articles, + 'feed_url': feed_url, + 'feed_type': feed_type, + 'cache_update': cache_update, + 'source_used': source_used, + } + +def get_latest_articles_from_link(url, count=5, last_articles_path="./temp/newest_posts.json"): + """ + 从指定链接获取最新的文章数据并与本地存储的上次的文章数据进行对比。 + + 参数： + url (str): 用于获取文章数据的链接。 + count (int): 获取文章数的最大数。如果小于则全部获取，如果文章数大于则只取前 count 篇文章。 + + 返回： + list: 更新的文章列表，如果没有更新的文章则返回 None。 + """ + # 本地存储上次文章数据的文件 + local_file = last_articles_path + + # 检查和解析 feed + session = requests.Session() + feed_type, feed_url = check_feed(url, session) + if feed_type == 'none': + logging.error(f"无法获取 {url} 的文章数据") + return None + + # 获取最新的文章数据 + latest_data = parse_feed(feed_url, session ,count) + latest_articles = latest_data['articles'] + + # 读取本地存储的上次的文章数据 + if os.path.exists(local_file): + with open(local_file, 'r', encoding='utf-8') as file: + last_data = json.load(file) + else: + last_data = {'articles': []} + + last_articles = last_data['articles'] + + # 找到更新的文章 + updated_articles = [] + last_titles = {article['link'] for article in last_articles} + + for article in latest_articles: + if article['link'] not in last_titles: + updated_articles.append(article) + + logging.info(f"从 {url} 获取到 {len(latest_articles)} 篇文章，其中 {len(updated_articles)} 篇为新文章") + + # 更新本地存储的文章数据 + with open(local_file, 'w', encoding='utf-8') as file: + json.dump({'articles': latest_articles}, file, ensure_ascii=False, indent=4) + + # 如果有更新的文章，返回这些文章，否则返回 None + return updated_articles if updated_articles else None + diff --git a/push_rss_update/__init__.py b/friend_circle_lite/utils/__init__.py similarity index 100% rename from push_rss_update/__init__.py rename to friend_circle_lite/utils/__init__.py diff --git a/friend_circle_lite/utils/cache.py b/friend_circle_lite/utils/cache.py new file mode 100644 index 00000000000..9f685387ab0 --- /dev/null +++ b/friend_circle_lite/utils/cache.py @@ -0,0 +1,35 @@ +import logging +from friend_circle_lite.utils.json import read_json, write_json + +def load_cache(cache_file: str): + if not cache_file: + return [] + + data = read_json(cache_file) + if data is None: + logging.info(f"缓存文件 {cache_file} 不存在或无法读取，将自动创建。") + return [] + + if not isinstance(data, list): + logging.warning(f"缓存文件 {cache_file} 格式异常（应为列表）。将忽略。") + return [] + + norm = [] + for item in data: + if not isinstance(item, dict): + continue + name = item.get('name') + url = item.get('url') + if name and url: + norm.append({'name': name, 'url': url, 'source': 'cache'}) + return norm + +def save_cache(cache_file: str, cache_items: list[dict]): + if not cache_file: + return + + out = [{'name': i['name'], 'url': i['url']} for i in cache_items] + if write_json(cache_file, out): + logging.info(f"缓存已保存到 {cache_file}（{len(out)} 条）。") + else: + logging.error(f"保存缓存文件 {cache_file} 失败") diff --git a/friend_circle_lite/utils/config.py b/friend_circle_lite/utils/config.py new file mode 100644 index 00000000000..babf81cde91 --- /dev/null +++ b/friend_circle_lite/utils/config.py @@ -0,0 +1,25 @@ +import yaml +import logging + +def load_config(config_file): + """ + 加载配置文件。 + + 参数： + config_file (str): 配置文件的路径。 + + 返回： + dict: 加载的配置数据。 + """ + try: + with open(config_file, 'r', encoding='utf-8') as file: + return yaml.safe_load(file) + except FileNotFoundError: + logging.error(f"配置文件 {config_file} 未找到") + return {} + except yaml.YAMLError as e: + logging.error(f"YAML解析错误: {str(e)}") + return {} + except Exception as e: + logging.error(f"加载配置文件时发生未知错误: {str(e)}") + return {} diff --git a/friend_circle_lite/utils/github.py b/friend_circle_lite/utils/github.py new file mode 100644 index 00000000000..2cbdc50a196 --- /dev/null +++ b/friend_circle_lite/utils/github.py @@ -0,0 +1,39 @@ +import logging +import requests +import re +from friend_circle_lite import HEADERS_JSON + +def extract_emails_from_issues(api_url): + """ + 从GitHub issues API中提取以[e-mail]开头的title中的邮箱地址。 + + 参数： + api_url (str): GitHub issues API的URL。 + + 返回： + dict: 包含所有提取的邮箱地址的字典。 + { + "emails": [ + "3162475700@qq.com" + ] + } + """ + try: + response = requests.get(api_url, headers=HEADERS_JSON, timeout=10) + response.raise_for_status() + issues = response.json() + except Exception as e: + logging.error(f"无法获取 GitHub issues 数据，错误信息: {e}") + return None + + email_pattern = re.compile(r'^\[邮箱订阅\](.+)$') + emails = [] + + for issue in issues: + title = issue.get("title", "") + match = email_pattern.match(title) + if match: + email = match.group(1).strip() + emails.append(email) + + return {"emails": emails} \ No newline at end of file diff --git a/friend_circle_lite/utils/json.py b/friend_circle_lite/utils/json.py new file mode 100644 index 00000000000..49b46570ea5 --- /dev/null +++ b/friend_circle_lite/utils/json.py @@ -0,0 +1,30 @@ +import json +import logging +from pathlib import Path +from typing import Any, Optional + +def read_json(file_path: str | Path) -> Optional[dict | list]: + """安全读取 JSON 文件，如果文件不存在或格式错误则返回 None""" + try: + with open(file_path, 'r', encoding='utf-8') as f: + return json.load(f) + except FileNotFoundError: + logging.warning(f"文件不存在: {file_path}") + return None + except json.JSONDecodeError: + logging.warning(f"JSON 格式错误: {file_path}") + return None + except Exception as e: + logging.warning(f"读取 JSON 文件时发生错误: {file_path}, 错误信息: {str(e)}") + return None + +def write_json(file_path: str | Path, data: Any) -> bool: + """安全写入 JSON 文件，返回是否写入成功""" + try: + Path(file_path).parent.mkdir(parents=True, exist_ok=True) + with open(file_path, 'w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, indent=2) + return True + except Exception as e: + logging.warning(f"写入 JSON 文件时发生错误: {file_path}, 错误信息: {str(e)}") + return False diff --git a/push_rss_update/send_email.py b/friend_circle_lite/utils/mail.py similarity index 99% rename from push_rss_update/send_email.py rename to friend_circle_lite/utils/mail.py index 485a082b840..bb7df1e8b92 100644 --- a/push_rss_update/send_email.py +++ b/friend_circle_lite/utils/mail.py @@ -1,6 +1,5 @@ import logging import smtplib -import socket import time import os from email.mime.multipart import MIMEMultipart diff --git a/friend_circle_lite/utils/time.py b/friend_circle_lite/utils/time.py new file mode 100644 index 00000000000..fc55391b3e8 --- /dev/null +++ b/friend_circle_lite/utils/time.py @@ -0,0 +1,42 @@ +import logging +from dateutil import parser +from datetime import datetime, timezone, timedelta + +def format_published_time(time_str): + """ + 格式化发布时间为统一格式 YYYY-MM-DD HH:MM + + 参数: + time_str (str): 输入的时间字符串，可能是多种格式。 + + 返回: + str: 格式化后的时间字符串，若解析失败返回空字符串。 + """ + # 尝试自动解析输入时间字符串 + try: + parsed_time = parser.parse(time_str, fuzzy=True) + except (ValueError, parser.ParserError): + # 定义支持的时间格式 + time_formats = [ + '%a, %d %b %Y %H:%M:%S %z', # Mon, 11 Mar 2024 14:08:32 +0000 + '%a, %d %b %Y %H:%M:%S GMT', # Wed, 19 Jun 2024 09:43:53 GMT + '%Y-%m-%dT%H:%M:%S%z', # 2024-03-11T14:08:32+00:00 + '%Y-%m-%dT%H:%M:%SZ', # 2024-03-11T14:08:32Z + '%Y-%m-%d %H:%M:%S', # 2024-03-11 14:08:32 + '%Y-%m-%d' # 2024-03-11 + ] + for fmt in time_formats: + try: + parsed_time = datetime.strptime(time_str, fmt) + break + except ValueError: + continue + else: + logging.warning(f"无法解析时间字符串：{time_str}") + return '' + + # 处理时区转换 + if parsed_time.tzinfo is None: + parsed_time = parsed_time.replace(tzinfo=timezone.utc) + shanghai_time = parsed_time.astimezone(timezone(timedelta(hours=8))) + return shanghai_time.strftime('%Y-%m-%d %H:%M') \ No newline at end of file diff --git a/friend_circle_lite/utils/url.py b/friend_circle_lite/utils/url.py new file mode 100644 index 00000000000..fa9791791da --- /dev/null +++ b/friend_circle_lite/utils/url.py @@ -0,0 +1,27 @@ +import logging +from urllib.parse import urlparse, urljoin +import re + +def replace_non_domain(link: str, blog_url: str) -> str: + """ + 暂未实现 + 检测并替换字符串中的非正常域名部分（如 IP 地址或 localhost），替换为 blog_url。 + 替换后强制使用 https，且考虑 blog_url 尾部是否有斜杠。 + + :param link: 原始地址字符串 + :param blog_url: 替换为的博客地址 + :return: 替换后的地址字符串 + """ + try: + parsed = urlparse(link) + if 'localhost' in parsed.netloc or re.match(r'^\d{1,3}(\.\d{1,3}){3}$', parsed.netloc): # IP地址或localhost + # 提取 path + query + path = parsed.path or '/' + if parsed.query: + path += '?' + parsed.query + return urljoin(blog_url.rstrip('/') + '/', path.lstrip('/')) + else: + return link # 合法域名则返回原链接 + except Exception as e: + logging.warning(f"替换链接时出错：{link}, error: {e}") + return link diff --git a/main/fclite.js b/main/fclite.js index 453a73904ae..ac6c12723ca 100644 --- a/main/fclite.js +++ b/main/fclite.js @@ -4,8 +4,8 @@ function initialize_fc_lite() { // 设置默认配置 UserConfig = { private_api_url: UserConfig?.private_api_url || "", - page_turning_number: UserConfig?.page_turning_number || 20, // 默认20篇 - error_img: UserConfig?.error_img || "https://fastly.jsdelivr.net/gh/willow-god/Friend-Circle-Lite@latest/static/favicon.ico" // 默认头像 + page_turning_number: UserConfig?.page_turning_number || 24, // 默认24篇 + error_img: UserConfig?.error_img || "https://fastly.jsdelivr.net/gh/willow-god/Friend-Circle-Lite/static/favicon.ico" // 默认头像 }; const root = document.getElementById('friend-circle-lite-root'); @@ -235,4 +235,4 @@ function whenDOMReady() { } whenDOMReady(); -document.addEventListener("pjax:complete", initialize_fc_lite); +document.addEventListener("pjax:complete", initialize_fc_lite); \ No newline at end of file diff --git a/rss_subscribe/email_template.html b/push_templates/default.html similarity index 100% rename from rss_subscribe/email_template.html rename to push_templates/default.html diff --git a/rss_subscribe/__init__.py b/rss_subscribe/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/rss_subscribe/push_article_update.py b/rss_subscribe/push_article_update.py deleted file mode 100644 index 813b2c3ac96..00000000000 --- a/rss_subscribe/push_article_update.py +++ /dev/null @@ -1,105 +0,0 @@ -import logging -import requests -import re -from friend_circle_lite.get_info import check_feed, parse_feed -import json -import os - -# 标准化的请求头 -HEADERS_JSON = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/123.0.0.0 Safari/537.36 " - "(Friend-Circle-Lite/1.0; +https://github.com/willow-god/Friend-Circle-Lite)" - ), - "X-Friend-Circle": "1.0" -} - - -def extract_emails_from_issues(api_url): - """ - 从GitHub issues API中提取以[e-mail]开头的title中的邮箱地址。 - - 参数： - api_url (str): GitHub issues API的URL。 - - 返回： - dict: 包含所有提取的邮箱地址的字典。 - { - "emails": [ - "3162475700@qq.com" - ] - } - """ - try: - response = requests.get(api_url, headers=HEADERS_JSON, timeout=10) - response.raise_for_status() - issues = response.json() - except Exception as e: - logging.error(f"无法获取 GitHub issues 数据，错误信息: {e}") - return None - - email_pattern = re.compile(r'^\[邮箱订阅\](.+)$') - emails = [] - - for issue in issues: - title = issue.get("title", "") - match = email_pattern.match(title) - if match: - email = match.group(1).strip() - emails.append(email) - - return {"emails": emails} - -def get_latest_articles_from_link(url, count=5, last_articles_path="./temp/newest_posts.json"): - """ - 从指定链接获取最新的文章数据并与本地存储的上次的文章数据进行对比。 - - 参数： - url (str): 用于获取文章数据的链接。 - count (int): 获取文章数的最大数。如果小于则全部获取，如果文章数大于则只取前 count 篇文章。 - - 返回： - list: 更新的文章列表，如果没有更新的文章则返回 None。 - """ - # 本地存储上次文章数据的文件 - local_file = last_articles_path - - # 检查和解析 feed - session = requests.Session() - feed_type, feed_url = check_feed(url, session) - if feed_type == 'none': - logging.error(f"无法获取 {url} 的文章数据") - return None - - # 获取最新的文章数据 - latest_data = parse_feed(feed_url, session ,count) - latest_articles = latest_data['articles'] - - # 读取本地存储的上次的文章数据 - if os.path.exists(local_file): - with open(local_file, 'r', encoding='utf-8') as file: - last_data = json.load(file) - else: - last_data = {'articles': []} - - last_articles = last_data['articles'] - - # 找到更新的文章 - updated_articles = [] - last_titles = {article['link'] for article in last_articles} - - for article in latest_articles: - if article['link'] not in last_titles: - updated_articles.append(article) - - logging.info(f"从 {url} 获取到 {len(latest_articles)} 篇文章，其中 {len(updated_articles)} 篇为新文章") - - # 更新本地存储的文章数据 - with open(local_file, 'w', encoding='utf-8') as file: - json.dump({'articles': latest_articles}, file, ensure_ascii=False, indent=4) - - # 如果有更新的文章，返回这些文章，否则返回 None - return updated_articles if updated_articles else None - diff --git a/run.py b/run.py index 8816cc9b0fc..c2f0b5f128b 100644 --- a/run.py +++ b/run.py @@ -1,20 +1,13 @@ import logging -import json import sys import os -from friend_circle_lite.get_info import ( - fetch_and_process_data, - marge_data_from_json_url, - marge_errors_from_json_url, - deal_with_large_data -) -from friend_circle_lite.get_conf import load_config -from rss_subscribe.push_article_update import ( - get_latest_articles_from_link, - extract_emails_from_issues -) -from push_rss_update.send_email import send_emails +from friend_circle_lite.all_friends import fetch_and_process_data, marge_data_from_json_url, marge_errors_from_json_url, deal_with_large_data +from friend_circle_lite.utils.json import write_json +from friend_circle_lite.utils.config import load_config +from friend_circle_lite.utils.mail import send_emails +from friend_circle_lite.single_friend import get_latest_articles_from_link +from friend_circle_lite.utils.github import extract_emails_from_issues # ========== 日志设置 ========== logging.basicConfig( @@ -22,42 +15,44 @@ format='😋 %(levelname)s: %(message)s' ) +# ========== 加载环境变量 ========== +if os.getenv("GITHUB_TOKEN") is None: + from dotenv import load_dotenv + load_dotenv() + # ========== 加载配置 ========== config = load_config("./conf.yaml") # ========== 爬虫模块 ========== if config["spider_settings"]["enable"]: + logging.info("✅ 爬虫已启用") - json_url = config['spider_settings']['json_url'] article_count = config['spider_settings']['article_count'] specific_rss = config['specific_RSS'] logging.info(f"📥 正在从 {json_url} 获取数据，每个博客获取 {article_count} 篇文章") result, lost_friends = fetch_and_process_data( - json_url=json_url, - specific_RSS=specific_rss, - count=article_count, - cache_file="./temp/cache.json" - ) # type: ignore + json_url = json_url, # 包含朋友信息的 JSON 文件的 URL。 + specific_RSS = specific_rss, # 包含特定 RSS 源的字典列表 [{name, url}]（来自 YAML）。 + count = article_count, # 获取每个博客的最大文章数。 + cache_file = "./temp/cache.json" # 缓存文件路径。 + ) if config["spider_settings"]["merge_result"]["enable"]: + merge_url = config['spider_settings']["merge_result"]['merge_json_url'] logging.info(f"🔀 合并功能开启，从 {merge_url} 获取外部数据") - result = marge_data_from_json_url(result, f"{merge_url}/all.json") lost_friends = marge_errors_from_json_url(lost_friends, f"{merge_url}/errors.json") article_count = len(result.get("article_data", [])) - logging.info(f"📦 数据获取完毕，共有 {article_count} 位好友的动态，正在处理数据") + logging.info(f"📦 数据获取完毕，共有 {article_count} 篇文章，正在处理数据") result = deal_with_large_data(result) - with open("all.json", "w", encoding="utf-8") as f: - json.dump(result, f, ensure_ascii=False, indent=2) - - with open("errors.json", "w", encoding="utf-8") as f: - json.dump(lost_friends, f, ensure_ascii=False, indent=2) + write_json("./all.json", result) + write_json("./errors.json", lost_friends) # ========== 邮箱推送准备 ========== SMTP_isReady = False diff --git a/server.py b/server.py index 2fb50f4dc2d..5616ba1fe55 100644 --- a/server.py +++ b/server.py @@ -5,9 +5,6 @@ import json import random -from friend_circle_lite.get_info import fetch_and_process_data, sort_articles_by_time -from friend_circle_lite.get_conf import load_config - app = FastAPI() # 设置静态文件目录 From 5a6d164023b6ef0034527120022e46093127a00b Mon Sep 17 00:00:00 2001 From: LiuShen <3162475700@qq.com> Date: Sat, 26 Jul 2025 19:41:36 +0800 Subject: [PATCH 02/30] =?UTF-8?q?=F0=9F=99=82=E6=9A=82=E6=97=B6=E6=B3=A8?= =?UTF-8?q?=E9=87=8A=E5=8A=A0=E8=BD=BD=E7=8E=AF=E5=A2=83=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E7=9A=84=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/run.py b/run.py index c2f0b5f128b..30517f35721 100644 --- a/run.py +++ b/run.py @@ -16,9 +16,9 @@ ) # ========== 加载环境变量 ========== -if os.getenv("GITHUB_TOKEN") is None: - from dotenv import load_dotenv - load_dotenv() +# if os.getenv("GITHUB_TOKEN") is None: +# from dotenv import load_dotenv +# load_dotenv() # ========== 加载配置 ========== config = load_config("./conf.yaml") From cb55a708febb4d0bdf6c83d250ba2267c831cc1f Mon Sep 17 00:00:00 2001 From: LiuShen <3162475700@qq.com> Date: Sat, 26 Jul 2025 20:00:16 +0800 Subject: [PATCH 03/30] =?UTF-8?q?=F0=9F=98=98=E5=B0=86cache.json=E4=B9=9F?= =?UTF-8?q?=E6=98=A0=E5=B0=84=E5=87=BA=E6=9D=A5=E6=96=B9=E4=BE=BF=E5=88=86?= =?UTF-8?q?=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/friend_circle_lite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/friend_circle_lite.yml b/.github/workflows/friend_circle_lite.yml index f215e5dd67a..c8d42f4f889 100644 --- a/.github/workflows/friend_circle_lite.yml +++ b/.github/workflows/friend_circle_lite.yml @@ -72,7 +72,7 @@ jobs: - name: Commit changes run: | mkdir pages - cp -r main ./static/index.html ./static/readme.md ./static/favicon.ico ./static/bg-light.webp ./static/bg-dark.webp all.json errors.json pages/ + cp -r main ./static/index.html ./static/readme.md ./static/favicon.ico ./static/bg-light.webp ./temp/cache.json ./static/bg-dark.webp all.json errors.json pages/ cd pages git init git add . From 94d2a83c8636a9ac2718fd8aad6308dec8a605f5 Mon Sep 17 00:00:00 2001 From: LiuShen <3162475700@qq.com> Date: Sun, 12 Oct 2025 16:58:45 +0800 Subject: [PATCH 04/30] =?UTF-8?q?=F0=9F=A4=94=E4=BF=AE=E6=94=B9=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E4=B8=AD=E7=9A=84=E6=9B=B4=E6=96=B0=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index 6ff5caddcec..80cfcc3012b 100644 --- a/readme.md +++ b/readme.md @@ -11,7 +11,7 @@ ## 开发进度 -### 2026-07-23 +### 2025-07-23 * 添加缓存文件，防止由于缓存导致多次请求 * 添加feed.php后缀的适配 From 78b4a483e1acbd41d54ef96884d52723bbae6953 Mon Sep 17 00:00:00 2001 From: LiuShen <3162475700@qq.com> Date: Wed, 26 Nov 2025 23:33:26 +0800 Subject: [PATCH 05/30] =?UTF-8?q?=F0=9F=99=82=E6=B7=BB=E5=8A=A0edgeone=20p?= =?UTF-8?q?age=E8=B7=A8=E5=9F=9F=E9=85=8D=E7=BD=AE=EF=BC=8C=E9=98=B2?= =?UTF-8?q?=E6=AD=A2=E7=94=B1=E4=BA=8E=E8=B7=A8=E5=9F=9F=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E6=97=A0=E6=B3=95=E5=9C=A8=E5=89=8D=E7=AB=AF=E5=8A=A0=E8=BD=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/friend_circle_lite.yml | 2 +- .gitignore | 1 + static/edgeone.json | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 static/edgeone.json diff --git a/.github/workflows/friend_circle_lite.yml b/.github/workflows/friend_circle_lite.yml index c8d42f4f889..ad36365c1a4 100644 --- a/.github/workflows/friend_circle_lite.yml +++ b/.github/workflows/friend_circle_lite.yml @@ -72,7 +72,7 @@ jobs: - name: Commit changes run: | mkdir pages - cp -r main ./static/index.html ./static/readme.md ./static/favicon.ico ./static/bg-light.webp ./temp/cache.json ./static/bg-dark.webp all.json errors.json pages/ + cp -r main ./static/edgeone.json ./static/index.html ./static/readme.md ./static/favicon.ico ./static/bg-light.webp ./temp/cache.json ./static/bg-dark.webp all.json errors.json pages/ cd pages git init git add . diff --git a/.gitignore b/.gitignore index e52e7fb4080..a6ab3cec929 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ # 忽略数据文件 *.json +!edgeone.json *.bat diff --git a/static/edgeone.json b/static/edgeone.json new file mode 100644 index 00000000000..1d4df629b8d --- /dev/null +++ b/static/edgeone.json @@ -0,0 +1,21 @@ +{ + "headers": [ + { + "source": "/*", + "headers": [ + { + "key": "Access-Control-Allow-Origin", + "value": "*" + }, + { + "key": "Access-Control-Allow-Methods", + "value": "GET, POST, OPTIONS, PUT, DELETE" + }, + { + "key": "Access-Control-Allow-Headers", + "value": "*" + } + ] + } + ] +} From 822fc42a532861547d684f3146b8dbe6a1286c58 Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Sun, 15 Mar 2026 22:31:19 +0800 Subject: [PATCH 06/30] =?UTF-8?q?=F0=9F=98=98=E6=B7=BB=E5=8A=A0Netlify?= =?UTF-8?q?=E8=B7=A8=E5=9F=9F=E8=AE=BE=E7=BD=AE=EF=BC=8C=E8=A7=A3=E5=86=B3?= =?UTF-8?q?=E7=94=B1=E4=BA=8E=E9=83=A8=E5=88=86=E6=96=87=E7=AB=A0=E8=B6=85?= =?UTF-8?q?=E6=9C=9F=E8=BF=87=E4=BA=8E=E4=B8=A5=E9=87=8D=E5=AF=BC=E8=87=B4?= =?UTF-8?q?=E5=A7=8B=E7=BB=88=E6=98=BE=E7=A4=BA=E5=9C=A8=E6=A0=87=E5=A4=B4?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98(#66)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/friend_circle_lite.yml | 12 +++++--- friend_circle_lite/all_friends.py | 38 +++++++++++++++++++----- run.py | 5 +++- static/_headers | 4 +++ 4 files changed, 46 insertions(+), 13 deletions(-) create mode 100644 static/_headers diff --git a/.github/workflows/friend_circle_lite.yml b/.github/workflows/friend_circle_lite.yml index ad36365c1a4..9172c381e1e 100644 --- a/.github/workflows/friend_circle_lite.yml +++ b/.github/workflows/friend_circle_lite.yml @@ -7,6 +7,7 @@ on: env: TZ: Asia/Shanghai + PAGE_BRANCH: page jobs: friend-circle-lite: @@ -69,15 +70,18 @@ jobs: git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' - - name: Commit changes + - name: Build static publish directory run: | mkdir pages - cp -r main ./static/edgeone.json ./static/index.html ./static/readme.md ./static/favicon.ico ./static/bg-light.webp ./temp/cache.json ./static/bg-dark.webp all.json errors.json pages/ + cp -r main ./static/edgeone.json ./static/_headers ./static/index.html ./static/readme.md ./static/favicon.ico ./static/bg-light.webp ./temp/cache.json ./static/bg-dark.webp all.json errors.json pages/ + + - name: Publish static assets to branches + run: | cd pages git init git add . git commit -m "⏱️ $(date +"%Y年%m月%d日-%H时%M分") GitHub Actions定时更新" - git push --force https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git HEAD:page + git push --force https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git HEAD:${{ env.PAGE_BRANCH }} - name: Delete Workflow Runs uses: Mattraks/delete-workflow-runs@v2 @@ -93,4 +97,4 @@ jobs: permissions: actions: write steps: - - uses: liskin/gh-workflow-keepalive@v1 \ No newline at end of file + - uses: liskin/gh-workflow-keepalive@v1 diff --git a/friend_circle_lite/all_friends.py b/friend_circle_lite/all_friends.py index 956f1aa4708..9c50bd7df9c 100644 --- a/friend_circle_lite/all_friends.py +++ b/friend_circle_lite/all_friends.py @@ -1,7 +1,7 @@ import string import requests import logging -from datetime import datetime +from datetime import datetime, timedelta from zoneinfo import ZoneInfo import requests from concurrent.futures import ThreadPoolExecutor, as_completed @@ -146,12 +146,13 @@ def fetch_and_process_data(json_url: str, specific_RSS: list = None, count: int return result, error_friends_info -def sort_articles_by_time(data): +def sort_articles_by_time(data, future_tolerance_days=2): """ - 对文章数据按时间排序 + 对文章数据按时间排序，并过滤严重超前于当前时间的文章 参数： data (dict): 包含文章信息的字典 + future_tolerance_days (int): 允许文章发布时间最多超前当前时间的天数 返回： dict: 按时间排序后的文章信息字典 @@ -162,14 +163,33 @@ def sort_articles_by_time(data): article['created'] = '2024-01-01 00:00' # 输出警告信息 logging.warning(f"文章 {article['title']} 未包含时间信息，已设置为默认时间 2024-01-01 00:00") - + + now = datetime.now(ZoneInfo("Asia/Shanghai")).replace(tzinfo=None) + max_allowed_time = now + timedelta(days=future_tolerance_days) + if 'article_data' in data: + filtered_articles = [] + removed_count = 0 + + for article in data['article_data']: + article_time = datetime.strptime(article['created'], '%Y-%m-%d %H:%M') + if article_time > max_allowed_time: + removed_count += 1 + logging.warning( + f"文章 {article['title']} 的时间 {article['created']} 超出当前时间 {future_tolerance_days} 天以上，已跳过显示" + ) + continue + filtered_articles.append(article) + sorted_articles = sorted( - data['article_data'], + filtered_articles, key=lambda x: datetime.strptime(x['created'], '%Y-%m-%d %H:%M'), reverse=True ) data['article_data'] = sorted_articles + + if removed_count: + logging.info(f"已过滤 {removed_count} 篇未来时间异常的文章") return data def marge_data_from_json_url(data, marge_json_url): @@ -225,18 +245,20 @@ def marge_errors_from_json_url(errors, marge_json_url): logging.info(f"合并错误信息完成，合并后共有 {len(filtered_errors)} 位朋友") return filtered_errors -def deal_with_large_data(result): +def deal_with_large_data(result, future_tolerance_days=2): """ 处理文章数据，保留前150篇及其作者在后续文章中的出现。 参数： result (dict): 包含统计数据和文章数据的字典。 + future_tolerance_days (int): 允许文章发布时间最多超前当前时间的天数。 返回： dict: 处理后的数据，只包含需要的文章。 """ - result = sort_articles_by_time(result) + result = sort_articles_by_time(result, future_tolerance_days=future_tolerance_days) article_data = result.get("article_data", []) + result["statistical_data"]["article_num"] = len(article_data) # 检查文章数量是否大于 150 max_articles = 150 @@ -257,4 +279,4 @@ def deal_with_large_data(result): result["statistical_data"]["article_num"] = len(filtered_articles) logging.info(f"数据处理完成，保留 {len(filtered_articles)} 篇文章") - return result \ No newline at end of file + return result diff --git a/run.py b/run.py index 30517f35721..dcafda771b9 100644 --- a/run.py +++ b/run.py @@ -9,6 +9,8 @@ from friend_circle_lite.single_friend import get_latest_articles_from_link from friend_circle_lite.utils.github import extract_emails_from_issues +FUTURE_ARTICLE_TOLERANCE_DAYS = 2 + # ========== 日志设置 ========== logging.basicConfig( level=logging.INFO, @@ -49,7 +51,8 @@ article_count = len(result.get("article_data", [])) logging.info(f"📦 数据获取完毕，共有 {article_count} 篇文章，正在处理数据") - result = deal_with_large_data(result) + future_tolerance_days = FUTURE_ARTICLE_TOLERANCE_DAYS + result = deal_with_large_data(result, future_tolerance_days=future_tolerance_days) write_json("./all.json", result) write_json("./errors.json", lost_friends) diff --git a/static/_headers b/static/_headers new file mode 100644 index 00000000000..97f1890683d --- /dev/null +++ b/static/_headers @@ -0,0 +1,4 @@ +/* + Access-Control-Allow-Origin: * + Access-Control-Allow-Methods: GET, POST, OPTIONS, PUT, DELETE + Access-Control-Allow-Headers: * \ No newline at end of file From 498df0377464ce6898ad40a3fbfa3858065d54a7 Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Wed, 15 Apr 2026 19:35:16 +0800 Subject: [PATCH 07/30] =?UTF-8?q?=F0=9F=98=81=E5=88=9D=E6=AD=A5=E9=87=8D?= =?UTF-8?q?=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/friend_circle_lite.yml | 5 +- friend_circle_lite/all_friends.py | 244 ++-------------- friend_circle_lite/app_config.py | 160 ++++++++++ friend_circle_lite/application.py | 197 +++++++++++++ friend_circle_lite/cache_store.py | 166 +++++++++++ friend_circle_lite/crawler_service.py | 272 +++++++++++++++++ friend_circle_lite/feed_service.py | 154 ++++++++++ friend_circle_lite/models.py | 164 +++++++++++ friend_circle_lite/single_friend.py | 355 ++++++----------------- friend_circle_lite/utils/cache.py | 49 ++-- friend_circle_lite/utils/config.py | 28 +- requirements.txt | 3 +- run.py | 170 ++--------- 13 files changed, 1282 insertions(+), 685 deletions(-) create mode 100644 friend_circle_lite/app_config.py create mode 100644 friend_circle_lite/application.py create mode 100644 friend_circle_lite/cache_store.py create mode 100644 friend_circle_lite/crawler_service.py create mode 100644 friend_circle_lite/feed_service.py create mode 100644 friend_circle_lite/models.py diff --git a/.github/workflows/friend_circle_lite.yml b/.github/workflows/friend_circle_lite.yml index 9172c381e1e..f08ffe73190 100644 --- a/.github/workflows/friend_circle_lite.yml +++ b/.github/workflows/friend_circle_lite.yml @@ -12,6 +12,9 @@ env: jobs: friend-circle-lite: runs-on: ubuntu-latest + permissions: + contents: write + actions: write steps: - name: Pull Latest Repository @@ -73,7 +76,7 @@ jobs: - name: Build static publish directory run: | mkdir pages - cp -r main ./static/edgeone.json ./static/_headers ./static/index.html ./static/readme.md ./static/favicon.ico ./static/bg-light.webp ./temp/cache.json ./static/bg-dark.webp all.json errors.json pages/ + cp -r main ./static/edgeone.json ./static/_headers ./static/index.html ./static/readme.md ./static/favicon.ico ./static/bg-light.webp ./static/bg-dark.webp all.json errors.json pages/ - name: Publish static assets to branches run: | diff --git a/friend_circle_lite/all_friends.py b/friend_circle_lite/all_friends.py index 9c50bd7df9c..e24d9cdb7b1 100644 --- a/friend_circle_lite/all_friends.py +++ b/friend_circle_lite/all_friends.py @@ -1,196 +1,32 @@ -import string -import requests -import logging -from datetime import datetime, timedelta -from zoneinfo import ZoneInfo -import requests -from concurrent.futures import ThreadPoolExecutor, as_completed -from friend_circle_lite.utils.cache import load_cache, save_cache -from friend_circle_lite.single_friend import process_friend -from friend_circle_lite import HEADERS_JSON, timeout - -def fetch_and_process_data(json_url: str, specific_RSS: list = None, count: int = 5, cache_file: str = None): - """ - 读取 JSON 数据并处理订阅信息，返回统计数据和文章信息。 - - 参数： - json_url (str): 包含朋友信息的 JSON 文件的 URL。 - count (int): 获取每个博客的最大文章数。 - specific_RSS (list): 包含特定 RSS 源的字典列表 [{name, url}]（来自 YAML）。 - cache_file (str): 缓存文件路径。 - - 返回： - (result_dict, error_friends_info_list) - """ - if specific_RSS is None: - specific_RSS = [] - - # 1. 加载缓存 - cache_list = load_cache(cache_file) - - # 2. 标记 YAML 条目 - manual_list = [] - for item in specific_RSS: - if isinstance(item, dict) and 'name' in item and 'url' in item: - manual_list.append({'name': item['name'], 'url': item['url'], 'source': 'manual'}) - - # 3. 合并（缓存先，YAML 后覆盖） - combined_map = {e['name']: e for e in cache_list} - for e in manual_list: # 手动优先 - combined_map[e['name']] = e - specific_and_cache = list(combined_map.values()) - - # 4. 建立方便判断的集合：手动源名称集合 - manual_name_set = {e['name'] for e in manual_list} - - # 5. 获取朋友列表 - session = requests.Session() - try: - response = session.get(json_url, headers=HEADERS_JSON, timeout=timeout) - friends_data = response.json() - except Exception as e: - logging.error(f"无法获取链接：{json_url} ：{e}", exc_info=True) - return None - - friends = friends_data.get('friends', []) - total_friends = len(friends) - active_friends = 0 - error_friends = 0 - total_articles = 0 - article_data = [] - error_friends_info = [] - cache_updates = [] # 用于收集缓存更新（线程安全：用局部列表 + 合并） - - # 6. 并发处理 - with ThreadPoolExecutor(max_workers=10) as executor: - future_to_friend = { - executor.submit(process_friend, friend, session, count, specific_and_cache): friend - for friend in friends - } - - for future in as_completed(future_to_friend): - friend = future_to_friend[future] - try: - result = future.result() - - # 拿回缓存更新意图 - upd = result.get('cache_update', {}) - if upd and upd.get('action') != 'none': - cache_updates.append(upd) - - if result['status'] == 'active': - active_friends += 1 - article_data.extend(result['articles']) - total_articles += len(result['articles']) - else: - error_friends += 1 - error_friends_info.append(friend) - - except Exception as e: - logging.error(f"处理 {friend} 时发生错误: {e}", exc_info=True) - error_friends += 1 - error_friends_info.append(friend) - - # 7. 处理缓存更新 - cache_map = {e['name']: e for e in cache_list} - - # 去重 & 过滤无效条目 - unique_updates = {} - for upd in cache_updates: - name = upd.get('name') - action = upd.get('action') - url = upd.get('url') - if not name: - continue - - # 过滤手动 YAML 的条目（不允许覆盖） - if name in manual_name_set: - continue +"""Legacy-compatible crawl entrypoints. - # 只缓存有效 RSS 地址 - if action == 'set': - if url and url != 'none' and url != '': - unique_updates[name] = {'action': 'set', 'url': url, 'reason': upd.get('reason', '')} - elif action == 'delete': - unique_updates[name] = {'action': 'delete', 'url': None, 'reason': upd.get('reason', '')} +The internal implementation is now delegated to `crawler_service`, but these +functions keep the existing public API stable for `run.py` and external users. +""" - # 应用缓存更新 - for name, upd in unique_updates.items(): - if upd['action'] == 'set': - cache_map[name] = {'name': name, 'url': upd['url'], 'source': 'cache'} - logging.info(f"缓存更新：SET {name} -> {upd['url']} ({upd['reason']})") - elif upd['action'] == 'delete': - if name in cache_map: - cache_map.pop(name) - logging.info(f"缓存更新：DELETE {name} ({upd['reason']})") - - # 8. 保存缓存 - save_cache(cache_file, list(cache_map.values())) +import logging - # 9. 汇总统计 - result = { - 'statistical_data': { - 'friends_num': total_friends, - 'active_num': active_friends, - 'error_num': error_friends, - 'article_num': total_articles, - 'last_updated_time': datetime.now(ZoneInfo("Asia/Shanghai")).strftime('%Y-%m-%d %H:%M:%S'), - }, - 'article_data': article_data, - } +import requests - logging.info( - f"数据处理完成，总共有 {total_friends} 位朋友，其中 {active_friends} 位博客可访问，" - f"{error_friends} 位博客无法访问。缓存更新 {len(unique_updates)} 条。" - ) +from friend_circle_lite import HEADERS_JSON, timeout +from friend_circle_lite.crawler_service import ( + FriendCircleCrawler, + limit_large_dataset as _limit_large_dataset, + sort_articles_by_time as _sort_articles_by_time, +) - return result, error_friends_info +def fetch_and_process_data(json_url: str, specific_RSS: list = None, count: int = 5, cache_file: str = None): + """Legacy wrapper around the new crawler orchestration service.""" + return FriendCircleCrawler( + json_url=json_url, + count=count, + specific_rss=specific_RSS, + cache_file=cache_file, + ).run() def sort_articles_by_time(data, future_tolerance_days=2): - """ - 对文章数据按时间排序，并过滤严重超前于当前时间的文章 - - 参数： - data (dict): 包含文章信息的字典 - future_tolerance_days (int): 允许文章发布时间最多超前当前时间的天数 - - 返回： - dict: 按时间排序后的文章信息字典 - """ - # 先确保每个元素存在时间 - for article in data['article_data']: - if article['created'] == '' or article['created'] == None: - article['created'] = '2024-01-01 00:00' - # 输出警告信息 - logging.warning(f"文章 {article['title']} 未包含时间信息，已设置为默认时间 2024-01-01 00:00") - - now = datetime.now(ZoneInfo("Asia/Shanghai")).replace(tzinfo=None) - max_allowed_time = now + timedelta(days=future_tolerance_days) - - if 'article_data' in data: - filtered_articles = [] - removed_count = 0 - - for article in data['article_data']: - article_time = datetime.strptime(article['created'], '%Y-%m-%d %H:%M') - if article_time > max_allowed_time: - removed_count += 1 - logging.warning( - f"文章 {article['title']} 的时间 {article['created']} 超出当前时间 {future_tolerance_days} 天以上，已跳过显示" - ) - continue - filtered_articles.append(article) - - sorted_articles = sorted( - filtered_articles, - key=lambda x: datetime.strptime(x['created'], '%Y-%m-%d %H:%M'), - reverse=True - ) - data['article_data'] = sorted_articles - - if removed_count: - logging.info(f"已过滤 {removed_count} 篇未来时间异常的文章") - return data + """Legacy wrapper around the refactored sort helper.""" + return _sort_articles_by_time(data, future_tolerance_days=future_tolerance_days) def marge_data_from_json_url(data, marge_json_url): """ @@ -246,37 +82,5 @@ def marge_errors_from_json_url(errors, marge_json_url): return filtered_errors def deal_with_large_data(result, future_tolerance_days=2): - """ - 处理文章数据，保留前150篇及其作者在后续文章中的出现。 - - 参数： - result (dict): 包含统计数据和文章数据的字典。 - future_tolerance_days (int): 允许文章发布时间最多超前当前时间的天数。 - - 返回： - dict: 处理后的数据，只包含需要的文章。 - """ - result = sort_articles_by_time(result, future_tolerance_days=future_tolerance_days) - article_data = result.get("article_data", []) - result["statistical_data"]["article_num"] = len(article_data) - - # 检查文章数量是否大于 150 - max_articles = 150 - if len(article_data) > max_articles: - logging.info("数据量较大，开始进行处理...") - # 获取前 max_articles 篇文章的作者集合 - top_authors = {article["author"] for article in article_data[:max_articles]} - - # 从第 {max_articles + 1} 篇开始过滤，只保留前 max_articles 篇出现过的作者的文章 - filtered_articles = article_data[:max_articles] + [ - article for article in article_data[max_articles:] - if article["author"] in top_authors - ] - - # 更新结果中的 article_data - result["article_data"] = filtered_articles - # 更新结果中的统计数据 - result["statistical_data"]["article_num"] = len(filtered_articles) - logging.info(f"数据处理完成，保留 {len(filtered_articles)} 篇文章") - - return result + """Legacy wrapper around the refactored dataset trimming helper.""" + return _limit_large_dataset(result, future_tolerance_days=future_tolerance_days) diff --git a/friend_circle_lite/app_config.py b/friend_circle_lite/app_config.py new file mode 100644 index 00000000000..ce3c9609e86 --- /dev/null +++ b/friend_circle_lite/app_config.py @@ -0,0 +1,160 @@ +"""Application configuration models. + +This module converts the raw YAML structure into typed configuration objects so +that the rest of the application can depend on explicit fields instead of a +loosely typed nested dictionary. + +The external YAML keys are preserved for backward compatibility. Internally, +snake_case names are used consistently. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + + +DEFAULT_CACHE_FILE = "./temp/feed_cache.sqlite3" +DEFAULT_NEWEST_POSTS_FILE = "./temp/newest_posts.json" +DEFAULT_ALL_JSON = "./all.json" +DEFAULT_ERRORS_JSON = "./errors.json" + + +@dataclass(slots=True) +class MergeResultConfig: + """Options for merging local crawl results with a remote Friend-Circle feed.""" + + enable: bool = False + merge_json_url: str = "" + + +@dataclass(slots=True) +class SpiderSettings: + """Crawler settings controlling source list and output density.""" + + enable: bool = True + json_url: str = "" + article_count: int = 5 + merge_result: MergeResultConfig = field(default_factory=MergeResultConfig) + + +@dataclass(slots=True) +class EmailPushConfig: + """Reserved configuration for the not-yet-implemented email push feature.""" + + enable: bool = False + to_email: str = "" + subject: str = "" + body_template: str = "" + + +@dataclass(slots=True) +class WebsiteInfo: + """Display metadata for outbound notifications.""" + + title: str = "" + + +@dataclass(slots=True) +class RssSubscribeConfig: + """Configuration for GitHub issue based email subscriptions.""" + + enable: bool = False + github_username: str = "" + github_repo: str = "" + your_blog_url: str = "" + email_template: str = "" + website_info: WebsiteInfo = field(default_factory=WebsiteInfo) + + +@dataclass(slots=True) +class SmtpConfig: + """SMTP connection settings used by all mail sending features.""" + + email: str = "" + server: str = "" + port: int = 0 + use_tls: bool = True + + +@dataclass(slots=True) +class RuntimePaths: + """Filesystem locations used by the runtime.""" + + cache_file: str = DEFAULT_CACHE_FILE + newest_posts_file: str = DEFAULT_NEWEST_POSTS_FILE + all_json_file: str = DEFAULT_ALL_JSON + errors_json_file: str = DEFAULT_ERRORS_JSON + + +@dataclass(slots=True) +class ApplicationConfig: + """Root application configuration assembled from the YAML file.""" + + spider_settings: SpiderSettings + email_push: EmailPushConfig + rss_subscribe: RssSubscribeConfig + smtp: SmtpConfig + specific_rss: list[dict] + runtime_paths: RuntimePaths = field(default_factory=RuntimePaths) + future_article_tolerance_days: int = 2 + + @classmethod + def from_dict(cls, data: dict) -> "ApplicationConfig": + """Create a typed config object from the raw YAML dictionary.""" + spider_raw = data.get("spider_settings", {}) + merge_raw = spider_raw.get("merge_result", {}) + email_push_raw = data.get("email_push", {}) + rss_subscribe_raw = data.get("rss_subscribe", {}) + website_info_raw = rss_subscribe_raw.get("website_info", {}) + smtp_raw = data.get("smtp", {}) + + return cls( + spider_settings=SpiderSettings( + enable=bool(spider_raw.get("enable", True)), + json_url=str(spider_raw.get("json_url", "")).strip(), + article_count=int(spider_raw.get("article_count", 5)), + merge_result=MergeResultConfig( + enable=bool(merge_raw.get("enable", False)), + merge_json_url=str(merge_raw.get("merge_json_url", "")).strip(), + ), + ), + email_push=EmailPushConfig( + enable=bool(email_push_raw.get("enable", False)), + to_email=str(email_push_raw.get("to_email", "")).strip(), + subject=str(email_push_raw.get("subject", "")).strip(), + body_template=str(email_push_raw.get("body_template", "")).strip(), + ), + rss_subscribe=RssSubscribeConfig( + enable=bool(rss_subscribe_raw.get("enable", False)), + github_username=str(rss_subscribe_raw.get("github_username", "")).strip(), + github_repo=str(rss_subscribe_raw.get("github_repo", "")).strip(), + your_blog_url=str(rss_subscribe_raw.get("your_blog_url", "")).strip(), + email_template=str(rss_subscribe_raw.get("email_template", "")).strip(), + website_info=WebsiteInfo( + title=str(website_info_raw.get("title", "")).strip(), + ), + ), + smtp=SmtpConfig( + email=str(smtp_raw.get("email", "")).strip(), + server=str(smtp_raw.get("server", "")).strip(), + port=int(smtp_raw.get("port", 0) or 0), + use_tls=bool(smtp_raw.get("use_tls", True)), + ), + specific_rss=list(data.get("specific_RSS", []) or []), + ) + + +@dataclass(slots=True) +class MailRuntime: + """Runtime SMTP credentials resolved from configuration and environment.""" + + sender_email: str + smtp_server: str + port: int + password: str + use_tls: bool + + @property + def is_ready(self) -> bool: + """Whether enough information is available to send email.""" + return bool(self.sender_email and self.smtp_server and self.port and self.password) diff --git a/friend_circle_lite/application.py b/friend_circle_lite/application.py new file mode 100644 index 00000000000..32b70b7dc50 --- /dev/null +++ b/friend_circle_lite/application.py @@ -0,0 +1,197 @@ +"""Top-level application orchestration. + +This module keeps the main script very small by moving the end-to-end workflow +into focused orchestration methods. +""" + +from __future__ import annotations + +import logging +import os +import sys + +from friend_circle_lite.all_friends import ( + deal_with_large_data, + fetch_and_process_data, + marge_data_from_json_url, + marge_errors_from_json_url, +) +from friend_circle_lite.app_config import ApplicationConfig, MailRuntime +from friend_circle_lite.single_friend import get_latest_articles_from_link +from friend_circle_lite.utils.github import extract_emails_from_issues +from friend_circle_lite.utils.json import write_json +from friend_circle_lite.utils.mail import send_emails + + +class FriendCircleLiteApplication: + """Application service coordinating crawl and notification workflows.""" + + def __init__(self, config: ApplicationConfig): + self.config = config + + def run(self) -> None: + """Execute the enabled application features in a stable order.""" + self.run_crawler_if_enabled() + mail_runtime = self.prepare_mail_runtime() + self.run_email_push_if_enabled(mail_runtime) + self.run_rss_subscription_if_enabled(mail_runtime) + + def run_crawler_if_enabled(self) -> None: + """Run the article crawl and persist public output files when enabled.""" + spider_settings = self.config.spider_settings + if not spider_settings.enable: + logging.info("⏭️ 爬虫未启用，跳过抓取流程") + return + + logging.info("✅ 爬虫已启用") + logging.info( + f"📥 正在从 {spider_settings.json_url} 获取数据，每个博客获取 {spider_settings.article_count} 篇文章" + ) + + crawl_result = fetch_and_process_data( + json_url=spider_settings.json_url, + specific_RSS=self.config.specific_rss, + count=spider_settings.article_count, + cache_file=self.config.runtime_paths.cache_file, + ) + if crawl_result is None: + logging.error("❌ 抓取流程失败，未生成任何输出文件") + return + + result, lost_friends = crawl_result + result, lost_friends = self._merge_remote_results_if_enabled(result, lost_friends) + + article_count = len(result.get("article_data", [])) + logging.info(f"📦 数据获取完毕，共有 {article_count} 篇文章，正在处理数据") + + result = deal_with_large_data( + result, + future_tolerance_days=self.config.future_article_tolerance_days, + ) + write_json(self.config.runtime_paths.all_json_file, result) + write_json(self.config.runtime_paths.errors_json_file, lost_friends) + + def prepare_mail_runtime(self) -> MailRuntime: + """Build SMTP runtime credentials from config and environment variables.""" + if not (self.config.email_push.enable or self.config.rss_subscribe.enable): + return MailRuntime(sender_email="", smtp_server="", port=0, password="", use_tls=False) + + logging.info("📨 推送功能已启用，正在准备中...") + smtp_conf = self.config.smtp + mail_runtime = MailRuntime( + sender_email=smtp_conf.email, + smtp_server=smtp_conf.server, + port=smtp_conf.port, + password=os.getenv("SMTP_PWD", ""), + use_tls=smtp_conf.use_tls, + ) + + logging.info(f"📡 SMTP 服务器：{mail_runtime.smtp_server}:{mail_runtime.port}") + if mail_runtime.is_ready: + logging.info(f"🔐 密码(部分)：{mail_runtime.password[:3]}*****") + else: + logging.error("❌ SMTP 信息不完整或环境变量 SMTP_PWD 未设置，无法发送邮件") + return mail_runtime + + def run_email_push_if_enabled(self, mail_runtime: MailRuntime) -> None: + """Keep the reserved email push entrypoint behavior unchanged.""" + if self.config.email_push.enable and mail_runtime.is_ready: + logging.info("📧 邮件推送已启用") + logging.info("⚠️ 抱歉，目前尚未实现邮件推送功能") + + def run_rss_subscription_if_enabled(self, mail_runtime: MailRuntime) -> None: + """Send subscription emails for newly discovered posts when enabled.""" + if not self.config.rss_subscribe.enable: + return + if not mail_runtime.is_ready: + logging.info("⏭️ RSS 订阅推送未执行，因为 SMTP 尚未就绪") + return + + logging.info("📰 RSS 订阅推送已启用") + github_username, github_repo = self._resolve_github_repo() + logging.info(f"👤 GitHub 用户名：{github_username}") + logging.info(f"📁 GitHub 仓库：{github_repo}") + + latest_articles = get_latest_articles_from_link( + url=self.config.rss_subscribe.your_blog_url, + count=5, + last_articles_path=self.config.runtime_paths.newest_posts_file, + ) + if not latest_articles: + logging.info("📭 无新文章，无需推送") + return + + logging.info(f"🆕 获取到的最新文章：{latest_articles}") + email_list = self._load_subscriber_emails(github_username, github_repo) + if not email_list: + logging.info("⚠️ 无订阅邮箱，请检查格式或是否有订阅者") + sys.exit(0) + + logging.info(f"📬 获取到邮箱列表：{email_list}") + for article in latest_articles: + template_data = self._build_email_template_data(article, github_username, github_repo) + send_emails( + emails=email_list["emails"], + sender_email=mail_runtime.sender_email, + smtp_server=mail_runtime.smtp_server, + port=mail_runtime.port, + password=mail_runtime.password, + subject=f"{self.config.rss_subscribe.website_info.title} の最新文章：{article['title']}", + body=self._build_plaintext_mail_body(article), + template_path=self.config.rss_subscribe.email_template, + template_data=template_data, + use_tls=mail_runtime.use_tls, + ) + + def _merge_remote_results_if_enabled(self, result: dict, lost_friends: list[list[str]]) -> tuple[dict, list[list[str]]]: + """Merge remote outputs when the self-hosted merge option is enabled.""" + merge_result = self.config.spider_settings.merge_result + if not merge_result.enable: + return result, lost_friends + + merge_url = merge_result.merge_json_url + logging.info(f"🔀 合并功能开启，从 {merge_url} 获取外部数据") + result = marge_data_from_json_url(result, f"{merge_url}/all.json") + lost_friends = marge_errors_from_json_url(lost_friends, f"{merge_url}/errors.json") + return result, lost_friends + + def _resolve_github_repo(self) -> tuple[str, str]: + """Resolve repository coordinates from env override or config.""" + fcl_repo = os.getenv("FCL_REPO") + if fcl_repo: + return tuple(fcl_repo.split("/", 1)) + return self.config.rss_subscribe.github_username, self.config.rss_subscribe.github_repo + + @staticmethod + def _load_subscriber_emails(github_username: str, github_repo: str) -> dict | None: + """Load subscriber emails from GitHub closed issues.""" + github_api_url = ( + f"https://api.github.com/repos/{github_username}/{github_repo}/issues" + f"?state=closed&label=subscribed&per_page=200" + ) + logging.info(f"🔎 正在从 GitHub 获取订阅邮箱：{github_api_url}") + return extract_emails_from_issues(github_api_url) + + def _build_email_template_data(self, article: dict, github_username: str, github_repo: str) -> dict[str, str]: + """Assemble template variables for one outbound notification email.""" + return { + "title": article["title"], + "summary": article["summary"], + "published": article["published"], + "link": article["link"], + "website_title": self.config.rss_subscribe.website_info.title, + "github_issue_url": ( + f"https://github.com/{github_username}/{github_repo}" + "/issues?q=is%3Aissue+is%3Aclosed" + ), + } + + @staticmethod + def _build_plaintext_mail_body(article: dict) -> str: + """Build the plain-text fallback body for one notification email.""" + return ( + f"📄 文章标题：{article['title']}\n" + f"🔗 链接：{article['link']}\n" + f"📝 简介：{article['summary']}\n" + f"🕒 发布时间：{article['published']}" + ) diff --git a/friend_circle_lite/cache_store.py b/friend_circle_lite/cache_store.py new file mode 100644 index 00000000000..d4a45fbadca --- /dev/null +++ b/friend_circle_lite/cache_store.py @@ -0,0 +1,166 @@ +"""Persistent RSS cache storage. + +SQLite is used for the feed cache because it is more robust than hand-edited +text formats for internal state: + +- schema is explicit and stable; +- writes are transactional; +- corruption risk from accidental manual edits is lower; +- Python ships with `sqlite3`, so no extra dependency is required. + +For smooth upgrades, this store can also migrate legacy cache data from the old +JSON cache file and the intermediate YAML cache file if they exist. +""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +from pathlib import Path + +import yaml + +from friend_circle_lite.models import CacheRecord + + +class FeedCacheStore: + """Persist and load discovered RSS endpoints using SQLite.""" + + def __init__(self, cache_path: str | Path | None): + self.cache_path = Path(cache_path) if cache_path else None + + def load_records(self) -> list[CacheRecord]: + """Load cache records from SQLite, migrating legacy formats if needed.""" + if not self.cache_path: + return [] + + if self.cache_path.exists(): + return self._load_from_sqlite() + + migrated_records = self._load_legacy_records() + if migrated_records: + if self.save_records(migrated_records): + logging.info(f"已迁移 {len(migrated_records)} 条 RSS 缓存记录到 {self.cache_path}。") + return migrated_records + + logging.info(f"缓存文件 {self.cache_path} 不存在，将在首次成功抓取后创建。") + return [] + + def save_records(self, records: list[CacheRecord]) -> bool: + """Persist cache records to the SQLite database.""" + if not self.cache_path: + return True + + try: + self.cache_path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(self.cache_path) as connection: + self._ensure_schema(connection) + connection.execute("DELETE FROM feed_cache") + connection.executemany( + "INSERT INTO feed_cache(name, url, source) VALUES (?, ?, ?)", + [(record.name, record.url, record.source) for record in sorted(records, key=lambda item: item.name)], + ) + connection.commit() + logging.info(f"缓存已保存到 {self.cache_path}（{len(records)} 条）。") + return True + except Exception as exc: + logging.error(f"保存缓存文件失败: {self.cache_path}, 错误信息: {exc}") + return False + + def _load_from_sqlite(self) -> list[CacheRecord]: + """Load records from the current SQLite cache file.""" + try: + with sqlite3.connect(self.cache_path) as connection: + self._ensure_schema(connection) + rows = connection.execute( + "SELECT name, url, source FROM feed_cache ORDER BY name" + ).fetchall() + except Exception as exc: + logging.warning(f"读取 SQLite 缓存失败: {self.cache_path}, 错误信息: {exc}") + return [] + + return [ + CacheRecord(name=name, url=url, source=source or "cache") + for name, url, source in rows + if name and url + ] + + @staticmethod + def _ensure_schema(connection: sqlite3.Connection) -> None: + """Create the cache table when it does not exist yet.""" + connection.execute( + """ + CREATE TABLE IF NOT EXISTS feed_cache ( + name TEXT PRIMARY KEY, + url TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'cache' + ) + """ + ) + + def _load_legacy_records(self) -> list[CacheRecord]: + """Read old cache formats for seamless upgrades.""" + json_records = self._load_legacy_json_cache() + if json_records: + return json_records + + yaml_records = self._load_legacy_yaml_cache() + if yaml_records: + return yaml_records + + return [] + + def _load_legacy_json_cache(self) -> list[CacheRecord]: + """Read the previous JSON cache file format.""" + if not self.cache_path: + return [] + + legacy_path = self.cache_path.with_name("cache.json") + if not legacy_path.exists(): + return [] + + try: + with open(legacy_path, "r", encoding="utf-8") as file: + payload = json.load(file) + except Exception as exc: + logging.warning(f"读取旧缓存文件失败: {legacy_path}, 错误信息: {exc}") + return [] + + if not isinstance(payload, list): + return [] + + return self._normalize_legacy_items(payload) + + def _load_legacy_yaml_cache(self) -> list[CacheRecord]: + """Read the temporary YAML cache format used during refactoring.""" + if not self.cache_path: + return [] + + legacy_path = self.cache_path.with_name("feed_cache.yaml") + if not legacy_path.exists(): + return [] + + try: + with open(legacy_path, "r", encoding="utf-8") as file: + payload = yaml.safe_load(file) or {} + except Exception as exc: + logging.warning(f"读取旧 YAML 缓存失败: {legacy_path}, 错误信息: {exc}") + return [] + + items = payload.get("feeds", []) if isinstance(payload, dict) else [] + return self._normalize_legacy_items(items) + + @staticmethod + def _normalize_legacy_items(items: list[object]) -> list[CacheRecord]: + """Normalize legacy cache items into typed cache records.""" + records: list[CacheRecord] = [] + for item in items: + if not isinstance(item, dict): + continue + name = str(item.get("name", "")).strip() + url = str(item.get("url", "")).strip() + source = str(item.get("source", "cache")).strip() or "cache" + if name and url: + records.append(CacheRecord(name=name, url=url, source=source)) + return records diff --git a/friend_circle_lite/crawler_service.py b/friend_circle_lite/crawler_service.py new file mode 100644 index 00000000000..34302ac0e35 --- /dev/null +++ b/friend_circle_lite/crawler_service.py @@ -0,0 +1,272 @@ +"""High-level crawler orchestration. + +This module contains the system-level services that coordinate website loading, +RSS discovery, parsing, cache updates, result aggregation, and legacy output +formatting. +""" + +from __future__ import annotations + +import logging +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +import requests + +from friend_circle_lite import HEADERS_JSON, timeout +from friend_circle_lite.cache_store import FeedCacheStore +from friend_circle_lite.feed_service import FeedDiscoveryService, FeedParserService +from friend_circle_lite.models import Article, CacheRecord, CacheUpdate, CrawlResult, CrawlStatistics, FeedEndpoint, Website + + +class WebsiteFeedResolver: + """Resolve which feed endpoint should be used for a website. + + Resolution order is kept compatible with the previous implementation: + manual configuration first, then cache, and finally automatic discovery. + """ + + def __init__(self, discovery_service: FeedDiscoveryService, configured_feeds: list[CacheRecord]): + self.discovery_service = discovery_service + self.feed_lookup = {item.name: item for item in configured_feeds} + + def resolve(self, website: Website) -> FeedEndpoint | None: + configured = self.feed_lookup.get(website.name) + if configured: + logging.info(f"“{website.name}” 使用预设 RSS 源：{configured.url} （source={configured.source}）。") + return FeedEndpoint(url=configured.url, feed_type="specific", source=configured.source) + + discovered = self.discovery_service.discover(website.url) + if discovered: + logging.info(f"“{website.name}” 自动探测 RSS：type：{discovered.feed_type}, url：{discovered.url} 。") + return discovered + + +class WebsiteCrawler: + """Crawl one website and produce a normalized result.""" + + def __init__(self, parser_service: FeedParserService, resolver: WebsiteFeedResolver): + self.parser_service = parser_service + self.resolver = resolver + + def crawl(self, website: Website, count: int) -> CrawlResult: + """Crawl one website while preserving legacy cache repair behavior.""" + endpoint = self.resolver.resolve(website) + cache_update = CacheUpdate(action="none", name=website.name) + + if endpoint and endpoint.source == "auto": + cache_update = CacheUpdate(action="set", name=website.name, url=endpoint.url, reason="auto_discovered") + + articles = self._parse_articles(endpoint, website, count) + parse_error = endpoint is not None and not articles + + if parse_error and endpoint and endpoint.source in ("cache", "unknown"): + logging.info(f"缓存 RSS 无效，重新探测：{website.name} ({website.url})。") + rediscovered = self.resolver.discovery_service.discover(website.url) + if rediscovered: + articles = self._parse_articles(rediscovered, website, count) + if articles: + endpoint = rediscovered + cache_update = CacheUpdate(action="set", name=website.name, url=rediscovered.url, reason="repair_cache") + else: + endpoint = None + cache_update = CacheUpdate(action="delete", name=website.name, url=None, reason="remove_invalid") + else: + endpoint = None + cache_update = CacheUpdate(action="delete", name=website.name, url=None, reason="remove_invalid") + + status = "active" if articles else "error" + if not articles: + if endpoint is None: + logging.warning(f"{website.name} 的博客 {website.url} 未找到有效 RSS。") + else: + logging.warning(f"{website.name} 的 RSS {endpoint.url} 未解析出文章。") + + return CrawlResult( + website=website, + status=status, + articles=articles, + feed_url=endpoint.url if endpoint else None, + feed_type=endpoint.feed_type if endpoint else "none", + source_used=endpoint.source if endpoint else "none", + cache_update=cache_update, + ) + + def _parse_articles(self, endpoint: FeedEndpoint | None, website: Website, count: int) -> list[Article]: + if endpoint is None: + return [] + + articles = self.parser_service.parse(endpoint.url, count=count, blog_url=website.url) + for article in articles: + article.author = website.name + article.avatar = website.avatar + logging.info(f"{website.name} 发布了新文章：{article.title}，时间：{article.published}，链接：{article.link}") + return articles + + +class FriendCircleCrawler: + """System-level orchestrator for crawling all configured websites.""" + + def __init__(self, json_url: str, count: int, specific_rss: list[dict] | None = None, cache_file: str | None = None): + self.json_url = json_url + self.count = count + self.specific_rss = specific_rss or [] + self.cache_store = FeedCacheStore(cache_file) + + def run(self) -> tuple[dict, list[list[str]]] | None: + """Fetch website list, crawl all websites, and build public outputs.""" + session = requests.Session() + websites = self._load_websites(session) + if websites is None: + return None + + cache_records = self.cache_store.load_records() + manual_records = self._build_manual_records() + merged_records = self._merge_feed_records(cache_records, manual_records) + manual_names = {record.name for record in manual_records} + + discovery_service = FeedDiscoveryService(session) + parser_service = FeedParserService(session) + resolver = WebsiteFeedResolver(discovery_service=discovery_service, configured_feeds=merged_records) + crawler = WebsiteCrawler(parser_service=parser_service, resolver=resolver) + + crawl_results: list[CrawlResult] = [] + with ThreadPoolExecutor(max_workers=10) as executor: + future_to_website = { + executor.submit(crawler.crawl, website, self.count): website + for website in websites + } + for future in as_completed(future_to_website): + website = future_to_website[future] + try: + crawl_results.append(future.result()) + except Exception as exc: + logging.error(f"处理 {website.to_error_payload()} 时发生错误: {exc}", exc_info=True) + crawl_results.append(CrawlResult(website=website, status="error")) + + self._apply_cache_updates(cache_records, crawl_results, manual_names) + + active_results = [result for result in crawl_results if result.status == "active"] + error_results = [result.website.to_error_payload() for result in crawl_results if result.status != "active"] + all_articles = [article.to_public_dict() for result in active_results for article in result.articles] + + statistics = CrawlStatistics.create( + friends_num=len(websites), + active_num=len(active_results), + error_num=len(error_results), + article_num=len(all_articles), + ) + result = { + "statistical_data": statistics.to_dict(), + "article_data": all_articles, + } + logging.info( + f"数据处理完成，总共有 {len(websites)} 位朋友，其中 {len(active_results)} 位博客可访问，" + f"{len(error_results)} 位博客无法访问。" + ) + return result, error_results + + def _load_websites(self, session: requests.Session) -> list[Website] | None: + try: + response = session.get(self.json_url, headers=HEADERS_JSON, timeout=timeout) + response.raise_for_status() + friends_data = response.json() + except Exception as exc: + logging.error(f"无法获取链接：{self.json_url} ：{exc}", exc_info=True) + return None + + websites: list[Website] = [] + for friend in friends_data.get("friends", []): + try: + websites.append(Website.from_friend_item(friend)) + except Exception: + logging.warning(f"发现格式异常的友链数据，已跳过: {friend!r}") + return websites + + def _build_manual_records(self) -> list[CacheRecord]: + manual_records: list[CacheRecord] = [] + for item in self.specific_rss: + if isinstance(item, dict) and item.get("name") and item.get("url"): + manual_records.append(CacheRecord(name=item["name"], url=item["url"], source="manual")) + return manual_records + + @staticmethod + def _merge_feed_records(cache_records: list[CacheRecord], manual_records: list[CacheRecord]) -> list[CacheRecord]: + merged = {record.name: record for record in cache_records} + for record in manual_records: + merged[record.name] = record + return list(merged.values()) + + def _apply_cache_updates(self, cache_records: list[CacheRecord], crawl_results: list[CrawlResult], manual_names: set[str]) -> None: + cache_map = {record.name: record for record in cache_records} + unique_updates: dict[str, CacheUpdate] = {} + + for result in crawl_results: + update = result.cache_update + if not update.name or update.action == "none" or update.name in manual_names: + continue + if update.action == "set" and update.url: + unique_updates[update.name] = update + elif update.action == "delete": + unique_updates[update.name] = update + + for name, update in unique_updates.items(): + if update.action == "set" and update.url: + cache_map[name] = CacheRecord(name=name, url=update.url, source="cache") + logging.info(f"缓存更新：SET {name} -> {update.url} ({update.reason})") + elif update.action == "delete" and name in cache_map: + cache_map.pop(name) + logging.info(f"缓存更新：DELETE {name} ({update.reason})") + + self.cache_store.save_records(list(cache_map.values())) + + +def sort_articles_by_time(data: dict, future_tolerance_days: int = 2) -> dict: + """Sort article payloads by time and remove far-future timestamps.""" + for article in data.get("article_data", []): + if not article.get("created"): + article["created"] = "2024-01-01 00:00" + logging.warning(f"文章 {article['title']} 未包含时间信息，已设置为默认时间 2024-01-01 00:00") + + now = datetime.now(ZoneInfo("Asia/Shanghai")).replace(tzinfo=None) + max_allowed_time = now + timedelta(days=future_tolerance_days) + filtered_articles = [] + removed_count = 0 + + for article in data.get("article_data", []): + article_time = datetime.strptime(article["created"], "%Y-%m-%d %H:%M") + if article_time > max_allowed_time: + removed_count += 1 + logging.warning( + f"文章 {article['title']} 的时间 {article['created']} 超出当前时间 {future_tolerance_days} 天以上，已跳过显示" + ) + continue + filtered_articles.append(article) + + filtered_articles.sort(key=lambda item: datetime.strptime(item["created"], "%Y-%m-%d %H:%M"), reverse=True) + data["article_data"] = filtered_articles + if removed_count: + logging.info(f"已过滤 {removed_count} 篇未来时间异常的文章") + return data + + +def limit_large_dataset(result: dict, future_tolerance_days: int = 2) -> dict: + """Keep the existing data trimming strategy for very large datasets.""" + result = sort_articles_by_time(result, future_tolerance_days=future_tolerance_days) + article_data = result.get("article_data", []) + result["statistical_data"]["article_num"] = len(article_data) + + max_articles = 150 + if len(article_data) > max_articles: + logging.info("数据量较大，开始进行处理...") + top_authors = {article["author"] for article in article_data[:max_articles]} + filtered_articles = article_data[:max_articles] + [ + article for article in article_data[max_articles:] + if article["author"] in top_authors + ] + result["article_data"] = filtered_articles + result["statistical_data"]["article_num"] = len(filtered_articles) + logging.info(f"数据处理完成，保留 {len(filtered_articles)} 篇文章") + + return result diff --git a/friend_circle_lite/feed_service.py b/friend_circle_lite/feed_service.py new file mode 100644 index 00000000000..777b19e87b1 --- /dev/null +++ b/friend_circle_lite/feed_service.py @@ -0,0 +1,154 @@ +"""Feed discovery, parsing, and incremental tracking services.""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime +from pathlib import Path +from urllib.parse import urlparse + +import feedparser +import requests + +from friend_circle_lite import HEADERS_XML, timeout +from friend_circle_lite.models import Article, FeedEndpoint, Website +from friend_circle_lite.utils.time import format_published_time +from friend_circle_lite.utils.url import replace_non_domain + + +class FeedDiscoveryService: + """Discover an RSS or Atom endpoint for a website.""" + + POSSIBLE_FEEDS = [ + ("atom", "/atom.xml"), + ("rss", "/rss.xml"), + ("rss2", "/rss2.xml"), + ("rss3", "/rss.php"), + ("feed", "/feed"), + ("feed2", "/feed.xml"), + ("feed3", "/feed/"), + ("feed4", "/feed.php"), + ("index", "/index.xml"), + ] + + def __init__(self, session: requests.Session): + self.session = session + + def discover(self, website_url: str) -> FeedEndpoint | None: + """Try common feed endpoints and return the first valid match.""" + for feed_type, path in self.POSSIBLE_FEEDS: + feed_url = website_url.rstrip("/") + path + try: + response = self.session.get(feed_url, headers=HEADERS_XML, timeout=timeout) + except requests.RequestException: + continue + + if response.status_code != 200: + continue + + content_type = response.headers.get("Content-Type", "").lower() + if "xml" in content_type or "rss" in content_type or "atom" in content_type: + return FeedEndpoint(url=feed_url, feed_type=feed_type, source="auto") + + text_head = response.text[:1000].lower() + if " list[Article]: + """Parse a feed URL and return the newest `count` articles. + + The returned articles are normalized to the project's internal domain + model, while preserving the original public output fields. + """ + try: + response = self.session.get(feed_url, headers=HEADERS_XML, timeout=timeout) + response.encoding = response.apparent_encoding or "utf-8" + feed = feedparser.parse(response.text) + except Exception as exc: + logging.error(f"无法解析 FEED 地址：{feed_url} ，请自行排查原因！错误信息: {exc}") + return [] + + default_author = feed.feed.author if "author" in feed.feed else "" + articles: list[Article] = [] + + for entry in feed.entries: + published = self._extract_published_time(entry) + article_link = replace_non_domain(entry.link, blog_url) if "link" in entry else "" + article = Article( + title=entry.title if "title" in entry else "", + author=default_author, + link=article_link, + published=published, + summary=entry.summary if "summary" in entry else "", + content=entry.content[0].value if "content" in entry and entry.content else entry.description if "description" in entry else "", + ) + articles.append(article) + + valid_articles = [article for article in articles if article.published] + valid_articles.sort(key=lambda item: datetime.strptime(item.published, "%Y-%m-%d %H:%M"), reverse=True) + return valid_articles[:count] if count < len(valid_articles) else valid_articles + + @staticmethod + def _extract_published_time(entry) -> str: + """Extract a normalized publish time from a feed entry.""" + if "published" in entry: + return format_published_time(entry.published) + if "updated" in entry: + published = format_published_time(entry.updated) + logging.warning(f"文章 {entry.title} 未包含发布时间，已使用更新时间 {published}") + return published + + logging.warning(f"文章 {entry.title} 未包含任何时间信息, 请检查原文, 跳过该文章") + return "" + + +class LatestArticleTracker: + """Track whether a website published new posts since the last crawl.""" + + def __init__(self, storage_path: str | Path): + self.storage_path = Path(storage_path) + + def diff_and_persist(self, latest_articles: list[Article]) -> list[dict] | None: + """Return newly seen articles and update the local snapshot file.""" + previous_links = self._load_previous_links() + updated_articles = [article.to_tracking_dict() for article in latest_articles if article.link not in previous_links] + self._persist(latest_articles) + return updated_articles if updated_articles else None + + def _load_previous_links(self) -> set[str]: + if not self.storage_path.exists(): + return set() + try: + with open(self.storage_path, "r", encoding="utf-8") as file: + payload = json.load(file) + except Exception as exc: + logging.warning(f"读取最新文章缓存失败: {self.storage_path}, 错误信息: {exc}") + return set() + + articles = payload.get("articles", []) if isinstance(payload, dict) else [] + return {article.get("link", "") for article in articles if isinstance(article, dict) and article.get("link")} + + def _persist(self, latest_articles: list[Article]) -> None: + self.storage_path.parent.mkdir(parents=True, exist_ok=True) + payload = {"articles": [article.to_tracking_dict() for article in latest_articles]} + with open(self.storage_path, "w", encoding="utf-8") as file: + json.dump(payload, file, ensure_ascii=False, indent=4) + + +def extract_blog_origin(url: str) -> str: + """Return a normalized origin for display or author profile links.""" + parsed = urlparse(url) + if not parsed.scheme or not parsed.netloc: + return url + return f"{parsed.scheme}://{parsed.netloc}" diff --git a/friend_circle_lite/models.py b/friend_circle_lite/models.py new file mode 100644 index 00000000000..8b23946f997 --- /dev/null +++ b/friend_circle_lite/models.py @@ -0,0 +1,164 @@ +"""Domain models for Friend-Circle-Lite. + +These models centralize the core concepts used across the crawler so that the +transport layer, parsing logic, cache logic, and output formatting can evolve +independently. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from zoneinfo import ZoneInfo + + +SHANGHAI_TZ = ZoneInfo("Asia/Shanghai") + + +@dataclass(slots=True) +class Website: + """Represents a friend website entry from the upstream friend list.""" + + name: str + url: str + avatar: str = "" + + @classmethod + def from_friend_item(cls, raw_friend: list | tuple) -> "Website": + """Create a website from the existing `[name, url, avatar]` structure.""" + name, url, avatar = raw_friend + return cls(name=name, url=url, avatar=avatar or "") + + def to_error_payload(self) -> list[str]: + """Return the legacy structure used by `errors.json`.""" + return [self.name, self.url, self.avatar] + + +@dataclass(slots=True) +class Article: + """Represents one crawled article belonging to a website.""" + + title: str + author: str + link: str + published: str + summary: str = "" + content: str = "" + avatar: str = "" + + def to_public_dict(self) -> dict[str, str]: + """Return the legacy public article schema used by `all.json`.""" + return { + "title": self.title, + "created": self.published, + "link": self.link, + "author": self.author, + "avatar": self.avatar, + } + + def to_tracking_dict(self) -> dict[str, str]: + """Return the article schema used by the latest article tracker.""" + return { + "title": self.title, + "author": self.author, + "link": self.link, + "published": self.published, + "summary": self.summary, + "content": self.content, + } + + +@dataclass(slots=True) +class FeedEndpoint: + """Represents a concrete feed endpoint and how it was found.""" + + url: str + feed_type: str + source: str + + +@dataclass(slots=True) +class CacheRecord: + """Represents one cached RSS endpoint mapping for a website.""" + + name: str + url: str + source: str = "cache" + + def to_dict(self) -> dict[str, str]: + return { + "name": self.name, + "url": self.url, + } + + +@dataclass(slots=True) +class CacheUpdate: + """Describes how a crawl should update the persisted RSS cache.""" + + action: str = "none" + name: str | None = None + url: str | None = None + reason: str = "" + + def to_dict(self) -> dict[str, str | None]: + return { + "action": self.action, + "name": self.name, + "url": self.url, + "reason": self.reason, + } + + +@dataclass(slots=True) +class CrawlResult: + """Represents the crawl result for a single website.""" + + website: Website + status: str + articles: list[Article] = field(default_factory=list) + feed_url: str | None = None + feed_type: str = "none" + source_used: str = "none" + cache_update: CacheUpdate = field(default_factory=CacheUpdate) + + def to_legacy_dict(self) -> dict[str, object]: + return { + "name": self.website.name, + "status": self.status, + "articles": [article.to_public_dict() for article in self.articles], + "feed_url": self.feed_url, + "feed_type": self.feed_type, + "cache_update": self.cache_update.to_dict(), + "source_used": self.source_used, + } + + +@dataclass(slots=True) +class CrawlStatistics: + """Aggregated crawl statistics for the generated `all.json` output.""" + + friends_num: int = 0 + active_num: int = 0 + error_num: int = 0 + article_num: int = 0 + last_updated_time: str = "" + + @classmethod + def create(cls, friends_num: int, active_num: int, error_num: int, article_num: int) -> "CrawlStatistics": + return cls( + friends_num=friends_num, + active_num=active_num, + error_num=error_num, + article_num=article_num, + last_updated_time=datetime.now(SHANGHAI_TZ).strftime("%Y-%m-%d %H:%M:%S"), + ) + + def to_dict(self) -> dict[str, int | str]: + return { + "friends_num": self.friends_num, + "active_num": self.active_num, + "error_num": self.error_num, + "article_num": self.article_num, + "last_updated_time": self.last_updated_time, + } diff --git a/friend_circle_lite/single_friend.py b/friend_circle_lite/single_friend.py index d1ee1108b97..50cb4803264 100644 --- a/friend_circle_lite/single_friend.py +++ b/friend_circle_lite/single_friend.py @@ -1,154 +1,42 @@ -import logging -from datetime import datetime -import re -import os -import json -import requests -import feedparser -from friend_circle_lite import HEADERS_XML, timeout -from friend_circle_lite.utils.time import format_published_time -from friend_circle_lite.utils.url import replace_non_domain +"""Legacy-compatible single website helpers. -def check_feed(blog_url, session): - """ - 检查博客的 RSS 或 Atom 订阅链接。 +The project now uses dedicated domain models and services, but these helpers are +kept as a compatibility layer because existing entrypoints still import them. +""" - 优化点： - - 检查 HTTP 状态码。 - - 检查 Content-Type 是否包含 xml / rss / atom。 - - 检查响应内容前几百字节内是否有 RSS/Atom 的特征标签。 - """ - possible_feeds = [ - ('atom', '/atom.xml'), - ('rss', '/rss.xml'), # 2024-07-26 添加 /rss.xml内容的支持 - ('rss2', '/rss2.xml'), - ('rss3', '/rss.php'), # 2024-12-07 添加 /rss.php内容的支持 - ('feed', '/feed'), - ('feed2', '/feed.xml'), # 2024-07-26 添加 /feed.xml内容的支持 - ('feed3', '/feed/'), - ('feed4', '/feed.php'), # 2025-07-22 添加 /feed.php内容的支持 - ('index', '/index.xml') # 2024-07-25 添加 /index.xml内容的支持 - ] +from __future__ import annotations - for feed_type, path in possible_feeds: - feed_url = blog_url.rstrip('/') + path - try: - response = session.get(feed_url, headers=HEADERS_XML, timeout=timeout) - if response.status_code == 200: - # 检查 Content-Type - content_type = response.headers.get('Content-Type', '').lower() - if 'xml' in content_type or 'rss' in content_type or 'atom' in content_type: - return [feed_type, feed_url] - - # 如果 Content-Type 是 text/html 或未明确，但内容本身是 RSS - text_head = response.text[:1000].lower() # 读取前1000字符 - if (' {feed_url}）：{e}") - parse_error = True - - # ---- 4. 如果缓存 RSS 无效则重新探测 ---- - if parse_error and source_used in ('cache', 'unknown'): - logging.info(f"缓存 RSS 无效，重新探测：{name} ({blog_url})。") - new_type, new_url = check_feed(blog_url, session) - if new_type != 'none' and new_url: - try: - feed_info = parse_feed(new_url, session, count, blog_url) - if isinstance(feed_info, dict) and 'articles' in feed_info: - articles = [ - { - 'title': a['title'], - 'created': a['published'], - 'link': a['link'], - 'author': name, - 'avatar': avatar, - } - for a in feed_info['articles'] - ] - - for a in articles: - logging.info(f"{name} 发布了新文章：{a['title']}，时间：{a['created']}，链接：{a['link']}") - - feed_type, feed_url, source_used = new_type, new_url, 'auto' - cache_update = {'action': 'set', 'name': name, 'url': new_url, 'reason': 'repair_cache'} - parse_error = False - except Exception as e: - logging.warning(f"重新探测解析仍失败：{name} ({new_url})：{e}") - cache_update = {'action': 'delete', 'name': name, 'url': None, 'reason': 'remove_invalid'} - feed_type, feed_url = 'none', None + endpoint = None + cache_update = CacheUpdate(action='delete', name=website.name, url=None, reason='remove_invalid') else: - cache_update = {'action': 'delete', 'name': name, 'url': None, 'reason': 'remove_invalid'} - feed_type, feed_url = 'none', None - - # ---- 5. 最终状态 ---- - status = 'active' if articles else 'error' - if not articles: - if feed_type == 'none': - logging.warning(f"{name} 的博客 {blog_url} 未找到有效 RSS。") - else: - logging.warning(f"{name} 的 RSS {feed_url} 未解析出文章。") + endpoint = None + cache_update = CacheUpdate(action='delete', name=website.name, url=None, reason='remove_invalid') return { - 'name': name, - 'status': status, + 'name': website.name, + 'status': 'active' if articles else 'error', 'articles': articles, - 'feed_url': feed_url, - 'feed_type': feed_type, - 'cache_update': cache_update, - 'source_used': source_used, + 'feed_url': endpoint['url'] if endpoint else None, + 'feed_type': endpoint['feed_type'] if endpoint else 'none', + 'cache_update': cache_update.to_dict(), + 'source_used': endpoint['source'] if endpoint else 'none', } def get_latest_articles_from_link(url, count=5, last_articles_path="./temp/newest_posts.json"): - """ - 从指定链接获取最新的文章数据并与本地存储的上次的文章数据进行对比。 - - 参数： - url (str): 用于获取文章数据的链接。 - count (int): 获取文章数的最大数。如果小于则全部获取，如果文章数大于则只取前 count 篇文章。 - - 返回： - list: 更新的文章列表，如果没有更新的文章则返回 None。 - """ - # 本地存储上次文章数据的文件 - local_file = last_articles_path - - # 检查和解析 feed + """Return newly published articles relative to the last local snapshot.""" session = requests.Session() feed_type, feed_url = check_feed(url, session) if feed_type == 'none': logging.error(f"无法获取 {url} 的文章数据") return None - # 获取最新的文章数据 - latest_data = parse_feed(feed_url, session ,count) - latest_articles = latest_data['articles'] - - # 读取本地存储的上次的文章数据 - if os.path.exists(local_file): - with open(local_file, 'r', encoding='utf-8') as file: - last_data = json.load(file) - else: - last_data = {'articles': []} - - last_articles = last_data['articles'] - - # 找到更新的文章 - updated_articles = [] - last_titles = {article['link'] for article in last_articles} - - for article in latest_articles: - if article['link'] not in last_titles: - updated_articles.append(article) - - logging.info(f"从 {url} 获取到 {len(latest_articles)} 篇文章，其中 {len(updated_articles)} 篇为新文章") - - # 更新本地存储的文章数据 - with open(local_file, 'w', encoding='utf-8') as file: - json.dump({'articles': latest_articles}, file, ensure_ascii=False, indent=4) - - # 如果有更新的文章，返回这些文章，否则返回 None - return updated_articles if updated_articles else None + latest_articles = FeedParserService(session).parse(feed_url, count=count, blog_url=url) + updated_articles = LatestArticleTracker(last_articles_path).diff_and_persist(latest_articles) + logging.info( + f"从 {url} 获取到 {len(latest_articles)} 篇文章，其中 {0 if updated_articles is None else len(updated_articles)} 篇为新文章" + ) + return updated_articles diff --git a/friend_circle_lite/utils/cache.py b/friend_circle_lite/utils/cache.py index 9f685387ab0..f5803e639f7 100644 --- a/friend_circle_lite/utils/cache.py +++ b/friend_circle_lite/utils/cache.py @@ -1,35 +1,24 @@ -import logging -from friend_circle_lite.utils.json import read_json, write_json +"""Backward-compatible cache helpers. -def load_cache(cache_file: str): - if not cache_file: - return [] - - data = read_json(cache_file) - if data is None: - logging.info(f"缓存文件 {cache_file} 不存在或无法读取，将自动创建。") - return [] +The crawler now stores feed cache records as YAML through `FeedCacheStore`. +These wrappers keep the old function names available for any external callers. +""" - if not isinstance(data, list): - logging.warning(f"缓存文件 {cache_file} 格式异常（应为列表）。将忽略。") - return [] +from friend_circle_lite.cache_store import FeedCacheStore +from friend_circle_lite.models import CacheRecord - norm = [] - for item in data: - if not isinstance(item, dict): - continue - name = item.get('name') - url = item.get('url') - if name and url: - norm.append({'name': name, 'url': url, 'source': 'cache'}) - return norm -def save_cache(cache_file: str, cache_items: list[dict]): - if not cache_file: - return +def load_cache(cache_file: str): + """Load cache records and expose the legacy list-of-dicts structure.""" + records = FeedCacheStore(cache_file).load_records() + return [{"name": item.name, "url": item.url, "source": item.source} for item in records] - out = [{'name': i['name'], 'url': i['url']} for i in cache_items] - if write_json(cache_file, out): - logging.info(f"缓存已保存到 {cache_file}（{len(out)} 条）。") - else: - logging.error(f"保存缓存文件 {cache_file} 失败") + +def save_cache(cache_file: str, cache_items: list[dict]): + """Persist cache records while accepting the legacy input structure.""" + records = [ + CacheRecord(name=item["name"], url=item["url"], source=item.get("source", "cache")) + for item in cache_items + if item.get("name") and item.get("url") + ] + return FeedCacheStore(cache_file).save_records(records) diff --git a/friend_circle_lite/utils/config.py b/friend_circle_lite/utils/config.py index babf81cde91..37976d65e14 100644 --- a/friend_circle_lite/utils/config.py +++ b/friend_circle_lite/utils/config.py @@ -1,19 +1,18 @@ -import yaml +"""Configuration loading utilities.""" + +from __future__ import annotations + import logging -def load_config(config_file): - """ - 加载配置文件。 - - 参数： - config_file (str): 配置文件的路径。 - - 返回： - dict: 加载的配置数据。 - """ +import yaml + +from friend_circle_lite.app_config import ApplicationConfig + +def load_raw_config(config_file: str) -> dict: + """Load the raw YAML config dictionary from disk.""" try: with open(config_file, 'r', encoding='utf-8') as file: - return yaml.safe_load(file) + return yaml.safe_load(file) or {} except FileNotFoundError: logging.error(f"配置文件 {config_file} 未找到") return {} @@ -23,3 +22,8 @@ def load_config(config_file): except Exception as e: logging.error(f"加载配置文件时发生未知错误: {str(e)}") return {} + + +def load_config(config_file: str) -> ApplicationConfig: + """Load and validate the application configuration as typed objects.""" + return ApplicationConfig.from_dict(load_raw_config(config_file)) diff --git a/requirements.txt b/requirements.txt index bef98e81b15..873fccbba39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ -datetime python-dateutil==2.9.0.post0 requests feedparser==6.0.11 PyYAML==6.0.1 -jinja2==3.1.2 \ No newline at end of file +jinja2==3.1.2 diff --git a/run.py b/run.py index dcafda771b9..5447522ef53 100644 --- a/run.py +++ b/run.py @@ -1,163 +1,27 @@ -import logging -import sys -import os - -from friend_circle_lite.all_friends import fetch_and_process_data, marge_data_from_json_url, marge_errors_from_json_url, deal_with_large_data -from friend_circle_lite.utils.json import write_json -from friend_circle_lite.utils.config import load_config -from friend_circle_lite.utils.mail import send_emails -from friend_circle_lite.single_friend import get_latest_articles_from_link -from friend_circle_lite.utils.github import extract_emails_from_issues - -FUTURE_ARTICLE_TOLERANCE_DAYS = 2 - -# ========== 日志设置 ========== -logging.basicConfig( - level=logging.INFO, - format='😋 %(levelname)s: %(message)s' -) - -# ========== 加载环境变量 ========== -# if os.getenv("GITHUB_TOKEN") is None: -# from dotenv import load_dotenv -# load_dotenv() - -# ========== 加载配置 ========== -config = load_config("./conf.yaml") - -# ========== 爬虫模块 ========== -if config["spider_settings"]["enable"]: - - logging.info("✅ 爬虫已启用") - json_url = config['spider_settings']['json_url'] - article_count = config['spider_settings']['article_count'] - specific_rss = config['specific_RSS'] - - logging.info(f"📥 正在从 {json_url} 获取数据，每个博客获取 {article_count} 篇文章") - result, lost_friends = fetch_and_process_data( - json_url = json_url, # 包含朋友信息的 JSON 文件的 URL。 - specific_RSS = specific_rss, # 包含特定 RSS 源的字典列表 [{name, url}]（来自 YAML）。 - count = article_count, # 获取每个博客的最大文章数。 - cache_file = "./temp/cache.json" # 缓存文件路径。 - ) - - if config["spider_settings"]["merge_result"]["enable"]: - - merge_url = config['spider_settings']["merge_result"]['merge_json_url'] - logging.info(f"🔀 合并功能开启，从 {merge_url} 获取外部数据") - result = marge_data_from_json_url(result, f"{merge_url}/all.json") - lost_friends = marge_errors_from_json_url(lost_friends, f"{merge_url}/errors.json") - - article_count = len(result.get("article_data", [])) - logging.info(f"📦 数据获取完毕，共有 {article_count} 篇文章，正在处理数据") +"""Friend-Circle-Lite application entrypoint.""" - future_tolerance_days = FUTURE_ARTICLE_TOLERANCE_DAYS - result = deal_with_large_data(result, future_tolerance_days=future_tolerance_days) +from __future__ import annotations - write_json("./all.json", result) - write_json("./errors.json", lost_friends) - -# ========== 邮箱推送准备 ========== -SMTP_isReady = False - -sender_email = "" -server = "" -port = 0 -use_tls = False -password = "" - -if config["email_push"]["enable"] or config["rss_subscribe"]["enable"]: - logging.info("📨 推送功能已启用，正在准备中...") - - smtp_conf = config["smtp"] - sender_email = smtp_conf["email"] - server = smtp_conf["server"] - port = smtp_conf["port"] - use_tls = smtp_conf["use_tls"] - password = os.getenv("SMTP_PWD") - - logging.info(f"📡 SMTP 服务器：{server}:{port}") - if not password or not sender_email or not server or not port: - logging.error("❌ 环境变量 SMTP_PWD 未设置，无法发送邮件") - else: - logging.info(f"🔐 密码(部分)：{password[:3]}*****") - SMTP_isReady = True - -# ========== 邮件推送（待实现）========== -if config["email_push"]["enable"] and SMTP_isReady: - logging.info("📧 邮件推送已启用") - logging.info("⚠️ 抱歉，目前尚未实现邮件推送功能") - -# ========== RSS 订阅推送 ========== -if config["rss_subscribe"]["enable"] and SMTP_isReady: - logging.info("📰 RSS 订阅推送已启用") - - # 获取 GitHub 仓库信息 - fcl_repo = os.getenv('FCL_REPO') # 仓库内置 - if fcl_repo: - github_username, github_repo = fcl_repo.split('/') - else: - github_username = str(config["rss_subscribe"]["github_username"]).strip() - github_repo = str(config["rss_subscribe"]["github_repo"]).strip() +import logging - logging.info(f"👤 GitHub 用户名：{github_username}") - logging.info(f"📁 GitHub 仓库：{github_repo}") +from friend_circle_lite.application import FriendCircleLiteApplication +from friend_circle_lite.utils.config import load_config - your_blog_url = config["rss_subscribe"]["your_blog_url"] - email_template = config["rss_subscribe"]["email_template"] - website_title = config["rss_subscribe"]["website_info"]["title"] - latest_articles = get_latest_articles_from_link( - url=your_blog_url, - count=5, - last_articles_path="./temp/newest_posts.json" # 存储上一次的文章 +def configure_logging() -> None: + """Configure the global logging style used by the CLI entrypoint.""" + logging.basicConfig( + level=logging.INFO, + format="😋 %(levelname)s: %(message)s", ) - if not latest_articles: - logging.info("📭 无新文章，无需推送") - else: - logging.info(f"🆕 获取到的最新文章：{latest_articles}") - - github_api_url = ( - f"https://api.github.com/repos/{github_username}/{github_repo}/issues" - f"?state=closed&label=subscribed&per_page=200" - ) - logging.info(f"🔎 正在从 GitHub 获取订阅邮箱：{github_api_url}") - email_list = extract_emails_from_issues(github_api_url) - - if not email_list: - logging.info("⚠️ 无订阅邮箱，请检查格式或是否有订阅者") - sys.exit(0) - logging.info(f"📬 获取到邮箱列表：{email_list}") +def main() -> None: + """Load configuration and run the application.""" + configure_logging() + app = FriendCircleLiteApplication(load_config("./conf.yaml")) + app.run() - for article in latest_articles: - template_data = { - "title": article["title"], - "summary": article["summary"], - "published": article["published"], - "link": article["link"], - "website_title": website_title, - "github_issue_url": ( - f"https://github.com/{github_username}/{github_repo}" - "/issues?q=is%3Aissue+is%3Aclosed" - ), - } - send_emails( - emails=email_list["emails"], - sender_email=sender_email, - smtp_server=server, - port=port, - password=password, - subject=f"{website_title} の最新文章：{article['title']}", - body=( - f"📄 文章标题：{article['title']}\n" - f"🔗 链接：{article['link']}\n" - f"📝 简介：{article['summary']}\n" - f"🕒 发布时间：{article['published']}" - ), - template_path=email_template, - template_data=template_data, - use_tls=use_tls - ) +if __name__ == "__main__": + main() From 3e3d7345e285992555cef33a79b7b9d4f00dda0b Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Wed, 15 Apr 2026 22:01:10 +0800 Subject: [PATCH 08/30] =?UTF-8?q?=F0=9F=98=AB=E5=AE=8C=E5=96=84=E6=97=A5?= =?UTF-8?q?=E5=BF=97=EF=BC=8C=E6=95=B4=E5=90=88=E5=A4=87=E4=BB=BD=EF=BC=8C?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=B5=84=E6=BA=90=EF=BC=8C=E6=B7=BB=E5=8A=A0?= =?UTF-8?q?=E8=BF=81=E7=A7=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- friend_circle_lite/app_config.py | 4 +- friend_circle_lite/application.py | 4 +- friend_circle_lite/cache_store.py | 164 ++++++++++++++++++++++++-- friend_circle_lite/crawler_service.py | 27 +++-- friend_circle_lite/feed_service.py | 110 +++++++++++++---- friend_circle_lite/single_friend.py | 16 ++- 6 files changed, 273 insertions(+), 52 deletions(-) diff --git a/friend_circle_lite/app_config.py b/friend_circle_lite/app_config.py index ce3c9609e86..9068cf55ee5 100644 --- a/friend_circle_lite/app_config.py +++ b/friend_circle_lite/app_config.py @@ -13,8 +13,7 @@ from dataclasses import dataclass, field -DEFAULT_CACHE_FILE = "./temp/feed_cache.sqlite3" -DEFAULT_NEWEST_POSTS_FILE = "./temp/newest_posts.json" +DEFAULT_CACHE_FILE = "./temp/cache.sqlite3" DEFAULT_ALL_JSON = "./all.json" DEFAULT_ERRORS_JSON = "./errors.json" @@ -81,7 +80,6 @@ class RuntimePaths: """Filesystem locations used by the runtime.""" cache_file: str = DEFAULT_CACHE_FILE - newest_posts_file: str = DEFAULT_NEWEST_POSTS_FILE all_json_file: str = DEFAULT_ALL_JSON errors_json_file: str = DEFAULT_ERRORS_JSON diff --git a/friend_circle_lite/application.py b/friend_circle_lite/application.py index 32b70b7dc50..1fbdcc154a0 100644 --- a/friend_circle_lite/application.py +++ b/friend_circle_lite/application.py @@ -114,8 +114,8 @@ def run_rss_subscription_if_enabled(self, mail_runtime: MailRuntime) -> None: latest_articles = get_latest_articles_from_link( url=self.config.rss_subscribe.your_blog_url, - count=5, - last_articles_path=self.config.runtime_paths.newest_posts_file, + count=10, + last_articles_path=self.config.runtime_paths.cache_file, ) if not latest_articles: logging.info("📭 无新文章，无需推送") diff --git a/friend_circle_lite/cache_store.py b/friend_circle_lite/cache_store.py index d4a45fbadca..f5628ddb234 100644 --- a/friend_circle_lite/cache_store.py +++ b/friend_circle_lite/cache_store.py @@ -1,7 +1,7 @@ -"""Persistent RSS cache storage. +"""Persistent RSS cache and article tracking storage. -SQLite is used for the feed cache because it is more robust than hand-edited -text formats for internal state: +SQLite is used for both feed cache and article tracking because it is more robust +than hand-edited text formats for internal state: - schema is explicit and stable; - writes are transactional; @@ -17,11 +17,12 @@ import json import logging import sqlite3 +from datetime import datetime from pathlib import Path import yaml -from friend_circle_lite.models import CacheRecord +from friend_circle_lite.models import Article, CacheRecord class FeedCacheStore: @@ -41,10 +42,10 @@ def load_records(self) -> list[CacheRecord]: migrated_records = self._load_legacy_records() if migrated_records: if self.save_records(migrated_records): - logging.info(f"已迁移 {len(migrated_records)} 条 RSS 缓存记录到 {self.cache_path}。") + logging.info(f"已从旧格式迁移 {len(migrated_records)} 条 RSS 缓存到 SQLite") return migrated_records - logging.info(f"缓存文件 {self.cache_path} 不存在，将在首次成功抓取后创建。") + logging.info(f"RSS 缓存文件不存在，将在首次抓取后自动创建") return [] def save_records(self, records: list[CacheRecord]) -> bool: @@ -62,10 +63,10 @@ def save_records(self, records: list[CacheRecord]) -> bool: [(record.name, record.url, record.source) for record in sorted(records, key=lambda item: item.name)], ) connection.commit() - logging.info(f"缓存已保存到 {self.cache_path}（{len(records)} 条）。") + logging.info(f"RSS 缓存已保存（{len(records)} 条）") return True except Exception as exc: - logging.error(f"保存缓存文件失败: {self.cache_path}, 错误信息: {exc}") + logging.error(f"保存 RSS 缓存失败: {exc}") return False def _load_from_sqlite(self) -> list[CacheRecord]: @@ -77,7 +78,7 @@ def _load_from_sqlite(self) -> list[CacheRecord]: "SELECT name, url, source FROM feed_cache ORDER BY name" ).fetchall() except Exception as exc: - logging.warning(f"读取 SQLite 缓存失败: {self.cache_path}, 错误信息: {exc}") + logging.warning(f"读取 RSS 缓存失败: {exc}") return [] return [ @@ -124,7 +125,7 @@ def _load_legacy_json_cache(self) -> list[CacheRecord]: with open(legacy_path, "r", encoding="utf-8") as file: payload = json.load(file) except Exception as exc: - logging.warning(f"读取旧缓存文件失败: {legacy_path}, 错误信息: {exc}") + logging.warning(f"读取旧 JSON 缓存失败: {exc}") return [] if not isinstance(payload, list): @@ -145,7 +146,7 @@ def _load_legacy_yaml_cache(self) -> list[CacheRecord]: with open(legacy_path, "r", encoding="utf-8") as file: payload = yaml.safe_load(file) or {} except Exception as exc: - logging.warning(f"读取旧 YAML 缓存失败: {legacy_path}, 错误信息: {exc}") + logging.warning(f"读取旧 YAML 缓存失败: {exc}") return [] items = payload.get("feeds", []) if isinstance(payload, dict) else [] @@ -164,3 +165,144 @@ def _normalize_legacy_items(items: list[object]) -> list[CacheRecord]: if name and url: records.append(CacheRecord(name=name, url=url, source=source)) return records + + +class ArticleTrackingStore: + """Persist and load article tracking data using SQLite.""" + + def __init__(self, storage_path: str | Path | None, max_tracked_articles: int = 10): + self.storage_path = Path(storage_path) if storage_path else None + self.max_tracked_articles = max_tracked_articles + + def load_articles(self) -> list[Article]: + """Load tracked articles from SQLite, migrating from legacy JSON if needed.""" + if not self.storage_path: + return [] + + if self.storage_path.exists(): + return self._load_from_sqlite() + + # Try to migrate from legacy JSON format + migrated_articles = self._load_legacy_json() + if migrated_articles: + if self.save_articles(migrated_articles): + logging.info(f"已从旧 JSON 格式迁移 {len(migrated_articles)} 篇文章记录到 SQLite") + return migrated_articles + + logging.info(f"文章追踪数据不存在，这是首次运行") + return [] + + def save_articles(self, articles: list[Article]) -> bool: + """Persist articles to SQLite, keeping only the most recent max_tracked_articles.""" + if not self.storage_path: + return True + + try: + # Sort by date and keep only the most recent articles + valid_articles = [article for article in articles if article.published] + valid_articles.sort( + key=lambda item: datetime.strptime(item.published, "%Y-%m-%d %H:%M"), + reverse=True + ) + articles_to_save = valid_articles[:self.max_tracked_articles] + + self.storage_path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(self.storage_path) as connection: + self._ensure_schema(connection) + connection.execute("DELETE FROM article_tracking") + connection.executemany( + """INSERT INTO article_tracking(title, author, link, published, summary, content) + VALUES (?, ?, ?, ?, ?, ?)""", + [ + ( + article.title, + article.author, + article.link, + article.published, + article.summary, + article.content, + ) + for article in articles_to_save + ], + ) + connection.commit() + return True + except Exception as exc: + logging.error(f"保存文章追踪数据失败: {exc}") + return False + + def _load_from_sqlite(self) -> list[Article]: + """Load articles from the SQLite database.""" + try: + with sqlite3.connect(self.storage_path) as connection: + self._ensure_schema(connection) + rows = connection.execute( + """SELECT title, author, link, published, summary, content + FROM article_tracking + ORDER BY published DESC""" + ).fetchall() + except Exception as exc: + logging.warning(f"读取文章追踪数据失败: {exc}") + return [] + + return [ + Article( + title=title or "", + author=author or "", + link=link or "", + published=published or "", + summary=summary or "", + content=content or "", + ) + for title, author, link, published, summary, content in rows + ] + + @staticmethod + def _ensure_schema(connection: sqlite3.Connection) -> None: + """Create the article tracking table when it does not exist yet.""" + connection.execute( + """ + CREATE TABLE IF NOT EXISTS article_tracking ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + author TEXT NOT NULL, + link TEXT NOT NULL, + published TEXT NOT NULL, + summary TEXT, + content TEXT + ) + """ + ) + + def _load_legacy_json(self) -> list[Article]: + """Read the old JSON format for seamless upgrades.""" + if not self.storage_path: + return [] + + legacy_path = self.storage_path.with_name("newest_posts.json") + if not legacy_path.exists(): + return [] + + try: + with open(legacy_path, "r", encoding="utf-8") as file: + payload = json.load(file) + except Exception as exc: + logging.warning(f"读取旧 JSON 文章追踪文件失败: {exc}") + return [] + + articles_data = payload.get("articles", []) if isinstance(payload, dict) else [] + articles: list[Article] = [] + for item in articles_data: + if not isinstance(item, dict): + continue + articles.append( + Article( + title=item.get("title", ""), + author=item.get("author", ""), + link=item.get("link", ""), + published=item.get("published", ""), + summary=item.get("summary", ""), + content=item.get("content", ""), + ) + ) + return articles diff --git a/friend_circle_lite/crawler_service.py b/friend_circle_lite/crawler_service.py index 34302ac0e35..002c9d35640 100644 --- a/friend_circle_lite/crawler_service.py +++ b/friend_circle_lite/crawler_service.py @@ -34,12 +34,17 @@ def __init__(self, discovery_service: FeedDiscoveryService, configured_feeds: li def resolve(self, website: Website) -> FeedEndpoint | None: configured = self.feed_lookup.get(website.name) if configured: - logging.info(f"“{website.name}” 使用预设 RSS 源：{configured.url} （source={configured.source}）。") + if configured.source == 'manual': + logging.info(f"'{website.name}' 使用预设 RSS 源：{configured.url}") + elif configured.source == 'cache': + logging.info(f"'{website.name}' 使用缓存 RSS 源：{configured.url}") + else: + logging.info(f"'{website.name}' 使用 RSS 源：{configured.url} (来源: {configured.source})") return FeedEndpoint(url=configured.url, feed_type="specific", source=configured.source) discovered = self.discovery_service.discover(website.url) if discovered: - logging.info(f"“{website.name}” 自动探测 RSS：type：{discovered.feed_type}, url：{discovered.url} 。") + logging.info(f"'{website.name}' 自动探测到 RSS：{discovered.url}") return discovered @@ -62,26 +67,29 @@ def crawl(self, website: Website, count: int) -> CrawlResult: parse_error = endpoint is not None and not articles if parse_error and endpoint and endpoint.source in ("cache", "unknown"): - logging.info(f"缓存 RSS 无效，重新探测：{website.name} ({website.url})。") + logging.warning(f"'{website.name}' 缓存的 RSS 源无效，尝试重新探测...") rediscovered = self.resolver.discovery_service.discover(website.url) if rediscovered: articles = self._parse_articles(rediscovered, website, count) if articles: endpoint = rediscovered cache_update = CacheUpdate(action="set", name=website.name, url=rediscovered.url, reason="repair_cache") + logging.info(f"'{website.name}' 重新探测成功，更新缓存：{rediscovered.url}") else: endpoint = None cache_update = CacheUpdate(action="delete", name=website.name, url=None, reason="remove_invalid") + logging.warning(f"'{website.name}' 重新探测失败，删除无效缓存") else: endpoint = None cache_update = CacheUpdate(action="delete", name=website.name, url=None, reason="remove_invalid") + logging.warning(f"'{website.name}' 未找到有效 RSS，删除无效缓存") status = "active" if articles else "error" if not articles: if endpoint is None: - logging.warning(f"{website.name} 的博客 {website.url} 未找到有效 RSS。") + logging.warning(f"'{website.name}' 的博客 {website.url} 未找到有效 RSS。") else: - logging.warning(f"{website.name} 的 RSS {endpoint.url} 未解析出文章。") + logging.warning(f"'{website.name}' 的 RSS {endpoint.url} 未解析出文章。") return CrawlResult( website=website, @@ -214,10 +222,15 @@ def _apply_cache_updates(self, cache_records: list[CacheRecord], crawl_results: for name, update in unique_updates.items(): if update.action == "set" and update.url: cache_map[name] = CacheRecord(name=name, url=update.url, source="cache") - logging.info(f"缓存更新：SET {name} -> {update.url} ({update.reason})") + if update.reason == "auto_discovered": + logging.info(f"💾 缓存新增：{name} -> {update.url} (自动探测)") + elif update.reason == "repair_cache": + logging.info(f"💾 缓存修复：{name} -> {update.url} (重新探测)") + else: + logging.info(f"💾 缓存更新：{name} -> {update.url} ({update.reason})") elif update.action == "delete" and name in cache_map: cache_map.pop(name) - logging.info(f"缓存更新：DELETE {name} ({update.reason})") + logging.info(f"🗑️ 缓存删除：{name} (RSS 源失效)") self.cache_store.save_records(list(cache_map.values())) diff --git a/friend_circle_lite/feed_service.py b/friend_circle_lite/feed_service.py index 777b19e87b1..792564f9142 100644 --- a/friend_circle_lite/feed_service.py +++ b/friend_circle_lite/feed_service.py @@ -55,7 +55,7 @@ def discover(self, website_url: str) -> FeedEndpoint | None: if " list[Artic response.encoding = response.apparent_encoding or "utf-8" feed = feedparser.parse(response.text) except Exception as exc: - logging.error(f"无法解析 FEED 地址：{feed_url} ，请自行排查原因！错误信息: {exc}") + logging.error(f"解析 RSS 失败：{feed_url}，错误: {exc}") return [] default_author = feed.feed.author if "author" in feed.feed else "" @@ -116,34 +116,92 @@ def _extract_published_time(entry) -> str: class LatestArticleTracker: """Track whether a website published new posts since the last crawl.""" - def __init__(self, storage_path: str | Path): - self.storage_path = Path(storage_path) + def __init__(self, storage_path: str | Path, max_tracked_articles: int = 10): + from friend_circle_lite.cache_store import ArticleTrackingStore + self.store = ArticleTrackingStore(storage_path, max_tracked_articles) def diff_and_persist(self, latest_articles: list[Article]) -> list[dict] | None: - """Return newly seen articles and update the local snapshot file.""" - previous_links = self._load_previous_links() - updated_articles = [article.to_tracking_dict() for article in latest_articles if article.link not in previous_links] - self._persist(latest_articles) - return updated_articles if updated_articles else None - - def _load_previous_links(self) -> set[str]: - if not self.storage_path.exists(): - return set() - try: - with open(self.storage_path, "r", encoding="utf-8") as file: - payload = json.load(file) - except Exception as exc: - logging.warning(f"读取最新文章缓存失败: {self.storage_path}, 错误信息: {exc}") - return set() + """Return newly seen articles and update the local storage. + + Returns None if: + - This is the first run (no previous data exists) + - No new articles are found + - New articles exist but are not newer than the most recent tracked article + """ + previous_articles = self.store.load_articles() + + # First run: no previous data exists, skip sending to prevent sending old articles + if not previous_articles: + logging.info(f"首次运行：跳过推送以防止发送旧文章") + self.store.save_articles(latest_articles) + return None + + previous_latest_date = self._get_latest_date(previous_articles) + + # Find articles that are truly new (check only: link, title, published) + new_articles = [] + for article in latest_articles: + if self._is_truly_new_article(article, previous_articles): + new_articles.append(article) + + if not new_articles: + self.store.save_articles(latest_articles) + return None + + # Filter new articles: only keep those newer than the previous latest date + truly_new_articles = [] + for article in new_articles: + if not article.published: + continue + try: + article_date = datetime.strptime(article.published, "%Y-%m-%d %H:%M") + if previous_latest_date is None or article_date > previous_latest_date: + truly_new_articles.append(article) + except Exception as exc: + logging.warning(f"解析文章日期失败: {article.title}, 日期: {article.published}, 错误: {exc}") + continue + + self.store.save_articles(latest_articles) + + if truly_new_articles: + logging.info(f"发现 {len(truly_new_articles)} 篇新文章（日期比之前更新）") + return [article.to_tracking_dict() for article in truly_new_articles] + else: + logging.info(f"发现 {len(new_articles)} 篇新文章，但日期不够新，跳过推送") + return None - articles = payload.get("articles", []) if isinstance(payload, dict) else [] - return {article.get("link", "") for article in articles if isinstance(article, dict) and article.get("link")} + @staticmethod + def _is_truly_new_article(article: Article, previous_articles: list[Article]) -> bool: + """Check if an article is truly new by comparing link, title, and published date. + + An article is considered new only if its link, title, and published date + do not match any previous article (empty values are skipped). + """ + for prev in previous_articles: + # Check link, title, and published: if any non-empty field matches, it's not new + if article.link and article.link == prev.link: + return False + if article.title and article.title == prev.title: + return False + if article.published and article.published == prev.published: + return False + + return True - def _persist(self, latest_articles: list[Article]) -> None: - self.storage_path.parent.mkdir(parents=True, exist_ok=True) - payload = {"articles": [article.to_tracking_dict() for article in latest_articles]} - with open(self.storage_path, "w", encoding="utf-8") as file: - json.dump(payload, file, ensure_ascii=False, indent=4) + @staticmethod + def _get_latest_date(articles: list[Article]) -> datetime | None: + """Find the latest publish date from a list of articles.""" + latest_date = None + for article in articles: + if not article.published: + continue + try: + article_date = datetime.strptime(article.published, "%Y-%m-%d %H:%M") + if latest_date is None or article_date > latest_date: + latest_date = article_date + except Exception: + continue + return latest_date def extract_blog_origin(url: str) -> str: diff --git a/friend_circle_lite/single_friend.py b/friend_circle_lite/single_friend.py index 50cb4803264..6e240e18e0d 100644 --- a/friend_circle_lite/single_friend.py +++ b/friend_circle_lite/single_friend.py @@ -55,13 +55,20 @@ def process_friend(friend, session: requests.Session, count: int, specific_and_c endpoint = None cache_update = CacheUpdate(action='none', name=website.name) if entry: - endpoint = {'feed_type': 'specific', 'url': entry['url'], 'source': entry.get('source', 'unknown')} - logging.info(f"“{website.name}” 使用预设 RSS 源：{entry['url']} （source={entry.get('source', 'unknown')}）。") + source = entry.get('source', 'unknown') + endpoint = {'feed_type': 'specific', 'url': entry['url'], 'source': source} + if source == 'manual': + logging.info(f"'{website.name}' 使用预设 RSS 源：{entry['url']}") + elif source == 'cache': + logging.info(f"'{website.name}' 使用缓存 RSS 源：{entry['url']}") + else: + logging.info(f"'{website.name}' 使用 RSS 源：{entry['url']} (来源: {source})") else: feed_type, feed_url = check_feed(website.url, session) if feed_type != 'none' and feed_url: endpoint = {'feed_type': feed_type, 'url': feed_url, 'source': 'auto'} cache_update = CacheUpdate(action='set', name=website.name, url=feed_url, reason='auto_discovered') + logging.info(f"'{website.name}' 自动探测到 RSS：{feed_url}") articles = [] parse_error = endpoint is not None @@ -80,7 +87,7 @@ def process_friend(friend, session: requests.Session, count: int, specific_and_c parse_error = not articles if parse_error and endpoint and endpoint['source'] in ('cache', 'unknown'): - logging.info(f"缓存 RSS 无效，重新探测：{website.name} ({website.url})。") + logging.warning(f"'{website.name}' 缓存的 RSS 源无效，尝试重新探测...") new_feed_type, new_feed_url = check_feed(website.url, session) if new_feed_type != 'none' and new_feed_url: reparsed = FeedParserService(session).parse(new_feed_url, count=count, blog_url=website.url) @@ -97,12 +104,15 @@ def process_friend(friend, session: requests.Session, count: int, specific_and_c if articles: endpoint = {'feed_type': new_feed_type, 'url': new_feed_url, 'source': 'auto'} cache_update = CacheUpdate(action='set', name=website.name, url=new_feed_url, reason='repair_cache') + logging.info(f"'{website.name}' 重新探测成功，更新缓存：{new_feed_url}") else: endpoint = None cache_update = CacheUpdate(action='delete', name=website.name, url=None, reason='remove_invalid') + logging.warning(f"'{website.name}' 重新探测失败，删除无效缓存") else: endpoint = None cache_update = CacheUpdate(action='delete', name=website.name, url=None, reason='remove_invalid') + logging.warning(f"'{website.name}' 未找到有效 RSS，删除无效缓存") return { 'name': website.name, From e016ec9b0643af005413858aeff1a7647253634f Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Wed, 15 Apr 2026 22:08:20 +0800 Subject: [PATCH 09/30] =?UTF-8?q?=F0=9F=98=98=E5=AE=8C=E5=96=84=E6=97=B6?= =?UTF-8?q?=E9=97=B4=E8=A7=A3=E6=9E=90=E5=8A=9F=E8=83=BD=EF=BC=8C=E9=98=B2?= =?UTF-8?q?=E6=AD=A2=E5=87=BA=E7=8E=B0=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- friend_circle_lite/feed_service.py | 49 +++++++++++++++++++++++++++--- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/friend_circle_lite/feed_service.py b/friend_circle_lite/feed_service.py index 792564f9142..ade9aca5276 100644 --- a/friend_circle_lite/feed_service.py +++ b/friend_circle_lite/feed_service.py @@ -96,16 +96,57 @@ def parse(self, feed_url: str, count: int = 5, blog_url: str = "") -> list[Artic articles.append(article) valid_articles = [article for article in articles if article.published] - valid_articles.sort(key=lambda item: datetime.strptime(item.published, "%Y-%m-%d %H:%M"), reverse=True) - return valid_articles[:count] if count < len(valid_articles) else valid_articles + + # 过滤掉无法解析的日期格式 + def safe_parse_date(article): + try: + return datetime.strptime(article.published, "%Y-%m-%d %H:%M") + except ValueError: + logging.warning(f"文章 {article.title} 的发布时间格式异常: {article.published}，已跳过") + return None + + # 只保留能成功解析日期的文章 + valid_articles_with_dates = [] + for article in valid_articles: + parsed_date = safe_parse_date(article) + if parsed_date: + valid_articles_with_dates.append((article, parsed_date)) + + # 按日期排序 + valid_articles_with_dates.sort(key=lambda item: item[1], reverse=True) + sorted_articles = [item[0] for item in valid_articles_with_dates] + + return sorted_articles[:count] if count < len(sorted_articles) else sorted_articles @staticmethod def _extract_published_time(entry) -> str: """Extract a normalized publish time from a feed entry.""" + import time + + def convert_time_to_string(time_value): + """Convert various time formats to string.""" + if isinstance(time_value, str): + return time_value + elif isinstance(time_value, time.struct_time): + # 检查年份是否异常 + if time_value.tm_year < 1900: + logging.warning(f"文章 {entry.get('title', 'Unknown')} 的时间年份异常: {time_value.tm_year}，已跳过") + return "" + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time_value) + else: + logging.warning(f"文章 {entry.get('title', 'Unknown')} 的时间格式未知: {type(time_value)}，已跳过") + return "" + if "published" in entry: - return format_published_time(entry.published) + time_str = convert_time_to_string(entry.published) + if not time_str: + return "" + return format_published_time(time_str) if "updated" in entry: - published = format_published_time(entry.updated) + time_str = convert_time_to_string(entry.updated) + if not time_str: + return "" + published = format_published_time(time_str) logging.warning(f"文章 {entry.title} 未包含发布时间，已使用更新时间 {published}") return published From 6c97d85efd4cbb78e2b5580739de768245d55ead Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Wed, 15 Apr 2026 22:17:32 +0800 Subject: [PATCH 10/30] =?UTF-8?q?=F0=9F=A4=A3=E5=BC=BA=E5=88=B6=E7=BC=96?= =?UTF-8?q?=E7=A0=81=E4=B8=BAutf-8=E4=BB=A5=E8=A7=A3=E5=86=B3=E7=BC=96?= =?UTF-8?q?=E7=A0=81=E9=94=99=E8=AF=AF=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- friend_circle_lite/feed_service.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/friend_circle_lite/feed_service.py b/friend_circle_lite/feed_service.py index ade9aca5276..a9c92e79718 100644 --- a/friend_circle_lite/feed_service.py +++ b/friend_circle_lite/feed_service.py @@ -73,7 +73,8 @@ def parse(self, feed_url: str, count: int = 5, blog_url: str = "") -> list[Artic """ try: response = self.session.get(feed_url, headers=HEADERS_XML, timeout=timeout) - response.encoding = response.apparent_encoding or "utf-8" + # 强制使用 UTF-8 编码，因为 apparent_encoding 可能检测错误 + response.encoding = "utf-8" feed = feedparser.parse(response.text) except Exception as exc: logging.error(f"解析 RSS 失败：{feed_url}，错误: {exc}") From ed8799d244bb026df00ca6743399aabb01da72ea Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Thu, 16 Apr 2026 20:53:23 +0800 Subject: [PATCH 11/30] =?UTF-8?q?=F0=9F=98=81=E4=BF=AE=E6=94=B9=E5=89=8D?= =?UTF-8?q?=E7=AB=AF=E6=98=BE=E7=A4=BA=E6=95=88=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- main/fclite.css | 312 +++++++++++++++++++++++++++++++++++++--------- main/fclite.js | 98 ++++++++++++--- static/index.html | 81 +++++++++--- 3 files changed, 399 insertions(+), 92 deletions(-) diff --git a/main/fclite.css b/main/fclite.css index 52567f3b095..84ae6454b89 100644 --- a/main/fclite.css +++ b/main/fclite.css @@ -47,74 +47,254 @@ #random-article { display: flex; + flex-direction: column; + gap: 16px; position: relative; width: 100%; - margin: 8px 0; + margin: 0 0 16px 0; background-color: var(--container-bg-color); - border-radius: 10px; - border:1px solid var(--border-color); - height: 210px; - transition: border 0.3s; + border-radius: 8px; + border: 1px solid var(--border-color); + padding: 20px; + transition: all 0.3s ease; + overflow: hidden; + box-sizing: border-box; } -.random-container { - position: relative; - margin: 20px; - width: 90%; - height: 170px; +.random-top { + width: 100%; +} + +.random-stats { + display: grid; + grid-template-columns: repeat(4, 1fr); + gap: 12px; +} + +.stat-item { + text-align: center; + padding: 12px; + background-color: var(--background-color); + border-radius: 6px; + border: 1px solid var(--border-color); + transition: all 0.3s ease; } -.random-container:hover .random-title { - font-size: 32px; +.stat-item:hover { + border-color: var(--hover-color); } -.random-author { - font-size: 14px; +.stat-num { + font-size: 24px; + font-weight: 700; + color: var(--hover-color); + line-height: 1; + margin-bottom: 4px; +} + +.stat-text { + font-size: 11px; color: var(--author-color); - margin-bottom: 10px; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.random-content { + display: flex; + align-items: flex-end; + justify-content: space-between; + gap: 20px; +} + +.random-container { + flex: 1; + min-width: 0; } .random-container-title { - font-size: 20px; + font-size: 11px; font-weight: 700; - margin-bottom: 20px; + margin-bottom: 8px; + color: var(--text-color); + opacity: 0.6; + text-transform: uppercase; + letter-spacing: 1px; } .random-title { - margin-bottom: 10px; - font-size: 30px; - transition: font-size 0.3s ease-in-out; - white-space: nowrap; + font-size: 18px; + font-weight: 600; + line-height: 1.3; + margin-bottom: 8px; + transition: all 0.3s ease; + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; overflow: hidden; text-overflow: ellipsis; } +.random-meta { + display: flex; + gap: 8px; + font-size: 12px; +} + +.random-author, +.random-date { + display: inline-flex; + align-items: center; + gap: 4px; + padding: 4px 10px; + background-color: var(--background-color); + border: 1px solid var(--border-color); + border-radius: 12px; + color: var(--author-color); + transition: all 0.3s ease; +} + +.random-author:hover, +.random-date:hover { + border-color: var(--hover-color); +} + .random-button-container { - position: absolute; - bottom: 20px; - right: 20px; - border: none; - border-radius: 20px; - cursor: pointer; - font-size: 14px; - transition: background-color 0.3s ease-in-out; + display: flex; + gap: 10px; + align-items: center; + flex-shrink: 0; } .random-button-container a { - margin-right: 10px; - color: #aaaaaa !important; + padding: 8px 16px; + color: var(--text-color) !important; text-decoration: none !important; + border: 1px solid var(--border-color); + border-radius: 6px; + background-color: var(--background-color); + transition: all 0.3s ease; + font-size: 13px; + white-space: nowrap; +} + +.random-button-container a:hover { + border-color: var(--hover-color); + color: var(--hover-color) !important; } .random-link-button { - padding: 10px 20px; + padding: 8px 20px; border: none; - border-radius: 20px; + border-radius: 6px; background-color: var(--hover-color); color: #fff; cursor: pointer; + font-size: 13px; + font-weight: 500; + transition: all 0.3s ease; + white-space: nowrap; +} + +.random-link-button:hover { + opacity: 0.9; +} + +/* 响应式设计 */ +@media screen and (max-width: 768px) { + .random-stats { + grid-template-columns: repeat(2, 1fr); + } +} + +@media screen and (max-width: 600px) { + .random-content { + flex-direction: column; + align-items: center; + gap: 16px; + } + + .random-container { + text-align: center; + } + + .random-meta { + justify-content: center; + } + + .random-button-container { + justify-content: center; + width: 100%; + } +} + +@media screen and (max-width: 400px) { + .random-stats { + gap: 8px; + } + + .stat-item { + padding: 10px 8px; + } + + .stat-num { + font-size: 20px; + } +} + +/* 加载和错误状态样式 */ +.loading-placeholder, +.error-placeholder { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + padding: 40px 20px; + gap: 12px; +} + +.loading-spinner { + width: 40px; + height: 40px; + border: 3px solid var(--border-color); + border-top-color: var(--hover-color); + border-radius: 50%; + animation: spin 1s linear infinite; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} + +.loading-text { font-size: 14px; - transition: background-color 0.3s ease-in-out; + color: var(--author-color); +} + +.error-icon { + font-size: 48px; +} + +.error-text { + font-size: 14px; + color: var(--author-color); + text-align: center; +} + +.retry-button { + margin-top: 8px; + padding: 8px 20px; + border: none; + border-radius: 6px; + background-color: var(--hover-color); + color: #fff; + cursor: pointer; + font-size: 13px; + font-weight: 500; + transition: all 0.3s ease; +} + +.retry-button:hover { + opacity: 0.9; } .modal { @@ -180,6 +360,7 @@ height: 250px; right: -20px; bottom: -20px; + object-fit: cover; transition: transform 0.6s ease !important; } @@ -189,6 +370,7 @@ border-radius: 50% !important; width: 110px; height: 110px; + object-fit: cover; } #modal-author-name-link { @@ -258,36 +440,50 @@ .articles-container { display: grid; grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); - gap: 8px; + gap: 10px; width: 100%; } .card { background-color: var(--container-bg-color); - border-radius: 10px; - padding: 10px; + border-radius: 8px; + padding: 12px; border: 1px solid var(--border-color); position: relative; overflow: hidden; display: flex; flex-direction: column; justify-content: space-between; - height: 150px; - transition: border 0.3s; + height: 120px; + transition: all 0.3s ease; + opacity: 0; + transform: translateY(20px); + animation: fadeInUp 0.5s ease forwards; +} + +@keyframes fadeInUp { + to { + opacity: 1; + transform: translateY(0); + } +} + +.card:hover { + border-color: var(--hover-color); + transform: translateY(-4px); } -.card:hover, #random-article:hover { - border: 1px solid var(--hover-color); + border-color: var(--hover-color); } .card-title { z-index: 1; - font-size: 17px; + font-size: 16px; color: var(--text-color); - font-weight: 520; + font-weight: 600; cursor: pointer; - margin-bottom: 10px; + margin-bottom: 12px; line-height: 1.5; max-height: 4.5em; overflow: hidden; @@ -296,36 +492,35 @@ -webkit-line-clamp: 3; line-clamp: 3; -webkit-box-orient: vertical; - transition: color 0.3s; + transition: all 0.3s ease; } .card-title:hover { color: var(--hover-color); - text-decoration: underline; } .card-author, .card-date { font-size: 12px; color: var(--author-color); - padding: 5px; + padding: 6px 10px; line-height: 15px; } .card-author:hover { - box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2); + border-color: var(--hover-color); } .card-author { cursor: pointer; background-color: var(--background-color); border: 1px solid var(--border-color); - border-radius: 15px; + border-radius: 12px; display: flex; padding-right: 10px; width: fit-content; align-items: center; - transition: box-shadow 0.2s; + transition: all 0.3s ease; } #friend-circle-lite-root .card-author img { @@ -357,6 +552,7 @@ width: 140px; height: 140px; opacity: 0.4; + object-fit: cover; transition: transform 0.6s ease, bottom 0.3s ease, right 0.3s ease; } @@ -368,22 +564,26 @@ #load-more-btn { color: var(--text-color); - font-size: 15px; + font-size: 13px; + font-weight: 500; background-color: var(--container-bg-color); cursor: pointer; - width: 200px; - border-radius: 10px; + width: 140px; + max-width: 100%; + border-radius: 20px; border: 1px solid var(--border-color); - padding: 3px; - transition: all 0.3s; + padding: 8px 24px; + transition: all 0.3s ease; margin: 20px auto; display: block; } #load-more-btn:hover { background-color: var(--hover-color); - width: 300px; + border-color: var(--hover-color); color: white; + width: 180px; + max-width: 100%; } #stats-container { diff --git a/main/fclite.js b/main/fclite.js index ac6c12723ca..d4dc75d0761 100644 --- a/main/fclite.js +++ b/main/fclite.js @@ -17,6 +17,12 @@ function initialize_fc_lite() { const randomArticleContainer = document.createElement('div'); randomArticleContainer.id = 'random-article'; + randomArticleContainer.innerHTML = ` +
+ +
加载中...
+
+ `; root.appendChild(randomArticleContainer); const container = document.createElement('div'); @@ -29,7 +35,6 @@ function initialize_fc_lite() { loadMoreBtn.innerText = '再来亿点'; root.appendChild(loadMoreBtn); - // 创建统计信息容器 const statsContainer = document.createElement('div'); statsContainer.id = 'stats-container'; root.appendChild(statsContainer); @@ -51,40 +56,68 @@ function initialize_fc_lite() { } } + // 设置10秒超时 + const timeoutId = setTimeout(() => { + showError('加载超时，请刷新页面重试'); + }, 10000); + fetch(`${UserConfig.private_api_url}all.json`) - .then(response => response.json()) + .then(response => { + clearTimeout(timeoutId); + if (!response.ok) { + throw new Error('网络响应错误'); + } + return response.json(); + }) .then(data => { localStorage.setItem(cacheKey, JSON.stringify(data)); localStorage.setItem(cacheTimeKey, now.toString()); processArticles(data); }) + .catch(error => { + clearTimeout(timeoutId); + console.error('加载失败:', error); + showError('加载失败，请检查网络连接'); + }) .finally(() => { loadMoreBtn.innerText = '再来亿点'; // 恢复按钮文本 }); } + function showError(message) { + randomArticleContainer.innerHTML = ` +
+
⚠️
+
${message}
+ +
+ `; + } + function processArticles(data) { allArticles = data.article_data; // 处理统计数据 const stats = data.statistical_data; + statsContainer.innerHTML = `
Powered by: FriendCircleLite
Designed By: LiuShen
-
订阅:${stats.friends_num} 活跃:${stats.active_num} 总文章数:${stats.article_num}
更新时间:${stats.last_updated_time}
`; - displayRandomArticle(); // 显示随机友链卡片 + displayRandomArticle(stats); // 显示随机友链卡片，传入统计数据 const articles = allArticles.slice(start, start + UserConfig.page_turning_number); - articles.forEach(article => { + articles.forEach((article, index) => { const card = document.createElement('div'); card.className = 'card'; + card.style.animationDelay = `${index * 0.05}s`; const title = document.createElement('div'); title.className = 'card-title'; title.innerText = article.title; + title.title = article.title; // 添加完整标题的提示 card.appendChild(title); title.onclick = () => window.open(article.link, '_blank'); @@ -92,8 +125,8 @@ function initialize_fc_lite() { author.className = 'card-author'; const authorImg = document.createElement('img'); authorImg.className = 'no-lightbox'; - authorImg.src = article.avatar || UserConfig.error_img; // 使用默认头像 - authorImg.onerror = () => authorImg.src = UserConfig.error_img; // 头像加载失败时使用默认头像 + authorImg.src = article.avatar || UserConfig.error_img; + authorImg.onerror = () => authorImg.src = UserConfig.error_img; author.appendChild(authorImg); author.appendChild(document.createTextNode(article.author)); card.appendChild(author); @@ -110,7 +143,7 @@ function initialize_fc_lite() { const bgImg = document.createElement('img'); bgImg.className = 'card-bg no-lightbox'; bgImg.src = article.avatar || UserConfig.error_img; - bgImg.onerror = () => bgImg.src = UserConfig.error_img; // 头像加载失败时使用默认头像 + bgImg.onerror = () => bgImg.src = UserConfig.error_img; card.appendChild(bgImg); container.appendChild(card); @@ -124,25 +157,54 @@ function initialize_fc_lite() { } // 显示随机文章的逻辑 - function displayRandomArticle() { + function displayRandomArticle(stats) { const randomArticle = allArticles[Math.floor(Math.random() * allArticles.length)]; randomArticleContainer.innerHTML = ` -
-
随机钓鱼
-
${randomArticle.title}
-
作者: ${randomArticle.author}
+
+
+
+
${stats.friends_num}
+
订阅
+
+
+
${stats.active_num}
+
活跃
+
+
+
${stats.article_num}
+
文章
+
+
+
${stats.error_num}
+
失败
+
+
-
- 刷新 - +
+
+
🎲 随便转转
+
${randomArticle.title}
+
+ ✍️ ${randomArticle.author} + 📅 ${randomArticle.created.substring(0, 10)} +
+
+
+ 🔄 换一篇 + +
`; // 为刷新按钮添加事件监听器 const refreshBtn = document.getElementById('refresh-random-article'); refreshBtn.addEventListener('click', function (event) { - event.preventDefault(); // 阻止默认的跳转行为 - displayRandomArticle(); // 调用显示随机文章的逻辑 + event.preventDefault(); + randomArticleContainer.style.opacity = '0.5'; + setTimeout(() => { + displayRandomArticle(stats); + randomArticleContainer.style.opacity = '1'; + }, 200); }); } diff --git a/static/index.html b/static/index.html index c6500042755..b725caccdc0 100644 --- a/static/index.html +++ b/static/index.html @@ -16,7 +16,7 @@ background-attachment: fixed; background-repeat: no-repeat; background-position: center; - font-family: Arial, sans-serif; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif; display: flex; flex-direction: column; justify-content: center; @@ -32,21 +32,30 @@ position: fixed; bottom: 20px; right: 20px; - width: 90px; - height: 30px; - background-color: #007BFF; + width: 100px; + height: 40px; + background: linear-gradient(135deg, #007BFF, #0056b3); color: white; border: none; border-radius: 20px; - font-size: 16px; + font-size: 14px; + font-weight: 600; cursor: pointer; display: flex; align-items: center; justify-content: center; - box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2); + box-shadow: 0 4px 12px rgba(0, 123, 255, 0.3); + transition: all 0.3s ease; + } + + #theme-toggle:hover { + transform: translateY(-2px); + box-shadow: 0 6px 20px rgba(0, 123, 255, 0.4); } .container { - background: rgba(255, 255, 255, 0.8); + background: rgba(255, 255, 255, 0.9); + backdrop-filter: blur(20px); + -webkit-backdrop-filter: blur(20px); display: flex; text-align: center; align-items: center; @@ -54,6 +63,7 @@ width: 100%; flex-direction: column; justify-content: center; + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1); } .root-container { @@ -73,10 +83,26 @@ width: 150px; height: 150px; border-radius: 50%; + box-shadow: 0 8px 24px rgba(0, 123, 255, 0.2); + transition: all 0.3s ease; + animation: float 3s ease-in-out infinite; } + + .avatar:hover { + transform: scale(1.05); + box-shadow: 0 12px 32px rgba(0, 123, 255, 0.3); + } + + @keyframes float { + 0%, 100% { transform: translateY(0px); } + 50% { transform: translateY(-10px); } + } + p { color: #666; margin-bottom: 30px; + font-size: 18px; + line-height: 1.6; } .button-container { margin-top: 20px; @@ -88,17 +114,20 @@ background-color: rgb(255, 255, 255); border: 2px solid #007BFF; color: #007BFF; - padding: 10px 20px; + padding: 12px 28px; border-radius: 25px; text-decoration: none; font-size: 16px; - font-weight: bold; + font-weight: 600; display: inline-block; - transition: background-color 0.3s, color 0.3s; + transition: all 0.3s ease; + box-shadow: 0 4px 12px rgba(0, 123, 255, 0.15); } .button:hover { background-color: #007BFF; color: white; + transform: translateY(-2px); + box-shadow: 0 6px 20px rgba(0, 123, 255, 0.3); } .card-author, @@ -108,19 +137,27 @@ .scroll-down-icon { position: absolute; - bottom: 20px; + bottom: 30px; height: 24px; color: #007BFF; animation: bounce 1.5s infinite; + cursor: pointer; + transition: all 0.3s ease; + } + + .scroll-down-icon:hover { + transform: scale(1.1); + opacity: 1 !important; } + .scroll-down-icon::before, .scroll-down-icon::after { content: ''; position: absolute; top: 50%; left: 50%; - width: 12px; - height: 12px; + width: 14px; + height: 14px; opacity: .8; border-left: 3px solid #007BFF; border-bottom: 3px solid #007BFF; @@ -148,7 +185,9 @@ } [data-theme="dark"] .container { - background: rgba(30, 30, 30, 0.8); + background: rgba(30, 30, 30, 0.9); + backdrop-filter: blur(20px); + -webkit-backdrop-filter: blur(20px); } [data-theme="dark"] p { @@ -166,8 +205,14 @@ color: white; } -[data-theme="dark"] .scroll-down-icon { - color: #007BFF; +[data-theme="dark"] .scroll-down-icon::before, +[data-theme="dark"] .scroll-down-icon::after { + border-left-color: #007BFF; + border-bottom-color: #007BFF; +} + +[data-theme="dark"] .avatar { + box-shadow: 0 8px 24px rgba(0, 123, 255, 0.3); } @@ -213,8 +258,8 @@ button.textContent = newTheme === 'light' ? '暗色模式' : '亮色模式'; }); - - + + \ No newline at end of file From 758b9c91fe3319e970c543acb7f41588596b410b Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Thu, 16 Apr 2026 22:24:50 +0800 Subject: [PATCH 12/30] =?UTF-8?q?=F0=9F=98=81=E4=BF=AE=E5=A4=8D=E8=AE=A2?= =?UTF-8?q?=E9=98=85=E9=93=BE=E6=8E=A5=E4=B8=AD=E7=9A=84=E7=9B=B8=E5=AF=B9?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E6=96=87=E7=AB=A0=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- friend_circle_lite/utils/url.py | 34 ++++++++++++++++++++++++--------- main/fclite.css | 2 +- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/friend_circle_lite/utils/url.py b/friend_circle_lite/utils/url.py index fa9791791da..3674055be44 100644 --- a/friend_circle_lite/utils/url.py +++ b/friend_circle_lite/utils/url.py @@ -4,24 +4,40 @@ def replace_non_domain(link: str, blog_url: str) -> str: """ - 暂未实现 - 检测并替换字符串中的非正常域名部分（如 IP 地址或 localhost），替换为 blog_url。 - 替换后强制使用 https，且考虑 blog_url 尾部是否有斜杠。 + 检测并处理相对地址、非正常域名（如 IP 地址或 localhost）。 + - 如果是相对地址（无协议和域名），自动拼接 blog_url + - 如果是 localhost 或 IP 地址，替换为 blog_url + - 如果是正常的绝对地址，直接返回 :param link: 原始地址字符串 - :param blog_url: 替换为的博客地址 - :return: 替换后的地址字符串 + :param blog_url: 博客的基础地址 + :return: 处理后的完整地址字符串 """ + if not link: + return link + try: parsed = urlparse(link) - if 'localhost' in parsed.netloc or re.match(r'^\d{1,3}(\.\d{1,3}){3}$', parsed.netloc): # IP地址或localhost - # 提取 path + query + + # 情况1: 相对地址（没有 scheme 和 netloc） + # 例如: "/post/article.html" 或 "post/article.html" + if not parsed.scheme and not parsed.netloc: + # 使用 urljoin 来正确处理相对路径 + return urljoin(blog_url, link) + + # 情况2: localhost 或 IP 地址 + if 'localhost' in parsed.netloc or re.match(r'^\d{1,3}(\.\d{1,3}){3}$', parsed.netloc): + # 提取 path + query + fragment path = parsed.path or '/' if parsed.query: path += '?' + parsed.query + if parsed.fragment: + path += '#' + parsed.fragment return urljoin(blog_url.rstrip('/') + '/', path.lstrip('/')) - else: - return link # 合法域名则返回原链接 + + # 情况3: 正常的绝对地址，直接返回 + return link + except Exception as e: logging.warning(f"替换链接时出错：{link}, error: {e}") return link diff --git a/main/fclite.css b/main/fclite.css index 84ae6454b89..a1efaed9b6a 100644 --- a/main/fclite.css +++ b/main/fclite.css @@ -503,7 +503,7 @@ .card-date { font-size: 12px; color: var(--author-color); - padding: 6px 10px; + padding: 4px 10px 4px 2px; line-height: 15px; } From 70674fb5f3b8ed4a078f5d5c822ecf61f27ef70e Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Thu, 16 Apr 2026 22:48:52 +0800 Subject: [PATCH 13/30] =?UTF-8?q?=F0=9F=98=98=E6=B7=BB=E5=8A=A0=E5=85=B6?= =?UTF-8?q?=E4=BB=96=E4=B8=8D=E5=B8=B8=E8=A7=81=E4=BD=86=E5=B8=B8=E7=94=A8?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- friend_circle_lite/feed_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/friend_circle_lite/feed_service.py b/friend_circle_lite/feed_service.py index a9c92e79718..159606c289d 100644 --- a/friend_circle_lite/feed_service.py +++ b/friend_circle_lite/feed_service.py @@ -30,6 +30,7 @@ class FeedDiscoveryService: ("feed3", "/feed/"), ("feed4", "/feed.php"), ("index", "/index.xml"), + ("atom2", "/feed.atom"), ] def __init__(self, session: requests.Session): From 7d414eef03739c42e7b027ad0380bb9aad3feceb Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Thu, 16 Apr 2026 22:50:32 +0800 Subject: [PATCH 14/30] =?UTF-8?q?=F0=9F=98=98=E6=B7=BB=E5=8A=A0=E5=85=B6?= =?UTF-8?q?=E4=BB=96=E4=B8=8D=E5=B8=B8=E8=A7=81=E4=BD=86=E5=B8=B8=E7=94=A8?= =?UTF-8?q?=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- friend_circle_lite/feed_service.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/friend_circle_lite/feed_service.py b/friend_circle_lite/feed_service.py index 159606c289d..7714e7682c1 100644 --- a/friend_circle_lite/feed_service.py +++ b/friend_circle_lite/feed_service.py @@ -21,16 +21,17 @@ class FeedDiscoveryService: """Discover an RSS or Atom endpoint for a website.""" POSSIBLE_FEEDS = [ - ("atom", "/atom.xml"), - ("rss", "/rss.xml"), - ("rss2", "/rss2.xml"), - ("rss3", "/rss.php"), - ("feed", "/feed"), - ("feed2", "/feed.xml"), - ("feed3", "/feed/"), - ("feed4", "/feed.php"), - ("index", "/index.xml"), - ("atom2", "/feed.atom"), + ("rss1", "/feed"), # WordPress / 最常见 + ("rss2", "/feed/"), # WordPress 兼容写法 + ("rss3", "/rss.xml"), # 很多传统站点 + ("rss4", "/atom.xml"), # 静态博客常见（Hugo / Jekyll） + ("rss5", "/feed.xml"), # 通用型 + ("rss6", "/index.xml"), # Hugo / 一些静态站 + ("rss7", "/feed.atom"), # Atom 明确路径 + ("rss8", "/rss2.xml"), # 老系统遗留 + ("rss9", "/rss/feed.xml"),# 少见但存在 + ("rss10", "/rss.php"), # 老 PHP 程序 + ("rss11", "/feed.php"), # 同上 ] def __init__(self, session: requests.Session): From a67ea0214e55b17783dced8a5d1f8838c7f82d62 Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Sun, 24 May 2026 23:56:41 +0800 Subject: [PATCH 15/30] =?UTF-8?q?=F0=9F=98=98=E5=AE=8C=E5=85=A8=E5=88=A0?= =?UTF-8?q?=E9=99=A4=E5=8E=9F=E6=9C=89=E7=9A=84FastAPI=E9=83=A8=E7=BD=B2?= =?UTF-8?q?=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deploy.sh | 60 +++------------------- readme.md | 50 +++++++++--------- server.py | 92 ---------------------------------- server/requirements-server.txt | 2 - 4 files changed, 33 insertions(+), 171 deletions(-) delete mode 100644 server.py delete mode 100644 server/requirements-server.txt diff --git a/deploy.sh b/deploy.sh index ac0aa082748..9276130eec4 100644 --- a/deploy.sh +++ b/deploy.sh @@ -1,62 +1,16 @@ #!/bin/bash -# 获取当前脚本所在目录 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -# 定义日志文件路径 -# CRON_LOG_FILE="$SCRIPT_DIR/cron_grab.log" -# API_LOG_FILE="$SCRIPT_DIR/api_grab.log" +cd "$SCRIPT_DIR" || exit 1 -# # 定义要执行的命令 -# COMMAND="python3 $SCRIPT_DIR/run.py" +python3 run.py -# # 定义定时任务的执行间隔（例如每四小时一次） -# INTERVAL="4" - -# 添加定时任务到 crontab -# (crontab -l 2>/dev/null; echo "0 */$INTERVAL * * * $COMMAND >> $CRON_LOG_FILE 2>&1 && echo '运行成功'") | crontab - - -# echo "====================================" -# echo "定时爬取 成功设置，时间间隔：4h" -# echo "定时任务日志：$CRON_LOG_FILE" -# echo "====================================" - - - -#!/bin/bash - -# 获取当前脚本所在目录 -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" - -# 定义 API 服务的启动命令 -API_COMMAND="python3 $SCRIPT_DIR/server.py" +mkdir -p pages +cp -r main static all.json errors.json pages/ echo "====================================" - -# 后台运行服务端，将数据映射到API -echo "****正在启动API服务****" -nohup $API_COMMAND &>/dev/null & -API_PID=$! -sleep 5 # 等待API服务启动，可能需要调整等待时间 - -echo "API 服务已启动：http://localhost:1223" -echo "API 服务进程号：$API_PID" -echo "API 服务关闭命令：kill -9 $API_PID" -echo "文档地址：https://blog.liushen.fun/posts/4dc716ec/" -echo "====================================" - -# 用户选择是否执行爬取 -read -p "选择操作：0 - 退出, 1 - 执行一次爬取: " USER_CHOICE - -if [ "$USER_CHOICE" -eq 1 ]; then - echo "****正在执行一次爬取****" - python3 $SCRIPT_DIR/run.py - echo "****爬取成功****" -else - echo "退出选项被选择，掰掰！" - +echo "静态文件已生成到 pages/ 目录" +echo "请将 pages/ 目录作为静态网站根目录部署" +echo "部署后检查 /all.json 是否可访问" echo "====================================" -echo "定时抓取的部分请自行设置，如果有宝塔等面板可以按照说明直接添加，如果没有宝塔可以查看本脚本上面屏蔽的部分，自行添加到 crontab 中" -echo "====================================" - -fi diff --git a/readme.md b/readme.md index 80cfcc3012b..0b5b41c5c4b 100644 --- a/readme.md +++ b/readme.md @@ -11,6 +11,10 @@ ## 开发进度 +### 2026-05-24 + +* 移除原先基于 FastAPI 的简陋后端部署方式，后续自部署统一采用生成静态文件后作为纯静态网站托管的纯净态方式。 + ### 2025-07-23 * 添加缓存文件，防止由于缓存导致多次请求 @@ -106,11 +110,11 @@ - **爬取文章**: 爬取所有友链的文章，结果放置在根目录的all.json文件中，方便读取并部署到前端。 - **邮箱推送更新(对作者推送所有友链更新)**: 作者可以通过邮箱订阅所有rss的更新（未来开发）。 - **issue邮箱订阅(对访客实时推送最新文章邮件)**: 基于`GitHub issue`的博客更新邮件订阅功能，游客可以通过简单的提交`issue`进行邮箱订阅站点更新，删除对应`issue`即可取消订阅。 -- **文件分离**: 将前后端分离，前端文件放在page分支，后端文件放在主分支 +- **文件分离**: 将生成任务和静态展示分离，前端文件与生成后的 `all.json` 可直接作为静态网站托管。 ## 特点介绍 -* **轻量化**：对比原版友链朋友圈的功能，该友圈功能简洁，去掉了设置和fastAPI的臃肿，仅保留关键内容。 +* **轻量化**：对比原版友链朋友圈的功能，该友圈功能简洁，去掉了设置和 FastAPI 的臃肿，仅保留关键内容。 * **无数据库**：因为内容较少，我采用`json`直接存储文章信息，减少数据库操作，提升`action`运行效率。 * **部署简单**：原版友链朋友圈由于功能多，导致部署较为麻烦，本方案仅需简单的部署action即可使用，vercel仅用于部署前端静态页面和实时获取最新内容。 * **文件占用**：对比原版`4MB`的`bundle.js`文件大小，本项目仅需要`5.50KB`的`fclite.min.js`文件即可轻量的展示到前端。 @@ -366,15 +370,13 @@ ## 自部署使用方法 +自部署后续统一采用纯静态方式：本项目只负责定时生成 `all.json`、`errors.json` 等数据文件，生成完成后把 `static`、`main` 和数据文件作为静态网站托管即可，不再启动 FastAPI 后端服务。 + 如果你有一台境内服务器，你也可以通过以下操作将其部署到你的服务器上，操作如下： ### 前置工作 -确保你的服务器有定时任务 `crontab` 功能包，一般是linux自带，如果你没有宝塔等可以管理定时任务的面板工具，可能需要你自行了解定时工具并导入，本教程提供了简单的介绍。 - -### 宿主机环境 - -> 适用于宝塔面板等宿主机直接有Python环境的场景下 +确保你的服务器有 Python 运行环境，以及定时任务 `crontab`、宝塔、1Panel 等任意一种可定时执行命令的工具。 首先克隆仓库并进入对应路径： @@ -383,40 +385,40 @@ git clone https://github.com/willow-god/Friend-Circle-Lite.git cd Friend-Circle-Lite ``` -由于不存在issue，所以不支持邮箱推送(主要是懒得分类写了，要不然还得从secret中获取密码的功能剥离QAQ)，请将除第一部分抓取以外的功能均设置为false +由于不存在 issue，所以不支持邮箱推送(主要是懒得分类写了，要不然还得从secret中获取密码的功能剥离QAQ)，请将除第一部分抓取以外的功能均设置为false。 -下载服务相关包，其中 `requirements-server.txt` 是部署API服务所用包， `requirements.txt` 是抓取服务所用包，请均下载一遍。 +安装抓取服务所需依赖： ```bash pip install -r ./requirements.txt -pip install -r ./server/requirements-server.txt ``` -#### 部署API服务 +### 生成静态文件 -如果环境配置完毕，你可以进入目录路径后直接运行`deploy.sh`脚本启动API服务： +执行一次抓取命令： ```bash -chmod +x ./deploy.sh -./deploy.sh +python run.py ``` -其中的注释应该是较为详细的，如果部署成功你可以使用以下命令进行测试，如果获取到了首页html内容则成功： +执行完成后，根目录会生成或更新 `all.json`、`errors.json` 等数据文件。将以下内容放到你的静态网站目录即可： -```bash -curl 127.0.0.1:1223 -``` +- `static/` 目录中的静态页面和资源 +- `main/` 目录中的前端样式与脚本 +- `all.json`、`errors.json` 数据文件 -这个端口号可以修改，在server.py最后一行修改数字即可，如果你想删除该API服务，可以使用ps找到对应进程并使用Kill命令杀死进程： +如果希望在宿主机上直接整理出可发布目录，也可以运行： ```bash -ps aux | grep python -kill -9 [这里填写上面查询结果中对应的进程号] +chmod +x ./deploy.sh +./deploy.sh ``` -### Docker环境 +脚本会执行 `python3 run.py`，并将 `main`、`static`、`all.json`、`errors.json` 复制到 `pages/` 目录。将 `pages/` 目录作为静态网站根目录部署即可。部署完成后，检查 `/all.json` 是否有数据，如果有，则部署成功。 + +### 1Panel / Docker 环境 -> 目前Docker部署方式和1Panel强相关，如果有其他需要docker环境部署的，可以参考，但是可能需要自行摸索 +> 目前 Docker 部署方式和 1Panel 强相关；这里的思路是先在面板里跑一次生成命令，把生成后的静态文件放到网站目录，再按静态网站托管。 由于主包也用上了1Panel，所以捣鼓了一下，得益于1Panel可以方便的创建运行环境，所以可以基本做到无占用的API，因为除了执行action的时候，其他时间完全是纯静态的。 @@ -459,7 +461,7 @@ merge_result: > 宿主机环境只需要直接执行python ./run.py即可执行抓取 -由于原生的crontab可能较为复杂，如果有兴趣可以查看./deploy.sh文件中，屏蔽掉的部分，这里我不会细讲，这里我主要讲解宝塔面板添加定时任务，这样可以最大程度减少内存占用，其他面板服务类似： +由于原生的 crontab 可能较为复杂，这里主要讲解宝塔面板添加定时任务，这样可以最大程度减少内存占用，其他面板服务类似： ![](./static/baota.png) diff --git a/server.py b/server.py deleted file mode 100644 index 5616ba1fe55..00000000000 --- a/server.py +++ /dev/null @@ -1,92 +0,0 @@ -from fastapi import FastAPI -from fastapi.staticfiles import StaticFiles -from fastapi.responses import FileResponse, HTMLResponse, JSONResponse -from starlette.middleware.cors import CORSMiddleware -import json -import random - -app = FastAPI() - -# 设置静态文件目录 -app.mount("/static", StaticFiles(directory="static"), name="static") -app.mount("/main", StaticFiles(directory="main"), name="main") - -# 添加 CORS 中间件 -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) -# 返回图标图片 -@app.get("/favicon.ico", response_class=HTMLResponse) -async def favicon(): - return FileResponse('static/favicon.ico') - -# 返回背景图片 -@app.get("/bg-light.webp", response_class=HTMLResponse) -async def bg_light(): - return FileResponse('static/bg-light.webp') - -# 返回背景图片 -@app.get("/bg-dark.webp", response_class=HTMLResponse) -async def bg_dark(): - return FileResponse('static/bg-dark.webp') - -# 返回资源文件 -# 返回 CSS 文件 -@app.get("/fclite.css", response_class=HTMLResponse) -async def get_fclite_css(): - return FileResponse('./main/fclite.css') - -# 返回 JS 文件 -@app.get("/fclite.js", response_class=HTMLResponse) -async def get_fclite_js(): - return FileResponse('./main/fclite.js') - -@app.get("/", response_class=HTMLResponse) -async def root(): - return FileResponse('./static/index.html') - -@app.get('/all.json') -async def get_all_articles(): - try: - with open('./all.json', 'r', encoding='utf-8') as f: - articles_data = json.load(f) - return JSONResponse(content=articles_data) - except FileNotFoundError: - return JSONResponse(content={"error": "File not found"}, status_code=404) - except json.JSONDecodeError: - return JSONResponse(content={"error": "Failed to decode JSON"}, status_code=500) - -@app.get('/errors.json') -async def get_error_friends(): - try: - with open('./errors.json', 'r', encoding='utf-8') as f: - errors_data = json.load(f) - return JSONResponse(content=errors_data) - except FileNotFoundError: - return JSONResponse(content={"error": "File not found"}, status_code=404) - except json.JSONDecodeError: - return JSONResponse(content={"error": "Failed to decode JSON"}, status_code=500) - -@app.get('/random') -async def get_random_article(): - try: - with open('./all.json', 'r', encoding='utf-8') as f: - articles_data = json.load(f) - if articles_data.get("article_data"): - random_article = random.choice(articles_data["article_data"]) - return JSONResponse(content=random_article) - else: - return JSONResponse(content={"error": "No articles available"}, status_code=404) - except FileNotFoundError: - return JSONResponse(content={"error": "File not found"}, status_code=404) - except json.JSONDecodeError: - return JSONResponse(content={"error": "Failed to decode JSON"}, status_code=500) - -if __name__ == '__main__': - # 启动FastAPI应用 - import uvicorn - uvicorn.run(app, host='0.0.0.0', port=1223) diff --git a/server/requirements-server.txt b/server/requirements-server.txt deleted file mode 100644 index f0615cfd093..00000000000 --- a/server/requirements-server.txt +++ /dev/null @@ -1,2 +0,0 @@ -fastapi -uvicorn \ No newline at end of file From 07a4001921b13a0ae2ffd81c74025bf8ef7daedb Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Sun, 31 May 2026 20:38:48 +0800 Subject: [PATCH 16/30] =?UTF-8?q?=F0=9F=98=98=E8=9E=8D=E5=90=88=E5=8F=8B?= =?UTF-8?q?=E9=93=BE=E6=A3=80=E6=9F=A5=E9=A1=B9=E7=9B=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/friend_circle_lite.yml | 2 +- CHANGELOG.md | 335 ++++++++ conf.yaml | 38 +- deploy.sh | 4 +- friend_circle_lite/all_friends.py | 149 +++- friend_circle_lite/app_config.py | 57 +- friend_circle_lite/application.py | 37 +- friend_circle_lite/cache_store.py | 180 ++++- friend_circle_lite/crawler_service.py | 102 ++- friend_circle_lite/link_check_service.py | 283 +++++++ friend_circle_lite/models.py | 111 ++- main/fclite.js | 11 +- readme.md | 50 +- static/index.html | 984 +++++++++++++++++------ 14 files changed, 2047 insertions(+), 296 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 friend_circle_lite/link_check_service.py diff --git a/.github/workflows/friend_circle_lite.yml b/.github/workflows/friend_circle_lite.yml index f08ffe73190..3096cd41dca 100644 --- a/.github/workflows/friend_circle_lite.yml +++ b/.github/workflows/friend_circle_lite.yml @@ -76,7 +76,7 @@ jobs: - name: Build static publish directory run: | mkdir pages - cp -r main ./static/edgeone.json ./static/_headers ./static/index.html ./static/readme.md ./static/favicon.ico ./static/bg-light.webp ./static/bg-dark.webp all.json errors.json pages/ + cp -r main ./static/edgeone.json ./static/_headers ./static/index.html ./static/readme.md ./static/favicon.ico ./static/bg-light.webp ./static/bg-dark.webp all.json link.json errors.json pages/ - name: Publish static assets to branches run: | diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000000..dc9c529f6ff --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,335 @@ +# Changelog + +## [2.0.0] - 2026-05-31 + +### 🎉 重大更新：友链可达性检测与数据结构重构 + +本次更新合并了 [check-flink](https://github.com/willow-god/check-flink) 项目的友链可达性检测功能，并对数据结构进行了重大重构，实现了友圈文章与友链状态的分离。 + +--- + +## 📋 目录 + +- [核心功能变更](#核心功能变更) +- [数据结构变更](#数据结构变更) +- [配置文件变更](#配置文件变更) +- [前端页面变更](#前端页面变更) +- [迁移指南](#迁移指南) +- [兼容性说明](#兼容性说明) + +--- + +## 🚀 核心功能变更 + +### 新增功能 + +#### 1. 友链可达性检测 +- **直连检测**：直接访问友链站点，验证可达性 +- **代理检测**：通过代理服务（如 Cloudflare Worker）检测被墙站点 +- **API 检测**：通过第三方 API 兜底检测（如 xxapi.cn） +- **反链检测**：检测友链页面是否包含你的站点链接 +- **智能缓存**：检测结果缓存 24 小时，避免频繁请求 +- **失败计数**：记录连续失败次数，便于识别长期失效友链 + +#### 2. 数据合并功能重构 +- **独立配置**：`merge_settings` 从 `spider_settings` 中独立出来 +- **并列控制**：友圈文章和友链可达性数据分别控制是否合并 +- **智能合并**： + - 可达性优先级：direct > proxy > api > none + - 延迟取最优：选择响应时间更短的结果 + - 反链取并集：任一环境检测到反链即为有反链 + - 失败次数取最小：选择失败次数较少的结果 + +#### 3. 前端展示页面 +- **新首页**：继承 check-flink 风格，同时展示友链状态和友圈文章 +- **友链状态卡片**：显示可达性、响应时间、失败次数、反链状态 +- **状态过滤**：支持按全部/失效/可抓取/仅API/较慢筛选 +- **搜索功能**：同时搜索友链、文章、作者 + +--- + +## 📊 数据结构变更 + +### 旧结构（v1.x） + +```json +// all.json（混合数据） +{ + "statistical_data": { + "friends_num": 199, + "active_num": 155, + "article_num": 250, + "error_num": 15 + }, + "article_data": [...], + "link_check_data": [...], // ❌ 已移除 + "friend_data": [...] // ❌ 已移除 +} + +// errors.json +[ + ["站点名", "站点地址", "头像地址"], + ... +] +``` + +### 新结构（v2.0） + +```json +// all.json（仅友圈文章） +{ + "statistical_data": { + "friends_num": 199, + "active_num": 155, + "article_num": 250, + "error_num": 15 + }, + "article_data": [...] +} + +// link.json（友链可达性）✨ 新增 +{ + "statistical_data": { + "link_total_num": 199, + "link_reachable_num": 187, + "link_unreachable_num": 12, + "crawl_allowed_num": 167, + "api_only_num": 20, + "has_author_link_num": 131, + "link_last_checked_time": "2026-05-31 19:33:00" + }, + "link_data": [ + { + "name": "清羽飞扬", + "link": "https://blog.liushen.fun/", + "link_page": "https://blog.liushen.fun/link/", + "avatar": "https://blog.liushen.fun/favicon.ico", + "reachable": true, + "crawlable": true, + "method": "direct", + "latency": 0.07, + "fail_count": 0, + "checked_at": "2026-05-31 19:33:00", + "has_backlink": true, + "reason": "allowed_by_direct" + }, + ... + ] +} + +// errors.json（真正不可达的友链） +[ + ["站点名", "站点地址", "头像地址"], + ... +] +``` + +### 字段说明 + +#### link.json 中的 link_data 字段 + +| 字段 | 类型 | 说明 | +|------|------|------| +| `name` | string | 友链名称 | +| `link` | string | 友链地址 | +| `link_page` | string | 友链页地址（用于反链检测） | +| `avatar` | string | 头像地址 | +| `reachable` | boolean | 是否可达 | +| `crawlable` | boolean | 是否可抓取（直连或代理可达） | +| `method` | string | 最佳检测方式：`direct`/`proxy`/`api`/`disabled` | +| `latency` | number | 响应延迟（秒），-1 表示不可达 | +| `fail_count` | number | 连续失败次数 | +| `checked_at` | string | 检测时间 | +| `has_backlink` | boolean\|null | 是否有反链，null 表示未检测 | +| `reason` | string | RSS 抓取决策原因 | + +--- + +## ⚙️ 配置文件变更 + +### 旧配置（v1.x） + +```yaml +spider_settings: + enable: true + json_url: "https://blog.liushen.fun/friend.json" + article_count: 5 + merge_result: # ❌ 已废弃 + enable: false + merge_json_url: "https://fc.liushen.fun" +``` + +### 新配置（v2.0） + +```yaml +spider_settings: + enable: true + json_url: "https://blog.liushen.fun/friend.json" + article_count: 5 + +# ✨ 新增：友链可达性检测配置 +link_check: + enable: true # 是否启用友链可达性检测 + max_age_hours: 24 # 缓存时间（小时） + timeout: 15 # 请求超时时间（秒） + max_workers: 10 # 并发检测数量 + proxy_url: "" # 代理前缀（如 Cloudflare Worker） + status_api_url: "https://v2.xxapi.cn/api/status?url={url}" # 兜底 API + enable_backlink_check: true # 是否检测反链 + author_url: "blog.liushen.fun" # 你的站点域名 + +# ✨ 新增：数据合并配置（独立出来） +merge_settings: + enable: false # 是否启用数据合并 + remote_base_url: "https://fc.liushen.fun" # 远程数据源基础 URL + merge_article_data: true # 是否合并友圈文章数据 + merge_link_check_data: true # 是否合并友链可达性数据 +``` + +### 配置变更说明 + +#### 1. `merge_result` → `merge_settings` +- **位置变更**：从 `spider_settings.merge_result` 移至顶级 `merge_settings` +- **字段重命名**： + - `merge_json_url` → `remote_base_url` + - 新增 `merge_article_data`（友圈文章合并开关） + - 新增 `merge_link_check_data`（友链可达性合并开关） + +#### 2. 友链数据格式 +- **旧格式（3字段）**：`["站点名", "站点地址", "头像地址"]` +- **新格式（4字段）**：`["站点名", "站点地址", "友链页地址", "头像地址"]` +- **兼容性**：两种格式均支持，自动识别 + +--- + +## 🎨 前端页面变更 + +### 文件变更 + +| 文件 | 状态 | 说明 | +|------|------|------| +| `static/index.html` | ✏️ 重写 | 继承 check-flink 风格，展示友链状态+友圈文章 | +| `static/status.html` | ❌ 删除 | 功能已合并到 index.html | +| `main/fclite.js` | ✅ 保持 | 不变，供外部引用 | +| `main/fclite.css` | ✅ 保持 | 不变，供外部引用 | + +### 新首页特性 + +- **诗词背景**：李白《登金陵凤凰台》 +- **统计卡片**：友链总数、可抓取、文章总数、失效友链 +- **友链状态展示**： + - 状态点动画（绿色=正常，黄色=较慢，蓝色=API，红色=失效） + - 失败次数显示 + - 反链图标（❤️ 有反链 / 💔 无反链） +- **友圈文章展示**：标题固定两行高度，作者+时间 +- **搜索过滤**：同时搜索友链、文章、作者 + +--- + +## 🔄 迁移指南 + +### 对于使用者 + +#### 1. 更新配置文件 + +如果你之前启用了 `merge_result`，需要调整配置： + +```yaml +# 旧配置 +spider_settings: + merge_result: + enable: true + merge_json_url: "https://example.com" + +# 新配置 +merge_settings: + enable: true + remote_base_url: "https://example.com" + merge_article_data: true + merge_link_check_data: true +``` + +#### 2. 更新友链数据格式（可选） + +如果想启用反链检测，建议更新为 4 字段格式： + +```json +{ + "friends": [ + ["清羽飞扬", "https://blog.liushen.fun/", "https://blog.liushen.fun/link/", "https://blog.liushen.fun/favicon.ico"] + ] +} +``` + +#### 3. 更新 GitHub Actions + +如果自定义了 workflow，需要添加 `link.json` 到发布文件： + +```yaml +- name: Build static publish directory + run: | + mkdir pages + cp -r main static all.json link.json errors.json pages/ +``` + +### 对于外部引用者 + +#### 如果你引用了 `all.json` + +✅ **无需修改**，`all.json` 结构保持兼容，仅移除了 `link_check_data` 和 `friend_data` 字段。 + +#### 如果你引用了 `main/fclite.js` 和 `main/fclite.css` + +✅ **无需修改**，这两个文件保持不变。 + +#### 如果你需要友链可达性数据 + +✨ **新增引用** `link.json`： + +```javascript +fetch('/link.json') + .then(response => response.json()) + .then(data => { + const links = data.link_data; + const stats = data.statistical_data; + // 使用友链数据 + }); +``` + +--- + +## ✅ 兼容性说明 + +### 向后兼容 + +- ✅ `all.json` 结构保持兼容（仅移除冗余字段） +- ✅ `errors.json` 结构不变 +- ✅ `main/fclite.js` 和 `main/fclite.css` 不变 +- ✅ 友链数据支持 3 字段和 4 字段格式 +- ✅ 旧版引用者无需修改代码 + +### 不兼容变更 + +- ❌ `all.json` 不再包含 `link_check_data` 和 `friend_data` +- ❌ `spider_settings.merge_result` 配置已废弃 +- ❌ `static/status.html` 已删除 + +### 升级建议 + +1. **最小升级**：仅更新代码，不修改配置 → 友链检测默认启用，但反链检测默认关闭 +2. **推荐升级**：更新配置 + 更新友链数据为 4 字段格式 → 完整体验所有新功能 +3. **完整升级**：更新配置 + 更新友链数据 + 启用数据合并 → 国内外数据智能合并 + +--- + +## 📝 相关链接 + +- [Friend-Circle-Lite 主仓库](https://github.com/willow-god/Friend-Circle-Lite) +- [check-flink 原项目](https://github.com/willow-god/check-flink) +- [详细文档](https://blog.liushen.fun/posts/4dc716ec/) + +--- + +## 🙏 致谢 + +感谢所有使用 Friend-Circle-Lite 的朋友们，以及为项目提供反馈和建议的贡献者！ diff --git a/conf.yaml b/conf.yaml index b5a406e1768..b93d4db9a2f 100644 --- a/conf.yaml +++ b/conf.yaml @@ -3,16 +3,42 @@ # enable: 是否启用爬虫 # json_url: 请填写对应格式json的地址，仅支持网络地址 # article_count: 请填写每个博客需要获取的最大文章数量 -# marge_result: 是否合并多个json文件，若为true则会合并指定网络地址和本地地址的json文件 -# enable: 是否启用合并功能，该功能提供与自部署的友链合并功能，可以解决服务器部分国外网站无法访问的问题 -# marge_json_path: 请填写网络地址的json文件，用于合并，不带空格！！！ spider_settings: enable: true json_url: "https://blog.liushen.fun/friend.json" article_count: 5 - merge_result: - enable: false - merge_json_url: "https://fc.liushen.fun" + +# 数据合并配置 +# 解释：合并多个数据源的结果，比如国内和国外执行结果，解决部分网站访问受限问题 +# enable: 是否启用数据合并功能 +# remote_base_url: 远程数据源的基础 URL，会自动拼接 /all.json、/link.json、/errors.json +# merge_article_data: 是否合并友圈文章数据（all.json） +# merge_link_check_data: 是否合并友链可达性数据（link.json），如果远程没有此文件会自动跳过 +merge_settings: + enable: false + remote_base_url: "https://fc.liushen.fun" + merge_article_data: true + merge_link_check_data: true + +# 友链可达性检测配置 +# 解释：先检查友链站点是否可达，再决定是否继续抓取该站点 RSS，检测结果会写入 all.json 供前端展示 +# enable: 是否启用友链可达性检测 +# max_age_hours: 同一友链检测结果缓存时间，默认 24 小时 +# timeout: 单次请求超时时间 +# max_workers: 并发检测数量 +# proxy_url: 可选代理前缀，比如 Cloudflare Worker 转发地址，留空则不启用 +# status_api_url: 兜底状态码 API，API-only 结果只用于可达性展示，不参与 RSS 抓取 +# enable_backlink_check: 是否检测友链页是否包含你的站点链接 +# author_url: 你的站点域名，用于反链检测，建议只填写域名 +link_check: + enable: true + max_age_hours: 24 + timeout: 15 + max_workers: 10 + proxy_url: "" + status_api_url: "https://v2.xxapi.cn/api/status?url={url}" + enable_backlink_check: true + author_url: "blog.liushen.fun" # 邮箱推送功能配置，暂未实现，等待后续开发 # 解释：每天为指定邮箱推送所有友链文章的更新，仅能指定一个 diff --git a/deploy.sh b/deploy.sh index 9276130eec4..6d3428e1da1 100644 --- a/deploy.sh +++ b/deploy.sh @@ -7,10 +7,10 @@ cd "$SCRIPT_DIR" || exit 1 python3 run.py mkdir -p pages -cp -r main static all.json errors.json pages/ +cp -r main static all.json link.json errors.json pages/ echo "====================================" echo "静态文件已生成到 pages/ 目录" echo "请将 pages/ 目录作为静态网站根目录部署" -echo "部署后检查 /all.json 是否可访问" +echo "部署后检查 /all.json 和 /link.json 是否可访问" echo "====================================" diff --git a/friend_circle_lite/all_friends.py b/friend_circle_lite/all_friends.py index e24d9cdb7b1..f55ca2b1f0a 100644 --- a/friend_circle_lite/all_friends.py +++ b/friend_circle_lite/all_friends.py @@ -15,13 +15,20 @@ sort_articles_by_time as _sort_articles_by_time, ) -def fetch_and_process_data(json_url: str, specific_RSS: list = None, count: int = 5, cache_file: str = None): +def fetch_and_process_data( + json_url: str, + specific_RSS: list = None, + count: int = 5, + cache_file: str = None, + link_check_config=None, +): """Legacy wrapper around the new crawler orchestration service.""" return FriendCircleCrawler( json_url=json_url, count=count, specific_rss=specific_RSS, cache_file=cache_file, + link_check_config=link_check_config, ).run() def sort_articles_by_time(data, future_tolerance_days=2): @@ -45,14 +52,148 @@ def marge_data_from_json_url(data, marge_json_url): except Exception as e: logging.error(f"无法获取链接：{marge_json_url}，出现的问题为：{e}", exc_info=True) return data - + if 'article_data' in marge_data: - logging.info(f"开始合并数据，原数据共有 {len(data['article_data'])} 篇文章，第三方数据共有 {len(marge_data['article_data'])} 篇文章") + logging.info(f"开始合并文章数据，原数据共有 {len(data['article_data'])} 篇文章，第三方数据共有 {len(marge_data['article_data'])} 篇文章") data['article_data'].extend(marge_data['article_data']) data['article_data'] = list({v['link']:v for v in data['article_data']}.values()) - logging.info(f"合并数据完成，现在共有 {len(data['article_data'])} 篇文章") + logging.info(f"合并文章数据完成，现在共有 {len(data['article_data'])} 篇文章") return data + +def merge_link_data_from_json_url(link_data, merge_json_url): + """ + 从另一个 link.json 文件中获取友链可达性数据并智能合并。 + + 合并策略： + - 可达性优先级：direct > proxy > api > none + - 延迟取最优（最小值） + - 反链取并集（任一为 true 则为 true） + - 失败次数取最小值 + + 参数： + link_data (dict): 本地友链数据，包含 statistical_data 和 link_data + merge_json_url (str): 远程 link.json 的 URL + + 返回： + dict: 合并后的友链数据 + """ + try: + response = requests.get(merge_json_url, headers=HEADERS_JSON, timeout=timeout) + remote_data = response.json() + except Exception as e: + logging.warning(f"无法获取友链数据：{merge_json_url}，跳过友链数据合并。错误：{e}") + return link_data + + if 'link_data' not in remote_data: + logging.warning(f"远程数据不包含 link_data 字段，跳过友链数据合并") + return link_data + + local_links = link_data.get('link_data', []) + remote_links = remote_data.get('link_data', []) + + logging.info(f"开始合并友链数据，本地 {len(local_links)} 条，远程 {len(remote_links)} 条") + + # 按 URL 建立索引 + link_map = {link['link']: link for link in local_links} + + for remote_link in remote_links: + url = remote_link['link'] + if url not in link_map: + # 新友链，直接添加 + link_map[url] = remote_link + else: + # 已存在，智能合并 + local_link = link_map[url] + link_map[url] = _merge_single_link(local_link, remote_link) + + merged_links = list(link_map.values()) + logging.info(f"合并友链数据完成，共有 {len(merged_links)} 条友链") + + # 重新计算统计数据 + merged_stats = _recalculate_link_statistics(merged_links) + + return { + 'statistical_data': merged_stats, + 'link_data': merged_links, + } + + +def _merge_single_link(local, remote): + """ + 合并单条友链数据，优先选择更好的检测结果。 + + 优先级： + 1. 可达性：direct > proxy > api > none + 2. 延迟：取最小值 + 3. 反链：任一为 true 则为 true + 4. 失败次数：取最小值 + """ + method_priority = {'direct': 4, 'proxy': 3, 'api': 2, 'disabled': 1, 'none': 0, '': 0} + + local_priority = method_priority.get(local.get('method', ''), 0) + remote_priority = method_priority.get(remote.get('method', ''), 0) + + # 选择优先级更高的作为基础 + if remote_priority > local_priority: + base = remote.copy() + alt = local + elif remote_priority < local_priority: + base = local.copy() + alt = remote + else: + # 优先级相同，选择延迟更低的 + local_latency = local.get('latency', 999) + remote_latency = remote.get('latency', 999) + if remote_latency >= 0 and (local_latency < 0 or remote_latency < local_latency): + base = remote.copy() + alt = local + else: + base = local.copy() + alt = remote + + # 反链取并集 + local_backlink = local.get('has_backlink') + remote_backlink = remote.get('has_backlink') + if local_backlink is True or remote_backlink is True: + base['has_backlink'] = True + elif local_backlink is False and remote_backlink is False: + base['has_backlink'] = False + # 否则保持 base 的值 + + # 失败次数取最小值 + local_fail = local.get('fail_count', 0) + remote_fail = remote.get('fail_count', 0) + base['fail_count'] = min(local_fail, remote_fail) + + # 检测时间取最新 + local_checked = local.get('checked_at', '') + remote_checked = remote.get('checked_at', '') + if remote_checked > local_checked: + base['checked_at'] = remote_checked + + return base + + +def _recalculate_link_statistics(links): + """重新计算合并后的友链统计数据。""" + reachable = [link for link in links if link.get('reachable')] + crawl_allowed = [link for link in links if link.get('crawlable')] + api_only = [link for link in links if link.get('method') == 'api'] + has_backlink = [link for link in links if link.get('has_backlink') is True] + checked_times = [link.get('checked_at', '') for link in links if link.get('checked_at')] + + return { + 'link_total_num': len(links), + 'link_reachable_num': len(reachable), + 'link_unreachable_num': len(links) - len(reachable), + 'crawl_allowed_num': len(crawl_allowed), + 'api_only_num': len(api_only), + 'has_author_link_num': len(has_backlink), + 'link_last_checked_time': max(checked_times) if checked_times else '', + } + + def marge_errors_from_json_url(errors, marge_json_url): """ 从另一个网络 JSON 文件中获取错误信息并遍历，删除在errors中， diff --git a/friend_circle_lite/app_config.py b/friend_circle_lite/app_config.py index 9068cf55ee5..34300ea919d 100644 --- a/friend_circle_lite/app_config.py +++ b/friend_circle_lite/app_config.py @@ -16,14 +16,17 @@ DEFAULT_CACHE_FILE = "./temp/cache.sqlite3" DEFAULT_ALL_JSON = "./all.json" DEFAULT_ERRORS_JSON = "./errors.json" +DEFAULT_LINK_JSON = "./link.json" @dataclass(slots=True) -class MergeResultConfig: - """Options for merging local crawl results with a remote Friend-Circle feed.""" +class MergeSettings: + """Options for merging local crawl results with remote data sources.""" enable: bool = False - merge_json_url: str = "" + remote_base_url: str = "" + merge_article_data: bool = True + merge_link_check_data: bool = True @dataclass(slots=True) @@ -33,7 +36,20 @@ class SpiderSettings: enable: bool = True json_url: str = "" article_count: int = 5 - merge_result: MergeResultConfig = field(default_factory=MergeResultConfig) + + +@dataclass(slots=True) +class LinkCheckConfig: + """Settings for friend link reachability checks.""" + + enable: bool = True + max_age_hours: int = 24 + timeout: int = 15 + max_workers: int = 10 + proxy_url: str = "" + status_api_url: str = "https://v2.xxapi.cn/api/status?url={url}" + enable_backlink_check: bool = False + author_url: str = "" @dataclass(slots=True) @@ -82,6 +98,7 @@ class RuntimePaths: cache_file: str = DEFAULT_CACHE_FILE all_json_file: str = DEFAULT_ALL_JSON errors_json_file: str = DEFAULT_ERRORS_JSON + link_json_file: str = DEFAULT_LINK_JSON @dataclass(slots=True) @@ -89,6 +106,8 @@ class ApplicationConfig: """Root application configuration assembled from the YAML file.""" spider_settings: SpiderSettings + merge_settings: MergeSettings + link_check: LinkCheckConfig email_push: EmailPushConfig rss_subscribe: RssSubscribeConfig smtp: SmtpConfig @@ -100,21 +119,35 @@ class ApplicationConfig: def from_dict(cls, data: dict) -> "ApplicationConfig": """Create a typed config object from the raw YAML dictionary.""" spider_raw = data.get("spider_settings", {}) - merge_raw = spider_raw.get("merge_result", {}) + merge_raw = data.get("merge_settings", {}) + link_check_raw = data.get("link_check", {}) email_push_raw = data.get("email_push", {}) rss_subscribe_raw = data.get("rss_subscribe", {}) website_info_raw = rss_subscribe_raw.get("website_info", {}) smtp_raw = data.get("smtp", {}) + runtime_raw = data.get("runtime_paths", {}) return cls( spider_settings=SpiderSettings( enable=bool(spider_raw.get("enable", True)), json_url=str(spider_raw.get("json_url", "")).strip(), article_count=int(spider_raw.get("article_count", 5)), - merge_result=MergeResultConfig( - enable=bool(merge_raw.get("enable", False)), - merge_json_url=str(merge_raw.get("merge_json_url", "")).strip(), - ), + ), + merge_settings=MergeSettings( + enable=bool(merge_raw.get("enable", False)), + remote_base_url=str(merge_raw.get("remote_base_url", "")).strip(), + merge_article_data=bool(merge_raw.get("merge_article_data", True)), + merge_link_check_data=bool(merge_raw.get("merge_link_check_data", True)), + ), + link_check=LinkCheckConfig( + enable=bool(link_check_raw.get("enable", True)), + max_age_hours=int(link_check_raw.get("max_age_hours", 24)), + timeout=int(link_check_raw.get("timeout", 15)), + max_workers=int(link_check_raw.get("max_workers", 10)), + proxy_url=str(link_check_raw.get("proxy_url", "")).strip(), + status_api_url=str(link_check_raw.get("status_api_url", "https://v2.xxapi.cn/api/status?url={url}")).strip(), + enable_backlink_check=bool(link_check_raw.get("enable_backlink_check", False)), + author_url=str(link_check_raw.get("author_url", "")).strip(), ), email_push=EmailPushConfig( enable=bool(email_push_raw.get("enable", False)), @@ -139,6 +172,12 @@ def from_dict(cls, data: dict) -> "ApplicationConfig": use_tls=bool(smtp_raw.get("use_tls", True)), ), specific_rss=list(data.get("specific_RSS", []) or []), + runtime_paths=RuntimePaths( + cache_file=str(runtime_raw.get("cache_file", DEFAULT_CACHE_FILE)).strip() or DEFAULT_CACHE_FILE, + all_json_file=str(runtime_raw.get("all_json_file", DEFAULT_ALL_JSON)).strip() or DEFAULT_ALL_JSON, + errors_json_file=str(runtime_raw.get("errors_json_file", DEFAULT_ERRORS_JSON)).strip() or DEFAULT_ERRORS_JSON, + link_json_file=str(runtime_raw.get("link_json_file", DEFAULT_LINK_JSON)).strip() or DEFAULT_LINK_JSON, + ), ) diff --git a/friend_circle_lite/application.py b/friend_circle_lite/application.py index 1fbdcc154a0..2498225eeaf 100644 --- a/friend_circle_lite/application.py +++ b/friend_circle_lite/application.py @@ -53,13 +53,14 @@ def run_crawler_if_enabled(self) -> None: specific_RSS=self.config.specific_rss, count=spider_settings.article_count, cache_file=self.config.runtime_paths.cache_file, + link_check_config=self.config.link_check, ) if crawl_result is None: logging.error("❌ 抓取流程失败，未生成任何输出文件") return - result, lost_friends = crawl_result - result, lost_friends = self._merge_remote_results_if_enabled(result, lost_friends) + result, lost_friends, link_payload = crawl_result + result, lost_friends, link_payload = self._merge_remote_results_if_enabled(result, lost_friends, link_payload) article_count = len(result.get("article_data", [])) logging.info(f"📦 数据获取完毕，共有 {article_count} 篇文章，正在处理数据") @@ -70,6 +71,7 @@ def run_crawler_if_enabled(self) -> None: ) write_json(self.config.runtime_paths.all_json_file, result) write_json(self.config.runtime_paths.errors_json_file, lost_friends) + write_json(self.config.runtime_paths.link_json_file, link_payload) def prepare_mail_runtime(self) -> MailRuntime: """Build SMTP runtime credentials from config and environment variables.""" @@ -143,17 +145,26 @@ def run_rss_subscription_if_enabled(self, mail_runtime: MailRuntime) -> None: use_tls=mail_runtime.use_tls, ) - def _merge_remote_results_if_enabled(self, result: dict, lost_friends: list[list[str]]) -> tuple[dict, list[list[str]]]: - """Merge remote outputs when the self-hosted merge option is enabled.""" - merge_result = self.config.spider_settings.merge_result - if not merge_result.enable: - return result, lost_friends - - merge_url = merge_result.merge_json_url - logging.info(f"🔀 合并功能开启，从 {merge_url} 获取外部数据") - result = marge_data_from_json_url(result, f"{merge_url}/all.json") - lost_friends = marge_errors_from_json_url(lost_friends, f"{merge_url}/errors.json") - return result, lost_friends + def _merge_remote_results_if_enabled( + self, result: dict, lost_friends: list[list[str]], link_payload: dict + ) -> tuple[dict, list[list[str]], dict]: + """Merge remote outputs when the merge option is enabled.""" + merge_settings = self.config.merge_settings + if not merge_settings.enable: + return result, lost_friends, link_payload + + remote_url = merge_settings.remote_base_url + logging.info(f"🔀 合并功能开启，从 {remote_url} 获取外部数据") + + if merge_settings.merge_article_data: + result = marge_data_from_json_url(result, f"{remote_url}/all.json") + lost_friends = marge_errors_from_json_url(lost_friends, f"{remote_url}/errors.json") + + if merge_settings.merge_link_check_data: + from friend_circle_lite.all_friends import merge_link_data_from_json_url + link_payload = merge_link_data_from_json_url(link_payload, f"{remote_url}/link.json") + + return result, lost_friends, link_payload def _resolve_github_repo(self) -> tuple[str, str]: """Resolve repository coordinates from env override or config.""" diff --git a/friend_circle_lite/cache_store.py b/friend_circle_lite/cache_store.py index f5628ddb234..cfa1c997f93 100644 --- a/friend_circle_lite/cache_store.py +++ b/friend_circle_lite/cache_store.py @@ -22,7 +22,7 @@ import yaml -from friend_circle_lite.models import Article, CacheRecord +from friend_circle_lite.models import Article, CacheRecord, LinkCheckRecord, LinkMethodStatus class FeedCacheStore: @@ -306,3 +306,181 @@ def _load_legacy_json(self) -> list[Article]: ) ) return articles + + +class LinkCheckStore: + """Persist friend link reachability checks using SQLite.""" + + def __init__(self, cache_path: str | Path | None): + self.cache_path = Path(cache_path) if cache_path else None + + def load_records(self, urls: list[str] | None = None) -> dict[str, LinkCheckRecord]: + if not self.cache_path or not self.cache_path.exists(): + return {} + + try: + with sqlite3.connect(self.cache_path) as connection: + self._ensure_schema(connection) + rows = connection.execute( + """ + SELECT url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, + best_method, best_latency, fail_count, backlink_checked, has_author_link, + rss_crawl_reason, direct_success, direct_status_code, direct_latency, + proxy_success, proxy_status_code, proxy_latency, api_success, + api_status_code, api_latency + FROM link_check_state + """ + ).fetchall() + except Exception as exc: + logging.warning(f"读取友链检测缓存失败: {exc}") + return {} + + allowed_urls = set(urls or []) + records: dict[str, LinkCheckRecord] = {} + for row in rows: + ( + url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, + best_method, best_latency, fail_count, backlink_checked, has_author_link, + rss_crawl_reason, direct_success, direct_status_code, direct_latency, + proxy_success, proxy_status_code, proxy_latency, api_success, + api_status_code, api_latency, + ) = row + if allowed_urls and url not in allowed_urls: + continue + records[url] = LinkCheckRecord( + name=name or "", + url=url or "", + avatar=avatar or "", + linkpage=linkpage or "", + checked_at=checked_at or "", + reachable=bool(reachable), + crawl_allowed=bool(crawl_allowed), + best_method=best_method or "none", + best_latency=best_latency if best_latency is not None else -1, + fail_count=fail_count or 0, + backlink_checked=bool(backlink_checked), + has_author_link=bool(has_author_link), + rss_crawl_reason=rss_crawl_reason or "blocked_unreachable", + direct=LinkMethodStatus(bool(direct_success), direct_status_code, direct_latency if direct_latency is not None else -1), + proxy=LinkMethodStatus(bool(proxy_success), proxy_status_code, proxy_latency if proxy_latency is not None else -1), + api=LinkMethodStatus(bool(api_success), api_status_code, api_latency if api_latency is not None else -1), + ) + return records + + def save_records(self, records: list[LinkCheckRecord]) -> bool: + if not self.cache_path: + return True + + try: + self.cache_path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(self.cache_path) as connection: + self._ensure_schema(connection) + connection.executemany( + """ + INSERT INTO link_check_state( + url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, + best_method, best_latency, fail_count, backlink_checked, has_author_link, + rss_crawl_reason, direct_success, direct_status_code, direct_latency, + proxy_success, proxy_status_code, proxy_latency, api_success, + api_status_code, api_latency + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(url) DO UPDATE SET + name = excluded.name, + avatar = excluded.avatar, + linkpage = excluded.linkpage, + checked_at = excluded.checked_at, + reachable = excluded.reachable, + crawl_allowed = excluded.crawl_allowed, + best_method = excluded.best_method, + best_latency = excluded.best_latency, + fail_count = excluded.fail_count, + backlink_checked = excluded.backlink_checked, + has_author_link = excluded.has_author_link, + rss_crawl_reason = excluded.rss_crawl_reason, + direct_success = excluded.direct_success, + direct_status_code = excluded.direct_status_code, + direct_latency = excluded.direct_latency, + proxy_success = excluded.proxy_success, + proxy_status_code = excluded.proxy_status_code, + proxy_latency = excluded.proxy_latency, + api_success = excluded.api_success, + api_status_code = excluded.api_status_code, + api_latency = excluded.api_latency + """, + [self._record_to_row(record) for record in records], + ) + connection.commit() + logging.info(f"友链检测缓存已保存（{len(records)} 条）") + return True + except Exception as exc: + logging.error(f"保存友链检测缓存失败: {exc}") + return False + + @staticmethod + def is_fresh(record: LinkCheckRecord, max_age_hours: int) -> bool: + if not record.checked_at: + return False + try: + checked_at = datetime.strptime(record.checked_at, "%Y-%m-%d %H:%M:%S") + except ValueError: + return False + age_seconds = (datetime.now() - checked_at).total_seconds() + return age_seconds < max_age_hours * 3600 + + @staticmethod + def _record_to_row(record: LinkCheckRecord) -> tuple: + return ( + record.url, + record.name, + record.avatar, + record.linkpage, + record.checked_at, + int(record.reachable), + int(record.crawl_allowed), + record.best_method, + record.best_latency, + record.fail_count, + int(record.backlink_checked), + int(record.has_author_link), + record.rss_crawl_reason, + int(record.direct.success), + record.direct.status_code, + record.direct.latency, + int(record.proxy.success), + record.proxy.status_code, + record.proxy.latency, + int(record.api.success), + record.api.status_code, + record.api.latency, + ) + + @staticmethod + def _ensure_schema(connection: sqlite3.Connection) -> None: + connection.execute( + """ + CREATE TABLE IF NOT EXISTS link_check_state ( + url TEXT PRIMARY KEY, + name TEXT NOT NULL, + avatar TEXT DEFAULT '', + linkpage TEXT DEFAULT '', + checked_at TEXT NOT NULL, + reachable INTEGER NOT NULL DEFAULT 0, + crawl_allowed INTEGER NOT NULL DEFAULT 0, + best_method TEXT NOT NULL DEFAULT 'none', + best_latency REAL DEFAULT -1, + fail_count INTEGER NOT NULL DEFAULT 0, + backlink_checked INTEGER NOT NULL DEFAULT 0, + has_author_link INTEGER NOT NULL DEFAULT 0, + rss_crawl_reason TEXT NOT NULL DEFAULT '', + direct_success INTEGER NOT NULL DEFAULT 0, + direct_status_code INTEGER, + direct_latency REAL DEFAULT -1, + proxy_success INTEGER NOT NULL DEFAULT 0, + proxy_status_code INTEGER, + proxy_latency REAL DEFAULT -1, + api_success INTEGER NOT NULL DEFAULT 0, + api_status_code INTEGER, + api_latency REAL DEFAULT -1 + ) + """ + ) diff --git a/friend_circle_lite/crawler_service.py b/friend_circle_lite/crawler_service.py index 002c9d35640..87a427f7254 100644 --- a/friend_circle_lite/crawler_service.py +++ b/friend_circle_lite/crawler_service.py @@ -15,9 +15,11 @@ import requests from friend_circle_lite import HEADERS_JSON, timeout -from friend_circle_lite.cache_store import FeedCacheStore +from friend_circle_lite.app_config import LinkCheckConfig +from friend_circle_lite.cache_store import FeedCacheStore, LinkCheckStore from friend_circle_lite.feed_service import FeedDiscoveryService, FeedParserService -from friend_circle_lite.models import Article, CacheRecord, CacheUpdate, CrawlResult, CrawlStatistics, FeedEndpoint, Website +from friend_circle_lite.link_check_service import LinkCheckService +from friend_circle_lite.models import Article, CacheRecord, CacheUpdate, CrawlResult, CrawlStatistics, FeedEndpoint, LinkCheckRecord, Website class WebsiteFeedResolver: @@ -116,11 +118,20 @@ def _parse_articles(self, endpoint: FeedEndpoint | None, website: Website, count class FriendCircleCrawler: """System-level orchestrator for crawling all configured websites.""" - def __init__(self, json_url: str, count: int, specific_rss: list[dict] | None = None, cache_file: str | None = None): + def __init__( + self, + json_url: str, + count: int, + specific_rss: list[dict] | None = None, + cache_file: str | None = None, + link_check_config: LinkCheckConfig | None = None, + ): self.json_url = json_url self.count = count self.specific_rss = specific_rss or [] self.cache_store = FeedCacheStore(cache_file) + self.link_check_config = link_check_config or LinkCheckConfig(enable=False) + self.link_check_store = LinkCheckStore(cache_file) def run(self) -> tuple[dict, list[list[str]]] | None: """Fetch website list, crawl all websites, and build public outputs.""" @@ -129,6 +140,13 @@ def run(self) -> tuple[dict, list[list[str]]] | None: if websites is None: return None + link_check_records = self._check_links(websites) + link_check_map = {record.url: record for record in link_check_records} + crawlable_websites = [website for website in websites if link_check_map.get(website.url, LinkCheckRecord.unchecked(website)).crawl_allowed] + skipped_count = len(websites) - len(crawlable_websites) + if skipped_count: + logging.info(f"🔎 根据友链可达性检测跳过 {skipped_count} 个不可抓取站点") + cache_records = self.cache_store.load_records() manual_records = self._build_manual_records() merged_records = self._merge_feed_records(cache_records, manual_records) @@ -143,7 +161,7 @@ def run(self) -> tuple[dict, list[list[str]]] | None: with ThreadPoolExecutor(max_workers=10) as executor: future_to_website = { executor.submit(crawler.crawl, website, self.count): website - for website in websites + for website in crawlable_websites } for future in as_completed(future_to_website): website = future_to_website[future] @@ -156,24 +174,88 @@ def run(self) -> tuple[dict, list[list[str]]] | None: self._apply_cache_updates(cache_records, crawl_results, manual_names) active_results = [result for result in crawl_results if result.status == "active"] - error_results = [result.website.to_error_payload() for result in crawl_results if result.status != "active"] + unreachable_results = [record for record in link_check_records if not record.reachable] + crawl_error_results = [result.website.to_error_payload() for result in crawl_results if result.status != "active"] + error_results = [[record.name, record.url, record.avatar] for record in unreachable_results] all_articles = [article.to_public_dict() for result in active_results for article in result.articles] statistics = CrawlStatistics.create( friends_num=len(websites), active_num=len(active_results), - error_num=len(error_results), + error_num=len(websites) - len(active_results), article_num=len(all_articles), ) + stats_payload = statistics.to_dict() + stats_payload.update(self._build_link_statistics(link_check_records)) result = { - "statistical_data": statistics.to_dict(), + "statistical_data": stats_payload, "article_data": all_articles, } + link_payload = self._build_link_payload(link_check_records) logging.info( - f"数据处理完成，总共有 {len(websites)} 位朋友，其中 {len(active_results)} 位博客可访问，" - f"{len(error_results)} 位博客无法访问。" + f"数据处理完成，总共有 {len(websites)} 位朋友，其中 {len(active_results)} 位博客可抓取到文章，" + f"{len(crawl_error_results)} 位博客 RSS 抓取失败，{len(unreachable_results)} 位友链不可达。" ) - return result, error_results + return result, error_results, link_payload + + def _check_links(self, websites: list[Website]) -> list[LinkCheckRecord]: + service = LinkCheckService(config=self.link_check_config, store=self.link_check_store) + return service.check_websites(websites) + + @staticmethod + def _build_link_statistics(records: list[LinkCheckRecord]) -> dict[str, int | str]: + reachable = [record for record in records if record.reachable] + crawl_allowed = [record for record in records if record.crawl_allowed] + api_only = [record for record in records if record.best_method == "api"] + has_author_link = [record for record in records if record.has_author_link] + checked_times = [record.checked_at for record in records if record.checked_at] + return { + "link_total_num": len(records), + "link_reachable_num": len(reachable), + "link_unreachable_num": len(records) - len(reachable), + "crawl_allowed_num": len(crawl_allowed), + "api_only_num": len(api_only), + "has_author_link_num": len(has_author_link), + "link_last_checked_time": max(checked_times) if checked_times else "", + } + + @staticmethod + def _build_link_payload(records: list[LinkCheckRecord]) -> dict[str, object]: + return { + "statistical_data": FriendCircleCrawler._build_link_statistics(records), + "link_data": [record.to_link_dict() for record in records], + } + + @staticmethod + def _build_friend_data( + websites: list[Website], + crawl_results: list[CrawlResult], + link_check_map: dict[str, LinkCheckRecord], + ) -> list[dict[str, object]]: + crawl_result_map = {result.website.url: result for result in crawl_results} + friend_data: list[dict[str, object]] = [] + for website in websites: + link_record = link_check_map.get(website.url) or LinkCheckRecord.unchecked(website) + crawl_result = crawl_result_map.get(website.url) + friend_data.append({ + "name": website.name, + "url": website.url, + "avatar": website.avatar, + "linkpage": website.linkpage, + "reachable": link_record.reachable, + "crawl_allowed": link_record.crawl_allowed, + "best_method": link_record.best_method, + "best_latency": link_record.best_latency, + "fail_count": link_record.fail_count, + "backlink_checked": link_record.backlink_checked, + "has_author_link": link_record.has_author_link, + "rss_crawl_reason": link_record.rss_crawl_reason, + "feed_status": crawl_result.status if crawl_result else "skipped", + "feed_url": crawl_result.feed_url if crawl_result else None, + "feed_type": crawl_result.feed_type if crawl_result else "none", + "article_count": len(crawl_result.articles) if crawl_result else 0, + }) + return friend_data def _load_websites(self, session: requests.Session) -> list[Website] | None: try: diff --git a/friend_circle_lite/link_check_service.py b/friend_circle_lite/link_check_service.py new file mode 100644 index 00000000000..626fd260dca --- /dev/null +++ b/friend_circle_lite/link_check_service.py @@ -0,0 +1,283 @@ +"""Friend link reachability checks used before RSS crawling.""" + +from __future__ import annotations + +import logging +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +from urllib.parse import quote, urlparse + +import requests + +from friend_circle_lite.app_config import LinkCheckConfig +from friend_circle_lite.cache_store import LinkCheckStore +from friend_circle_lite.models import LinkCheckRecord, LinkMethodStatus, Website + + +LINK_CHECK_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/123.0.0.0 Safari/537.36 " + "(Friend-Circle-Lite/2.0; +https://github.com/willow-god/Friend-Circle-Lite)" + ), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "zh-CN,zh;q=0.9", + "Connection": "keep-alive", + "X-Friend-Circle-Link-Check": "1.0", +} + +RAW_HEADERS = { + "User-Agent": LINK_CHECK_HEADERS["User-Agent"], + "X-Friend-Circle-Link-Check": "1.0", +} + + +class LinkCheckService: + """Check friend homepage reachability and cache results.""" + + def __init__(self, config: LinkCheckConfig, store: LinkCheckStore): + self.config = config + self.store = store + + def check_websites(self, websites: list[Website]) -> list[LinkCheckRecord]: + if not self.config.enable: + now = self._now_text() + return [self._build_disabled_record(website, now) for website in websites] + + cached_records = self.store.load_records([website.url for website in websites]) + records_by_url: dict[str, LinkCheckRecord] = {} + websites_to_check: list[Website] = [] + + for website in websites: + cached = cached_records.get(website.url) + if cached and self._can_reuse_cached_record(cached, website): + records_by_url[website.url] = self._refresh_cached_metadata(cached, website) + else: + websites_to_check.append(website) + + if websites_to_check: + logging.info(f"🔎 开始检测 {len(websites_to_check)} 个友链可达性") + checked_records = self._check_fresh_websites(websites_to_check, cached_records) + self.store.save_records(checked_records) + for record in checked_records: + records_by_url[record.url] = record + else: + logging.info("🔎 友链可达性检测缓存仍有效，本次复用缓存结果") + + return [records_by_url.get(website.url) or LinkCheckRecord.unchecked(website) for website in websites] + + def _check_fresh_websites(self, websites: list[Website], cached_records: dict[str, LinkCheckRecord]) -> list[LinkCheckRecord]: + records: list[LinkCheckRecord] = [] + with requests.Session() as session: + with ThreadPoolExecutor(max_workers=max(1, self.config.max_workers)) as executor: + future_to_website = { + executor.submit(self._check_website, session, website, cached_records.get(website.url)): website + for website in websites + } + for future in as_completed(future_to_website): + website = future_to_website[future] + try: + records.append(future.result()) + except Exception as exc: + logging.warning(f"友链 {website.name} 检测失败: {exc}") + records.append(self._build_failed_record(website, cached_records.get(website.url))) + return records + + def _check_website(self, session: requests.Session, website: Website, cached: LinkCheckRecord | None) -> LinkCheckRecord: + direct = self._request_method(session, website.url, "直接访问") + proxy = LinkMethodStatus() + api = LinkMethodStatus() + + if not direct.success: + proxy_url = self._build_proxy_url(website.url) + if proxy_url: + proxy = self._request_method(session, proxy_url, "代理访问") + + if not direct.success and not proxy.success: + api = self._request_api(session, website.url) + time.sleep(0.2) + + record = self._compose_record(website, cached, direct, proxy, api) + if record.reachable and self.config.enable_backlink_check and self.config.author_url and website.linkpage: + record.backlink_checked = True + record.has_author_link = self._check_author_link_in_page(session, website.linkpage) + return record + + def _request_method(self, session: requests.Session, url: str, desc: str) -> LinkMethodStatus: + if not self._is_url(url): + return LinkMethodStatus() + + response, latency = self._request_url(session, url, headers=LINK_CHECK_HEADERS, desc=desc) + if response is None: + return LinkMethodStatus(success=False, status_code=None, latency=latency) + success = response.status_code == 200 + if success: + logging.info(f"[{desc}] 成功访问: {url}，延迟 {latency} 秒") + else: + logging.warning(f"[{desc}] 状态码异常: {url} -> {response.status_code}") + return LinkMethodStatus(success=success, status_code=response.status_code, latency=latency) + + def _request_api(self, session: requests.Session, url: str) -> LinkMethodStatus: + if not self.config.status_api_url: + return LinkMethodStatus() + + api_url = self.config.status_api_url.format(url=quote(url, safe="")) + response, latency = self._request_url(session, api_url, headers=RAW_HEADERS, desc="API 检查", timeout=30) + if response is None: + return LinkMethodStatus(success=False, status_code=None, latency=latency) + + try: + payload = response.json() + status_code = int(payload.get("data", 0)) + success = int(payload.get("code", 0)) == 200 and status_code == 200 + if success: + logging.info(f"[API] 成功访问: {url}，状态码 200") + else: + logging.warning(f"[API] 状态异常: {url} -> [{payload.get('code')}, {payload.get('data')}]") + return LinkMethodStatus(success=success, status_code=status_code, latency=latency) + except Exception as exc: + logging.warning(f"[API] 解析响应失败: {url}，错误: {exc}") + return LinkMethodStatus(success=False, status_code=response.status_code, latency=latency) + + def _compose_record( + self, + website: Website, + cached: LinkCheckRecord | None, + direct: LinkMethodStatus, + proxy: LinkMethodStatus, + api: LinkMethodStatus, + ) -> LinkCheckRecord: + reachable = direct.success or proxy.success or api.success + crawl_allowed = direct.success or proxy.success + if direct.success: + best_method = "direct" + best_latency = direct.latency + reason = "allowed_by_direct" + elif proxy.success: + best_method = "proxy" + best_latency = proxy.latency + reason = "allowed_by_proxy" + elif api.success: + best_method = "api" + best_latency = api.latency + reason = "blocked_api_only" + else: + best_method = "none" + best_latency = -1 + reason = "blocked_unreachable" + + fail_count = 0 if reachable else ((cached.fail_count if cached else 0) + 1) + return LinkCheckRecord( + name=website.name, + url=website.url, + avatar=website.avatar, + linkpage=website.linkpage, + checked_at=self._now_text(), + reachable=reachable, + crawl_allowed=crawl_allowed, + best_method=best_method, + best_latency=best_latency, + fail_count=fail_count, + rss_crawl_reason=reason, + direct=direct, + proxy=proxy, + api=api, + ) + + def _check_author_link_in_page(self, session: requests.Session, linkpage_url: str) -> bool: + response, _ = self._request_url(session, linkpage_url, headers=RAW_HEADERS, desc="友链页面检测") + if not response: + return False + + author_url = self.config.author_url + if not author_url.startswith(("http://", "https://")): + author_url = "https://" + author_url + + variants = { + author_url, + author_url.replace("https://", "http://"), + author_url.replace("https://", "//"), + author_url.replace("https://", ""), + self.config.author_url, + "//" + self.config.author_url, + "https://" + self.config.author_url, + "http://" + self.config.author_url, + } + content = response.text + for variant in variants: + if ( + f'href="{variant}"' in content + or f"href='{variant}'" in content + or f'href="{variant}/"' in content + or f"href='{variant}/'" in content + or variant in content + ): + return True + return False + + def _request_url( + self, + session: requests.Session, + url: str, + headers: dict[str, str], + desc: str, + timeout: int | None = None, + ) -> tuple[requests.Response | None, float]: + try: + start_time = time.time() + response = session.get(url, headers=headers, timeout=timeout or self.config.timeout) + return response, round(time.time() - start_time, 2) + except requests.RequestException as exc: + logging.warning(f"[{desc}] 请求失败: {url}，错误: {exc}") + return None, -1 + + def _can_reuse_cached_record(self, cached: LinkCheckRecord, website: Website) -> bool: + if self.config.enable_backlink_check and cached.linkpage != website.linkpage: + return False + return self.store.is_fresh(cached, self.config.max_age_hours) + + @staticmethod + def _refresh_cached_metadata(cached: LinkCheckRecord, website: Website) -> LinkCheckRecord: + cached.name = website.name + cached.avatar = website.avatar + cached.linkpage = website.linkpage + return cached + + def _build_failed_record(self, website: Website, cached: LinkCheckRecord | None) -> LinkCheckRecord: + record = LinkCheckRecord.unchecked(website, self._now_text()) + record.fail_count = (cached.fail_count if cached else 0) + 1 + return record + + @staticmethod + def _build_disabled_record(website: Website, checked_at: str) -> LinkCheckRecord: + return LinkCheckRecord( + name=website.name, + url=website.url, + avatar=website.avatar, + linkpage=website.linkpage, + checked_at=checked_at, + reachable=True, + crawl_allowed=True, + best_method="disabled", + best_latency=-1, + rss_crawl_reason="link_check_disabled", + ) + + def _build_proxy_url(self, url: str) -> str: + if not self.config.proxy_url: + return "" + if "{}" in self.config.proxy_url: + return self.config.proxy_url.format(url) + if "{url}" in self.config.proxy_url: + return self.config.proxy_url.format(url=url) + return f"{self.config.proxy_url}{url}" + + @staticmethod + def _is_url(path: str) -> bool: + return urlparse(path).scheme in ("http", "https") + + @staticmethod + def _now_text() -> str: + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") diff --git a/friend_circle_lite/models.py b/friend_circle_lite/models.py index 8b23946f997..b8ab4390232 100644 --- a/friend_circle_lite/models.py +++ b/friend_circle_lite/models.py @@ -22,17 +22,120 @@ class Website: name: str url: str avatar: str = "" + linkpage: str = "" @classmethod - def from_friend_item(cls, raw_friend: list | tuple) -> "Website": - """Create a website from the existing `[name, url, avatar]` structure.""" - name, url, avatar = raw_friend - return cls(name=name, url=url, avatar=avatar or "") + def from_friend_item(cls, raw_friend: list | tuple | dict) -> "Website": + """Create a website from common friend link structures.""" + if isinstance(raw_friend, dict): + return cls( + name=str(raw_friend.get("name", "")).strip(), + url=str(raw_friend.get("link") or raw_friend.get("url") or "").strip(), + avatar=str(raw_friend.get("avatar", "")).strip(), + linkpage=str(raw_friend.get("linkpage", "")).strip(), + ) + + name = raw_friend[0] + url = raw_friend[1] + if len(raw_friend) > 3: + linkpage = raw_friend[2] + avatar = raw_friend[3] + else: + linkpage = "" + avatar = raw_friend[2] if len(raw_friend) > 2 else "" + return cls(name=str(name).strip(), url=str(url).strip(), avatar=str(avatar or "").strip(), linkpage=str(linkpage or "").strip()) def to_error_payload(self) -> list[str]: """Return the legacy structure used by `errors.json`.""" return [self.name, self.url, self.avatar] + def to_public_dict(self) -> dict[str, str]: + return { + "name": self.name, + "url": self.url, + "avatar": self.avatar, + "linkpage": self.linkpage, + } + + +@dataclass(slots=True) +class LinkMethodStatus: + """Status for one link-check method.""" + + success: bool = False + status_code: int | None = None + latency: float = -1 + + def to_dict(self) -> dict[str, bool | int | float | None]: + return { + "success": self.success, + "status_code": self.status_code, + "latency": self.latency, + } + + +@dataclass(slots=True) +class LinkCheckRecord: + """Reachability status for one friend website.""" + + name: str + url: str + avatar: str = "" + linkpage: str = "" + checked_at: str = "" + reachable: bool = False + crawl_allowed: bool = False + best_method: str = "none" + best_latency: float = -1 + fail_count: int = 0 + backlink_checked: bool = False + has_author_link: bool = False + rss_crawl_reason: str = "blocked_unreachable" + direct: LinkMethodStatus = field(default_factory=LinkMethodStatus) + proxy: LinkMethodStatus = field(default_factory=LinkMethodStatus) + api: LinkMethodStatus = field(default_factory=LinkMethodStatus) + + @classmethod + def unchecked(cls, website: Website, checked_at: str = "") -> "LinkCheckRecord": + return cls(name=website.name, url=website.url, avatar=website.avatar, linkpage=website.linkpage, checked_at=checked_at) + + def to_public_dict(self) -> dict[str, object]: + return { + "name": self.name, + "url": self.url, + "avatar": self.avatar, + "linkpage": self.linkpage, + "checked_at": self.checked_at, + "reachable": self.reachable, + "crawl_allowed": self.crawl_allowed, + "best_method": self.best_method, + "best_latency": self.best_latency, + "fail_count": self.fail_count, + "backlink_checked": self.backlink_checked, + "has_author_link": self.has_author_link, + "rss_crawl_reason": self.rss_crawl_reason, + "methods": { + "direct": self.direct.to_dict(), + "proxy": self.proxy.to_dict(), + "api": self.api.to_dict(), + }, + } + def to_link_dict(self) -> dict[str, object]: + return { + "name": self.name, + "link": self.url, + "link_page": self.linkpage, + "avatar": self.avatar, + "reachable": self.reachable, + "crawlable": self.crawl_allowed, + "method": self.best_method, + "latency": self.best_latency, + "fail_count": self.fail_count, + "checked_at": self.checked_at, + "has_backlink": self.has_author_link if self.backlink_checked else None, + "reason": self.rss_crawl_reason, + } + @dataclass(slots=True) class Article: diff --git a/main/fclite.js b/main/fclite.js index d4dc75d0761..d4cad5ea750 100644 --- a/main/fclite.js +++ b/main/fclite.js @@ -95,7 +95,8 @@ function initialize_fc_lite() { } function processArticles(data) { - allArticles = data.article_data; + allArticles = data.article_data || []; + // 处理统计数据 const stats = data.statistical_data; @@ -159,6 +160,14 @@ function initialize_fc_lite() { // 显示随机文章的逻辑 function displayRandomArticle(stats) { const randomArticle = allArticles[Math.floor(Math.random() * allArticles.length)]; + if (!randomArticle) { + randomArticleContainer.innerHTML = ` +
+
暂无可展示文章
+
+ `; + return; + } randomArticleContainer.innerHTML = `
diff --git a/readme.md b/readme.md index 0b5b41c5c4b..3c18accc8ea 100644 --- a/readme.md +++ b/readme.md @@ -11,6 +11,24 @@ ## 开发进度 +### 🎉 2026-05-31 - v2.0.0 重大更新 + +> **⚠️ 重要更新**：本次更新包含**数据结构变更**和**配置文件调整**，请查看 **[完整更新日志 (CHANGELOG.md)](./CHANGELOG.md)** 了解详情。 +> +> **✅ 兼容性更新**：保持向后兼容，旧版引用者无需修改代码，`all.json` 结构保持兼容，`main/fclite.js` 和 `main/fclite.css` 不变。 + +* **✨ 新增友链可达性检测功能**：合并 [check-flink](https://github.com/willow-god/check-flink) 项目，支持直连、代理、API 三种检测方式，并支持反链检测。 +* **📊 数据结构重构**：友链可达性数据独立输出到 `link.json`，`all.json` 仅保留友圈文章数据，实现数据分离。 +* **⚙️ 配置优化**：`merge_settings` 独立配置，友圈文章和友链可达性数据分别控制合并。 +* **🎨 前端页面重写**：新首页继承 check-flink 风格，同时展示友链状态和友圈文章。 +* **🔄 智能数据合并**：支持国内外数据源智能合并，可达性优先级、延迟、反链、失败次数均智能选择最优结果。 + +**详细说明**：[查看完整更新日志 →](./CHANGELOG.md) + +### 2026-05-25 + +* 合并友链可达性检测能力，检测结果会写入独立的 `link.json` 供前端按需获取，不可达或仅 API 可达的友链会跳过 RSS 抓取。 + ### 2026-05-24 * 移除原先基于 FastAPI 的简陋后端部署方式，后续自部署统一采用生成静态文件后作为纯静态网站托管的纯净态方式。 @@ -107,7 +125,8 @@ ## 项目介绍 -- **爬取文章**: 爬取所有友链的文章，结果放置在根目录的all.json文件中，方便读取并部署到前端。 +- **爬取文章**: 爬取所有可达友链的文章，结果放置在根目录的all.json文件中，方便读取并部署到前端。 +- **友链可达性检测**: 抓取前会检测友链是否可达，支持直连、代理和 API 三种检测方式；仅 API 可达的友链只展示状态，不参与 RSS 抓取，结果单独输出到 `link.json`。 - **邮箱推送更新(对作者推送所有友链更新)**: 作者可以通过邮箱订阅所有rss的更新（未来开发）。 - **issue邮箱订阅(对访客实时推送最新文章邮件)**: 基于`GitHub issue`的博客更新邮件订阅功能，游客可以通过简单的提交`issue`进行邮箱订阅站点更新，删除对应`issue`即可取消订阅。 - **文件分离**: 将生成任务和静态展示分离，前端文件与生成后的 `all.json` 可直接作为静态网站托管。 @@ -122,6 +141,8 @@ ## 功能概览 * 文章爬取 +* 友链可达性检测 +* 友链状态前端展示 * 暗色适配 * 显示作者所有文章 * 获取丢失友链数据 @@ -188,6 +209,31 @@ - `marge_json_path`：请填写网络地址的json文件，用于合并，不带空格！！！ + - **友链可达性检测配置** + 在抓取 RSS 前检测友链站点是否可访问，检测结果会写入独立的 `link.json` 并在 `status.html` 展示。友链数据兼容四字段格式：`["站点名称", "站点地址", "友链页地址", "头像地址"]`，如果第三个字段暂时没有可留空。 + + ```yaml + link_check: + enable: true + max_age_hours: 24 + timeout: 15 + max_workers: 10 + proxy_url: "" + status_api_url: "https://v2.xxapi.cn/api/status?url={url}" + enable_backlink_check: false + author_url: "" + ``` + + `enable`：是否启用友链可达性检测； + + `max_age_hours`：检测结果缓存时间，默认 24 小时，同一友链一天内不会重复检测； + + `proxy_url`：可选代理前缀，适合使用 Cloudflare Worker 等方式转发检测； + + `status_api_url`：兜底状态码 API。注意 API 只能确认状态码，无法提供页面内容，所以仅 API 可达的友链会展示为可达，但不会参与 RSS 抓取； + + `enable_backlink_check` 和 `author_url`：用于检测对方友链页是否包含你的站点链接，需要友链数据中包含 `linkpage` 字段。 + - **邮箱推送功能配置** 暂未实现，预留用于将每天的友链文章更新推送给指定邮箱。 @@ -370,7 +416,7 @@ ## 自部署使用方法 -自部署后续统一采用纯静态方式：本项目只负责定时生成 `all.json`、`errors.json` 等数据文件，生成完成后把 `static`、`main` 和数据文件作为静态网站托管即可，不再启动 FastAPI 后端服务。 +自部署后续统一采用纯静态方式：本项目只负责定时生成 `all.json`、`link.json`、`errors.json` 等数据文件，生成完成后把 `static`、`main` 和数据文件作为静态网站托管即可，不再启动 FastAPI 后端服务。友链检测缓存会保存在 `temp/cache.sqlite3` 中，用于判断同一友链是否需要在 24 小时后重新检测；静态网站只需要发布生成后的 JSON 和静态资源。 如果你有一台境内服务器，你也可以通过以下操作将其部署到你的服务器上，操作如下： diff --git a/static/index.html b/static/index.html index b725caccdc0..b3c4440b3d7 100644 --- a/static/index.html +++ b/static/index.html @@ -1,265 +1,763 @@ - - - - - - - - - Friend-Circle-Lite - - - - + + 100% { + transform: scale(1); + opacity: 1; + } + } + + .status-normal { + background: green; + } + + .status-normal::before { + background-color: rgba(0, 128, 0, 0.2); + } + + .status-slow { + background: rgb(255, 200, 0); + } + + .status-slow::before { + background-color: rgba(255, 174, 0, 0.2); + } + + .status-api { + background: #3498db; + } + + .status-api::before { + background-color: rgba(52, 152, 219, 0.2); + } + + .status-error { + background: red; + } + + .status-error::before { + background-color: rgba(255, 0, 0, 0.2); + } + + .backlink-status { + display: inline-flex; + align-items: center; + margin: 0 8px; + text-decoration: none; + transition: all 0.2s ease-in-out; + } + + .backlink-status:hover { + transform: scale(1.1); + } + + .backlink-icon { + font-size: 16px; + } + + .backlink-true { + color: #27ae60; + } + + .backlink-false { + color: #e74c3c; + } + + .status-indicators { + display: flex; + align-items: center; + gap: 8px; + flex-shrink: 0; + } + + .method-text, + .time-text { + color: #000000a5; + white-space: nowrap; + } + + .article-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(250px, 1fr)); + gap: 15px; + margin-top: 15px; + } + + .article-card { + display: flex; + flex-direction: column; + gap: 12px; + min-width: 0; + background: #fff; + border-radius: 8px; + padding: 15px; + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.1); + font-size: 1rem; + transition: all 0.2s ease-in-out; + } + + .article-link { + color: var(--primary-color); + text-decoration: none; + font-weight: bold; + line-height: 1.5; + min-height: 3em; + display: -webkit-box; + -webkit-line-clamp: 2; + line-clamp: 2; + -webkit-box-orient: vertical; + overflow: hidden; + text-overflow: ellipsis; + } + + .article-link:hover { + color: var(--secondary-color); + } + + .article-meta { + display: flex; + justify-content: space-between; + gap: 10px; + color: #000000a5; + font-size: 0.95rem; + } + + .article-author { + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; + } + + .empty-state { + grid-column: 1 / -1; + background: #fff; + border-radius: 8px; + padding: 20px; + color: #7f8c8d; + text-align: center; + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.1); + } + + footer { + margin-top: 50px; + padding-top: 30px; + border-top: 1px solid #ecf0f1; + width: 100%; + text-align: center; + color: #95a5a6; + line-height: 1.8; + } + + footer a { + color: #95a5a6; + text-decoration: none; + transition: all 0.2s ease-in-out; + } + + footer a:hover { + color: var(--secondary-color); + } + + .github-icon { + position: fixed; + bottom: 20px; + right: 20px; + z-index: 1000; + background: #fff; + border-radius: 50%; + box-shadow: 0 2px 6px rgba(0, 0, 0, 0.1); + width: 45px; + height: 45px; + display: flex; + justify-content: center; + align-items: center; + transition: all 0.2s ease-in-out; + } + + .github-icon:hover { + box-shadow: 0 3px 8px rgba(0, 0, 0, 0.3); + } + + .github-icon svg { + margin-top: 7px; + fill: #2c3e50; + width: 25px; + height: 25px; + } + + @media (max-width: 560px) { + body { + padding: 28px 12px; + } + + .container { + padding: 0; + } + + h1 { + font-size: 2.3rem; + } + + .toolbar { + align-items: stretch; + } + + .search-input { + max-width: none; + } + } + + + + +
+

凤凰台上凤凰游，凤去台空江自流。

+

吴宫花草埋幽径，晋代衣冠成古丘。

+

三山半落青天外，二水中分白鹭洲。

+

总为浮云能蔽日，长安不见使人愁。

+

- —— 节选自 李白《登金陵凤凰台》

+
+ +
+ + + + + +
+
- Avatar -

Friend-Circle-Lite
服务已运行

-
- 查看文档 - 测试接口 +
+

友链朋友圈状态

+

+ 基于 Friend-Circle-Lite 生成的纯静态数据，展示友链可达性与友圈文章。 +

+

更新时间：加载中...

+
+ all.json + link.json
- -
-
-
+ + +
+
+
+ + 友链总数 +
+ 加载中... +
+
+
+ + 可抓取友链 +
+ 加载中... +
+
+
+ + 错误友链 +
+ 加载中... +
+
+
+ + 友圈文章 +
+ 加载中... +
+
+ +
+
+ + + +
+ +
+ +
+ +

失效友链

+
+ +
+ +

其他友链

+
+ +
+ +

友圈文章

+
+ +
- - - - - - - - \ No newline at end of file + return ""; + } + + function renderArticles() { + const container = document.getElementById("article-container"); + const articles = state.articles.filter(matchKeywordForArticle).slice(0, 120); + container.innerHTML = articles.map((article) => ``).join("") || `
当前没有匹配文章
`; + } + + function getLinkStatus(link) { + if (!link.reachable) return { className: "status-error", title: "不可达" }; + if (link.method === "api") return { className: "status-api", title: "仅 API 可达" }; + if (link.latency > 4) return { className: "status-slow", title: "响应较慢" }; + if (link.method === "proxy") return { className: "status-api", title: "代理可达" }; + return { className: "status-normal", title: "直连可达" }; + } + + function matchFilter(link) { + if (state.filter === "error") return !link.reachable; + if (state.filter === "crawlable") return link.crawlable; + return true; + } + + function matchKeywordForLink(link) { + if (!state.keyword) return true; + return [link.name, link.link, link.link_page, link.reason] + .some((value) => String(value || "").toLowerCase().includes(state.keyword)); + } + + function matchKeywordForArticle(article) { + if (!state.keyword) return true; + return [article.title, article.author, article.link] + .some((value) => String(value || "").toLowerCase().includes(state.keyword)); + } + + function linkPriority(link) { + if (!link.reachable) return 0; + if (link.method === "api") return 1; + if (link.latency > 4) return 2; + return 3; + } + + function formatLatency(latency) { + return latency >= 0 ? `${latency}s` : "--"; + } + + function escapeHTML(value) { + return String(value || "").replace(/[&<>'"]/g, (char) => ({ + "&": "&", + "<": "<", + ">": ">", + "'": "'", + "\"": """, + }[char])); + } + + function escapeAttr(value) { + return escapeHTML(value); + } + + + From a50101a8ddd2180c54e333089f451f4e7f4ffd92 Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Sat, 6 Jun 2026 21:45:36 +0800 Subject: [PATCH 17/30] =?UTF-8?q?=F0=9F=A4=AA=E5=B0=86=E4=BB=A3=E7=90=86?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E5=8D=95=E7=8B=AC=E6=8F=90=E5=8F=96=E5=87=BA?= =?UTF-8?q?=E6=9D=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/friend_circle_lite.yml | 1 + CHANGELOG.md | 2 + conf.yaml | 14 ++++-- friend_circle_lite/all_friends.py | 8 ++-- friend_circle_lite/app_config.py | 15 ++++++- friend_circle_lite/application.py | 3 ++ friend_circle_lite/config_printer.py | 54 ++++++++++++++++++++++++ friend_circle_lite/crawler_service.py | 10 +++-- friend_circle_lite/link_check_service.py | 17 ++++---- readme.md | 2 +- 10 files changed, 105 insertions(+), 21 deletions(-) create mode 100644 friend_circle_lite/config_printer.py diff --git a/.github/workflows/friend_circle_lite.yml b/.github/workflows/friend_circle_lite.yml index 3096cd41dca..8d42e7e4024 100644 --- a/.github/workflows/friend_circle_lite.yml +++ b/.github/workflows/friend_circle_lite.yml @@ -55,6 +55,7 @@ jobs: - name: Check RSS feeds env: SMTP_PWD: ${{ secrets.SMTP_PWD }} + PROXY_URL: ${{ secrets.PROXY_URL }} FCL_REPO: ${{ github.repository }} run: | echo "Checking RSS feeds..." diff --git a/CHANGELOG.md b/CHANGELOG.md index dc9c529f6ff..c3a8a0eb071 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -174,6 +174,8 @@ link_check: timeout: 15 # 请求超时时间（秒） max_workers: 10 # 并发检测数量 proxy_url: "" # 代理前缀（如 Cloudflare Worker） + # ⚠️ 代理服务可能违反某些服务条款，请谨慎使用 + # 支持环境变量 LINK_CHECK_PROXY_URL 覆盖（优先级更高） status_api_url: "https://v2.xxapi.cn/api/status?url={url}" # 兜底 API enable_backlink_check: true # 是否检测反链 author_url: "blog.liushen.fun" # 你的站点域名 diff --git a/conf.yaml b/conf.yaml index b93d4db9a2f..537a07a106a 100644 --- a/conf.yaml +++ b/conf.yaml @@ -8,6 +8,16 @@ spider_settings: json_url: "https://blog.liushen.fun/friend.json" article_count: 5 +# 代理配置 +# 解释：用于友链检测和 RSS 抓取的代理服务，可以访问被墙站点并获取完整页面内容 +# proxy_url: 代理前缀，比如 Nginx 反向代理或 Cloudflare Worker +# 示例：https://nginx.430070.xyz/{url} 或 https://proxy.example.com?url={url} +# ⚠️ 注意：代理服务可能违反某些服务条款，请谨慎使用，建议仅用于调试 +# 支持环境变量 PROXY_URL 覆盖此配置（优先级更高） +# 留空则不使用代理 +proxy_settings: + proxy_url: "" + # 数据合并配置 # 解释：合并多个数据源的结果，比如国内和国外执行结果，解决部分网站访问受限问题 # enable: 是否启用数据合并功能 @@ -21,12 +31,11 @@ merge_settings: merge_link_check_data: true # 友链可达性检测配置 -# 解释：先检查友链站点是否可达，再决定是否继续抓取该站点 RSS，检测结果会写入 all.json 供前端展示 +# 解释：先检查友链站点是否可达，再决定是否继续抓取该站点 RSS，检测结果会写入 link.json 供前端展示 # enable: 是否启用友链可达性检测 # max_age_hours: 同一友链检测结果缓存时间，默认 24 小时 # timeout: 单次请求超时时间 # max_workers: 并发检测数量 -# proxy_url: 可选代理前缀，比如 Cloudflare Worker 转发地址，留空则不启用 # status_api_url: 兜底状态码 API，API-only 结果只用于可达性展示，不参与 RSS 抓取 # enable_backlink_check: 是否检测友链页是否包含你的站点链接 # author_url: 你的站点域名，用于反链检测，建议只填写域名 @@ -35,7 +44,6 @@ link_check: max_age_hours: 24 timeout: 15 max_workers: 10 - proxy_url: "" status_api_url: "https://v2.xxapi.cn/api/status?url={url}" enable_backlink_check: true author_url: "blog.liushen.fun" diff --git a/friend_circle_lite/all_friends.py b/friend_circle_lite/all_friends.py index f55ca2b1f0a..aabd71ca1d3 100644 --- a/friend_circle_lite/all_friends.py +++ b/friend_circle_lite/all_friends.py @@ -21,6 +21,7 @@ def fetch_and_process_data( count: int = 5, cache_file: str = None, link_check_config=None, + proxy_settings=None, ): """Legacy wrapper around the new crawler orchestration service.""" return FriendCircleCrawler( @@ -29,6 +30,7 @@ def fetch_and_process_data( specific_rss=specific_RSS, cache_file=cache_file, link_check_config=link_check_config, + proxy_settings=proxy_settings, ).run() def sort_articles_by_time(data, future_tolerance_days=2): @@ -50,7 +52,7 @@ def marge_data_from_json_url(data, marge_json_url): response = requests.get(marge_json_url, headers=HEADERS_JSON, timeout=timeout) marge_data = response.json() except Exception as e: - logging.error(f"无法获取链接：{marge_json_url}，出现的问题为：{e}", exc_info=True) + logging.error(f"无法获取链接：{marge_json_url} ，出现的问题为：{e}", exc_info=True) return data if 'article_data' in marge_data: @@ -82,7 +84,7 @@ def merge_link_data_from_json_url(link_data, merge_json_url): response = requests.get(merge_json_url, headers=HEADERS_JSON, timeout=timeout) remote_data = response.json() except Exception as e: - logging.warning(f"无法获取友链数据：{merge_json_url}，跳过友链数据合并。错误：{e}") + logging.warning(f"无法获取友链数据：{merge_json_url} ，跳过友链数据合并。错误：{e}") return link_data if 'link_data' not in remote_data: @@ -210,7 +212,7 @@ def marge_errors_from_json_url(errors, marge_json_url): response = requests.get(marge_json_url, timeout=10) # 设置请求超时时间 marge_errors = response.json() except Exception as e: - logging.error(f"无法获取链接：{marge_json_url}，出现的问题为：{e}", exc_info=True) + logging.error(f"无法获取链接：{marge_json_url} ，出现的问题为：{e}", exc_info=True) return errors # 提取 marge_errors 中的 URL diff --git a/friend_circle_lite/app_config.py b/friend_circle_lite/app_config.py index 34300ea919d..7635506cc74 100644 --- a/friend_circle_lite/app_config.py +++ b/friend_circle_lite/app_config.py @@ -10,6 +10,7 @@ from __future__ import annotations +import os from dataclasses import dataclass, field @@ -29,6 +30,13 @@ class MergeSettings: merge_link_check_data: bool = True +@dataclass(slots=True) +class ProxySettings: + """Proxy configuration for both link checking and RSS crawling.""" + + proxy_url: str = "" + + @dataclass(slots=True) class SpiderSettings: """Crawler settings controlling source list and output density.""" @@ -46,7 +54,6 @@ class LinkCheckConfig: max_age_hours: int = 24 timeout: int = 15 max_workers: int = 10 - proxy_url: str = "" status_api_url: str = "https://v2.xxapi.cn/api/status?url={url}" enable_backlink_check: bool = False author_url: str = "" @@ -106,6 +113,7 @@ class ApplicationConfig: """Root application configuration assembled from the YAML file.""" spider_settings: SpiderSettings + proxy_settings: ProxySettings merge_settings: MergeSettings link_check: LinkCheckConfig email_push: EmailPushConfig @@ -119,6 +127,7 @@ class ApplicationConfig: def from_dict(cls, data: dict) -> "ApplicationConfig": """Create a typed config object from the raw YAML dictionary.""" spider_raw = data.get("spider_settings", {}) + proxy_raw = data.get("proxy_settings", {}) merge_raw = data.get("merge_settings", {}) link_check_raw = data.get("link_check", {}) email_push_raw = data.get("email_push", {}) @@ -133,6 +142,9 @@ def from_dict(cls, data: dict) -> "ApplicationConfig": json_url=str(spider_raw.get("json_url", "")).strip(), article_count=int(spider_raw.get("article_count", 5)), ), + proxy_settings=ProxySettings( + proxy_url=os.getenv("PROXY_URL") or str(proxy_raw.get("proxy_url", "")).strip(), + ), merge_settings=MergeSettings( enable=bool(merge_raw.get("enable", False)), remote_base_url=str(merge_raw.get("remote_base_url", "")).strip(), @@ -144,7 +156,6 @@ def from_dict(cls, data: dict) -> "ApplicationConfig": max_age_hours=int(link_check_raw.get("max_age_hours", 24)), timeout=int(link_check_raw.get("timeout", 15)), max_workers=int(link_check_raw.get("max_workers", 10)), - proxy_url=str(link_check_raw.get("proxy_url", "")).strip(), status_api_url=str(link_check_raw.get("status_api_url", "https://v2.xxapi.cn/api/status?url={url}")).strip(), enable_backlink_check=bool(link_check_raw.get("enable_backlink_check", False)), author_url=str(link_check_raw.get("author_url", "")).strip(), diff --git a/friend_circle_lite/application.py b/friend_circle_lite/application.py index 2498225eeaf..930f64828bb 100644 --- a/friend_circle_lite/application.py +++ b/friend_circle_lite/application.py @@ -17,6 +17,7 @@ marge_errors_from_json_url, ) from friend_circle_lite.app_config import ApplicationConfig, MailRuntime +from friend_circle_lite.config_printer import print_startup_config from friend_circle_lite.single_friend import get_latest_articles_from_link from friend_circle_lite.utils.github import extract_emails_from_issues from friend_circle_lite.utils.json import write_json @@ -31,6 +32,7 @@ def __init__(self, config: ApplicationConfig): def run(self) -> None: """Execute the enabled application features in a stable order.""" + print_startup_config(self.config) self.run_crawler_if_enabled() mail_runtime = self.prepare_mail_runtime() self.run_email_push_if_enabled(mail_runtime) @@ -54,6 +56,7 @@ def run_crawler_if_enabled(self) -> None: count=spider_settings.article_count, cache_file=self.config.runtime_paths.cache_file, link_check_config=self.config.link_check, + proxy_settings=self.config.proxy_settings, ) if crawl_result is None: logging.error("❌ 抓取流程失败，未生成任何输出文件") diff --git a/friend_circle_lite/config_printer.py b/friend_circle_lite/config_printer.py new file mode 100644 index 00000000000..e87c82a4214 --- /dev/null +++ b/friend_circle_lite/config_printer.py @@ -0,0 +1,54 @@ +"""Configuration printer for startup diagnostics.""" + +import logging + + +def print_startup_config(config): + """Print all configuration settings at startup for debugging.""" + logging.info("=" * 60) + logging.info("🚀 Friend-Circle-Lite 启动配置") + logging.info("=" * 60) + + # Spider settings + logging.info("📡 爬虫配置:") + logging.info(f" - 启用状态: {'✅ 已启用' if config.spider_settings.enable else '❌ 已禁用'}") + if config.spider_settings.enable: + logging.info(f" - 数据源: {config.spider_settings.json_url}") + logging.info(f" - 每站文章数: {config.spider_settings.article_count}") + + # Proxy settings + logging.info("🔀 代理配置:") + if config.proxy_settings.proxy_url: + logging.info(f" - 代理地址: {config.proxy_settings.proxy_url}") + logging.info(f" - 用途: 友链检测 + RSS 抓取") + else: + logging.info(f" - 代理状态: ❌ 未配置") + + # Link check settings + logging.info("🔍 友链检测配置:") + logging.info(f" - 启用状态: {'✅ 已启用' if config.link_check.enable else '❌ 已禁用'}") + if config.link_check.enable: + logging.info(f" - 缓存时间: {config.link_check.max_age_hours} 小时") + logging.info(f" - 超时时间: {config.link_check.timeout} 秒") + logging.info(f" - 并发数: {config.link_check.max_workers}") + logging.info(f" - 反链检测: {'✅ 已启用' if config.link_check.enable_backlink_check else '❌ 已禁用'}") + if config.link_check.enable_backlink_check: + logging.info(f" - 站点域名: {config.link_check.author_url}") + + # Merge settings + logging.info("🔗 数据合并配置:") + logging.info(f" - 启用状态: {'✅ 已启用' if config.merge_settings.enable else '❌ 已禁用'}") + if config.merge_settings.enable: + logging.info(f" - 远程数据源: {config.merge_settings.remote_base_url}") + logging.info(f" - 合并文章数据: {'✅ 是' if config.merge_settings.merge_article_data else '❌ 否'}") + logging.info(f" - 合并友链数据: {'✅ 是' if config.merge_settings.merge_link_check_data else '❌ 否'}") + + # Email push settings + logging.info("📧 邮件推送配置:") + logging.info(f" - 启用状态: {'✅ 已启用' if config.email_push.enable else '❌ 已禁用'}") + + # RSS subscribe settings + logging.info("📮 RSS 订阅配置:") + logging.info(f" - 启用状态: {'✅ 已启用' if config.rss_subscribe.enable else '❌ 已禁用'}") + + logging.info("=" * 60) diff --git a/friend_circle_lite/crawler_service.py b/friend_circle_lite/crawler_service.py index 87a427f7254..5a215920f96 100644 --- a/friend_circle_lite/crawler_service.py +++ b/friend_circle_lite/crawler_service.py @@ -15,7 +15,7 @@ import requests from friend_circle_lite import HEADERS_JSON, timeout -from friend_circle_lite.app_config import LinkCheckConfig +from friend_circle_lite.app_config import LinkCheckConfig, ProxySettings from friend_circle_lite.cache_store import FeedCacheStore, LinkCheckStore from friend_circle_lite.feed_service import FeedDiscoveryService, FeedParserService from friend_circle_lite.link_check_service import LinkCheckService @@ -89,9 +89,9 @@ def crawl(self, website: Website, count: int) -> CrawlResult: status = "active" if articles else "error" if not articles: if endpoint is None: - logging.warning(f"'{website.name}' 的博客 {website.url} 未找到有效 RSS。") + logging.warning(f"'{website.name}' 的博客 {website.url} 未找到有效 RSS ") else: - logging.warning(f"'{website.name}' 的 RSS {endpoint.url} 未解析出文章。") + logging.warning(f"'{website.name}' 的 RSS {endpoint.url} 未解析出文章 ") return CrawlResult( website=website, @@ -125,12 +125,14 @@ def __init__( specific_rss: list[dict] | None = None, cache_file: str | None = None, link_check_config: LinkCheckConfig | None = None, + proxy_settings: ProxySettings | None = None, ): self.json_url = json_url self.count = count self.specific_rss = specific_rss or [] self.cache_store = FeedCacheStore(cache_file) self.link_check_config = link_check_config or LinkCheckConfig(enable=False) + self.proxy_settings = proxy_settings or ProxySettings() self.link_check_store = LinkCheckStore(cache_file) def run(self) -> tuple[dict, list[list[str]]] | None: @@ -199,7 +201,7 @@ def run(self) -> tuple[dict, list[list[str]]] | None: return result, error_results, link_payload def _check_links(self, websites: list[Website]) -> list[LinkCheckRecord]: - service = LinkCheckService(config=self.link_check_config, store=self.link_check_store) + service = LinkCheckService(config=self.link_check_config, proxy_settings=self.proxy_settings, store=self.link_check_store) return service.check_websites(websites) @staticmethod diff --git a/friend_circle_lite/link_check_service.py b/friend_circle_lite/link_check_service.py index 626fd260dca..3319b2b0136 100644 --- a/friend_circle_lite/link_check_service.py +++ b/friend_circle_lite/link_check_service.py @@ -10,7 +10,7 @@ import requests -from friend_circle_lite.app_config import LinkCheckConfig +from friend_circle_lite.app_config import LinkCheckConfig, ProxySettings from friend_circle_lite.cache_store import LinkCheckStore from friend_circle_lite.models import LinkCheckRecord, LinkMethodStatus, Website @@ -37,8 +37,9 @@ class LinkCheckService: """Check friend homepage reachability and cache results.""" - def __init__(self, config: LinkCheckConfig, store: LinkCheckStore): + def __init__(self, config: LinkCheckConfig, proxy_settings: ProxySettings, store: LinkCheckStore): self.config = config + self.proxy_settings = proxy_settings self.store = store def check_websites(self, websites: list[Website]) -> list[LinkCheckRecord]: @@ -266,13 +267,13 @@ def _build_disabled_record(website: Website, checked_at: str) -> LinkCheckRecord ) def _build_proxy_url(self, url: str) -> str: - if not self.config.proxy_url: + if not self.proxy_settings.proxy_url: return "" - if "{}" in self.config.proxy_url: - return self.config.proxy_url.format(url) - if "{url}" in self.config.proxy_url: - return self.config.proxy_url.format(url=url) - return f"{self.config.proxy_url}{url}" + if "{}" in self.proxy_settings.proxy_url: + return self.proxy_settings.proxy_url.format(url) + if "{url}" in self.proxy_settings.proxy_url: + return self.proxy_settings.proxy_url.format(url=url) + return f"{self.proxy_settings.proxy_url}{url}" @staticmethod def _is_url(path: str) -> bool: diff --git a/readme.md b/readme.md index 3c18accc8ea..ea4b4fff658 100644 --- a/readme.md +++ b/readme.md @@ -228,7 +228,7 @@ `max_age_hours`：检测结果缓存时间，默认 24 小时，同一友链一天内不会重复检测； - `proxy_url`：可选代理前缀，适合使用 Cloudflare Worker 等方式转发检测； + `proxy_url`：可选代理前缀，适合使用 Cloudflare Worker 等方式转发检测；⚠️ 代理服务可能违反某些服务条款，请谨慎使用，建议仅用于调试。支持环境变量 `LINK_CHECK_PROXY_URL` 覆盖此配置（优先级更高）。 `status_api_url`：兜底状态码 API。注意 API 只能确认状态码，无法提供页面内容，所以仅 API 可达的友链会展示为可达，但不会参与 RSS 抓取； From 673ee3bbde6f8e6c5d69802370d0de35d9976eb4 Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Sun, 7 Jun 2026 00:25:44 +0800 Subject: [PATCH 18/30] =?UTF-8?q?=F0=9F=98=82=E5=AE=8C=E5=96=84=E9=80=BB?= =?UTF-8?q?=E8=BE=91=EF=BC=8C=E5=B0=86=E7=BD=91=E7=AB=99=E5=8F=AF=E8=BE=BE?= =?UTF-8?q?=E6=80=A7=E6=A3=80=E6=B5=8B=E6=B7=B1=E5=BA=A6=E8=9E=8D=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- friend_circle_lite/all_friends.py | 229 +-------- friend_circle_lite/app_config.py | 208 +------- friend_circle_lite/application.py | 211 +------- friend_circle_lite/cache_store.py | 486 +----------------- friend_circle_lite/cli.py | 211 ++++++++ friend_circle_lite/config/__init__.py | 3 + friend_circle_lite/config/models.py | 208 ++++++++ friend_circle_lite/config/printer.py | 54 ++ friend_circle_lite/config_printer.py | 56 +- friend_circle_lite/crawler/__init__.py | 4 + friend_circle_lite/crawler/feed_service.py | 256 +++++++++ friend_circle_lite/crawler/service.py | 375 ++++++++++++++ .../crawler/single_site_legacy.py | 141 +++++ friend_circle_lite/crawler_service.py | 369 +------------ friend_circle_lite/domain/__init__.py | 3 + friend_circle_lite/domain/models.py | 267 ++++++++++ friend_circle_lite/feed_service.py | 258 +--------- friend_circle_lite/link_check_service.py | 286 +---------- friend_circle_lite/link_checker/__init__.py | 3 + friend_circle_lite/link_checker/service.py | 288 +++++++++++ friend_circle_lite/models.py | 267 +--------- friend_circle_lite/notifications/__init__.py | 4 + friend_circle_lite/notifications/github.py | 39 ++ friend_circle_lite/notifications/mail.py | 255 +++++++++ friend_circle_lite/outputs/__init__.py | 3 + friend_circle_lite/outputs/legacy_api.py | 239 +++++++++ friend_circle_lite/single_friend.py | 141 +---- friend_circle_lite/storage/__init__.py | 3 + friend_circle_lite/storage/sqlite_store.py | 486 ++++++++++++++++++ friend_circle_lite/utils/cache.py | 4 +- friend_circle_lite/utils/config.py | 2 +- friend_circle_lite/utils/github.py | 41 +- friend_circle_lite/utils/mail.py | 257 +-------- run.py | 2 +- tests/__init__.py | 1 + tests/test_refactor_contracts.py | 155 ++++++ 36 files changed, 3043 insertions(+), 2772 deletions(-) create mode 100644 friend_circle_lite/cli.py create mode 100644 friend_circle_lite/config/__init__.py create mode 100644 friend_circle_lite/config/models.py create mode 100644 friend_circle_lite/config/printer.py create mode 100644 friend_circle_lite/crawler/__init__.py create mode 100644 friend_circle_lite/crawler/feed_service.py create mode 100644 friend_circle_lite/crawler/service.py create mode 100644 friend_circle_lite/crawler/single_site_legacy.py create mode 100644 friend_circle_lite/domain/__init__.py create mode 100644 friend_circle_lite/domain/models.py create mode 100644 friend_circle_lite/link_checker/__init__.py create mode 100644 friend_circle_lite/link_checker/service.py create mode 100644 friend_circle_lite/notifications/__init__.py create mode 100644 friend_circle_lite/notifications/github.py create mode 100644 friend_circle_lite/notifications/mail.py create mode 100644 friend_circle_lite/outputs/__init__.py create mode 100644 friend_circle_lite/outputs/legacy_api.py create mode 100644 friend_circle_lite/storage/__init__.py create mode 100644 friend_circle_lite/storage/sqlite_store.py create mode 100644 tests/__init__.py create mode 100644 tests/test_refactor_contracts.py diff --git a/friend_circle_lite/all_friends.py b/friend_circle_lite/all_friends.py index aabd71ca1d3..ae84c05c2b0 100644 --- a/friend_circle_lite/all_friends.py +++ b/friend_circle_lite/all_friends.py @@ -1,229 +1,6 @@ -"""Legacy-compatible crawl entrypoints. +"""Backward-compatible crawl output API. -The internal implementation is now delegated to `crawler_service`, but these -functions keep the existing public API stable for `run.py` and external users. +New code should import from `friend_circle_lite.outputs.legacy_api`. """ -import logging - -import requests - -from friend_circle_lite import HEADERS_JSON, timeout -from friend_circle_lite.crawler_service import ( - FriendCircleCrawler, - limit_large_dataset as _limit_large_dataset, - sort_articles_by_time as _sort_articles_by_time, -) - -def fetch_and_process_data( - json_url: str, - specific_RSS: list = None, - count: int = 5, - cache_file: str = None, - link_check_config=None, - proxy_settings=None, -): - """Legacy wrapper around the new crawler orchestration service.""" - return FriendCircleCrawler( - json_url=json_url, - count=count, - specific_rss=specific_RSS, - cache_file=cache_file, - link_check_config=link_check_config, - proxy_settings=proxy_settings, - ).run() - -def sort_articles_by_time(data, future_tolerance_days=2): - """Legacy wrapper around the refactored sort helper.""" - return _sort_articles_by_time(data, future_tolerance_days=future_tolerance_days) - -def marge_data_from_json_url(data, marge_json_url): - """ - 从另一个 JSON 文件中获取数据并合并到原数据中。 - - 参数： - data (dict): 包含文章信息的字典 - marge_json_url (str): 包含另一个文章信息的 JSON 文件的 URL。 - - 返回： - dict: 合并后的文章信息字典，已去重处理 - """ - try: - response = requests.get(marge_json_url, headers=HEADERS_JSON, timeout=timeout) - marge_data = response.json() - except Exception as e: - logging.error(f"无法获取链接：{marge_json_url} ，出现的问题为：{e}", exc_info=True) - return data - - if 'article_data' in marge_data: - logging.info(f"开始合并文章数据，原数据共有 {len(data['article_data'])} 篇文章，第三方数据共有 {len(marge_data['article_data'])} 篇文章") - data['article_data'].extend(marge_data['article_data']) - data['article_data'] = list({v['link']:v for v in data['article_data']}.values()) - logging.info(f"合并文章数据完成，现在共有 {len(data['article_data'])} 篇文章") - return data - - -def merge_link_data_from_json_url(link_data, merge_json_url): - """ - 从另一个 link.json 文件中获取友链可达性数据并智能合并。 - - 合并策略： - - 可达性优先级：direct > proxy > api > none - - 延迟取最优（最小值） - - 反链取并集（任一为 true 则为 true） - - 失败次数取最小值 - - 参数： - link_data (dict): 本地友链数据，包含 statistical_data 和 link_data - merge_json_url (str): 远程 link.json 的 URL - - 返回： - dict: 合并后的友链数据 - """ - try: - response = requests.get(merge_json_url, headers=HEADERS_JSON, timeout=timeout) - remote_data = response.json() - except Exception as e: - logging.warning(f"无法获取友链数据：{merge_json_url} ，跳过友链数据合并。错误：{e}") - return link_data - - if 'link_data' not in remote_data: - logging.warning(f"远程数据不包含 link_data 字段，跳过友链数据合并") - return link_data - - local_links = link_data.get('link_data', []) - remote_links = remote_data.get('link_data', []) - - logging.info(f"开始合并友链数据，本地 {len(local_links)} 条，远程 {len(remote_links)} 条") - - # 按 URL 建立索引 - link_map = {link['link']: link for link in local_links} - - for remote_link in remote_links: - url = remote_link['link'] - if url not in link_map: - # 新友链，直接添加 - link_map[url] = remote_link - else: - # 已存在，智能合并 - local_link = link_map[url] - link_map[url] = _merge_single_link(local_link, remote_link) - - merged_links = list(link_map.values()) - logging.info(f"合并友链数据完成，共有 {len(merged_links)} 条友链") - - # 重新计算统计数据 - merged_stats = _recalculate_link_statistics(merged_links) - - return { - 'statistical_data': merged_stats, - 'link_data': merged_links, - } - - -def _merge_single_link(local, remote): - """ - 合并单条友链数据，优先选择更好的检测结果。 - - 优先级： - 1. 可达性：direct > proxy > api > none - 2. 延迟：取最小值 - 3. 反链：任一为 true 则为 true - 4. 失败次数：取最小值 - """ - method_priority = {'direct': 4, 'proxy': 3, 'api': 2, 'disabled': 1, 'none': 0, '': 0} - - local_priority = method_priority.get(local.get('method', ''), 0) - remote_priority = method_priority.get(remote.get('method', ''), 0) - - # 选择优先级更高的作为基础 - if remote_priority > local_priority: - base = remote.copy() - alt = local - elif remote_priority < local_priority: - base = local.copy() - alt = remote - else: - # 优先级相同，选择延迟更低的 - local_latency = local.get('latency', 999) - remote_latency = remote.get('latency', 999) - if remote_latency >= 0 and (local_latency < 0 or remote_latency < local_latency): - base = remote.copy() - alt = local - else: - base = local.copy() - alt = remote - - # 反链取并集 - local_backlink = local.get('has_backlink') - remote_backlink = remote.get('has_backlink') - if local_backlink is True or remote_backlink is True: - base['has_backlink'] = True - elif local_backlink is False and remote_backlink is False: - base['has_backlink'] = False - # 否则保持 base 的值 - - # 失败次数取最小值 - local_fail = local.get('fail_count', 0) - remote_fail = remote.get('fail_count', 0) - base['fail_count'] = min(local_fail, remote_fail) - - # 检测时间取最新 - local_checked = local.get('checked_at', '') - remote_checked = remote.get('checked_at', '') - if remote_checked > local_checked: - base['checked_at'] = remote_checked - - return base - - -def _recalculate_link_statistics(links): - """重新计算合并后的友链统计数据。""" - reachable = [link for link in links if link.get('reachable')] - crawl_allowed = [link for link in links if link.get('crawlable')] - api_only = [link for link in links if link.get('method') == 'api'] - has_backlink = [link for link in links if link.get('has_backlink') is True] - checked_times = [link.get('checked_at', '') for link in links if link.get('checked_at')] - - return { - 'link_total_num': len(links), - 'link_reachable_num': len(reachable), - 'link_unreachable_num': len(links) - len(reachable), - 'crawl_allowed_num': len(crawl_allowed), - 'api_only_num': len(api_only), - 'has_author_link_num': len(has_backlink), - 'link_last_checked_time': max(checked_times) if checked_times else '', - } - - -def marge_errors_from_json_url(errors, marge_json_url): - """ - 从另一个网络 JSON 文件中获取错误信息并遍历，删除在errors中， - 不存在于marge_errors中的友链信息。 - - 参数： - errors (list): 包含错误信息的列表 - marge_json_url (str): 包含另一个错误信息的 JSON 文件的 URL。 - - 返回： - list: 合并后的错误信息列表 - """ - try: - response = requests.get(marge_json_url, timeout=10) # 设置请求超时时间 - marge_errors = response.json() - except Exception as e: - logging.error(f"无法获取链接：{marge_json_url} ，出现的问题为：{e}", exc_info=True) - return errors - - # 提取 marge_errors 中的 URL - marge_urls = {item[1] for item in marge_errors} - - # 使用过滤器保留 errors 中在 marge_errors 中出现的 URL - filtered_errors = [error for error in errors if error[1] in marge_urls] - - logging.info(f"合并错误信息完成，合并后共有 {len(filtered_errors)} 位朋友") - return filtered_errors - -def deal_with_large_data(result, future_tolerance_days=2): - """Legacy wrapper around the refactored dataset trimming helper.""" - return _limit_large_dataset(result, future_tolerance_days=future_tolerance_days) +from friend_circle_lite.outputs.legacy_api import * # noqa: F401,F403 diff --git a/friend_circle_lite/app_config.py b/friend_circle_lite/app_config.py index 7635506cc74..d8811970127 100644 --- a/friend_circle_lite/app_config.py +++ b/friend_circle_lite/app_config.py @@ -1,208 +1,6 @@ -"""Application configuration models. +"""Backward-compatible configuration exports. -This module converts the raw YAML structure into typed configuration objects so -that the rest of the application can depend on explicit fields instead of a -loosely typed nested dictionary. - -The external YAML keys are preserved for backward compatibility. Internally, -snake_case names are used consistently. +New code should import from `friend_circle_lite.config.models`. """ -from __future__ import annotations - -import os -from dataclasses import dataclass, field - - -DEFAULT_CACHE_FILE = "./temp/cache.sqlite3" -DEFAULT_ALL_JSON = "./all.json" -DEFAULT_ERRORS_JSON = "./errors.json" -DEFAULT_LINK_JSON = "./link.json" - - -@dataclass(slots=True) -class MergeSettings: - """Options for merging local crawl results with remote data sources.""" - - enable: bool = False - remote_base_url: str = "" - merge_article_data: bool = True - merge_link_check_data: bool = True - - -@dataclass(slots=True) -class ProxySettings: - """Proxy configuration for both link checking and RSS crawling.""" - - proxy_url: str = "" - - -@dataclass(slots=True) -class SpiderSettings: - """Crawler settings controlling source list and output density.""" - - enable: bool = True - json_url: str = "" - article_count: int = 5 - - -@dataclass(slots=True) -class LinkCheckConfig: - """Settings for friend link reachability checks.""" - - enable: bool = True - max_age_hours: int = 24 - timeout: int = 15 - max_workers: int = 10 - status_api_url: str = "https://v2.xxapi.cn/api/status?url={url}" - enable_backlink_check: bool = False - author_url: str = "" - - -@dataclass(slots=True) -class EmailPushConfig: - """Reserved configuration for the not-yet-implemented email push feature.""" - - enable: bool = False - to_email: str = "" - subject: str = "" - body_template: str = "" - - -@dataclass(slots=True) -class WebsiteInfo: - """Display metadata for outbound notifications.""" - - title: str = "" - - -@dataclass(slots=True) -class RssSubscribeConfig: - """Configuration for GitHub issue based email subscriptions.""" - - enable: bool = False - github_username: str = "" - github_repo: str = "" - your_blog_url: str = "" - email_template: str = "" - website_info: WebsiteInfo = field(default_factory=WebsiteInfo) - - -@dataclass(slots=True) -class SmtpConfig: - """SMTP connection settings used by all mail sending features.""" - - email: str = "" - server: str = "" - port: int = 0 - use_tls: bool = True - - -@dataclass(slots=True) -class RuntimePaths: - """Filesystem locations used by the runtime.""" - - cache_file: str = DEFAULT_CACHE_FILE - all_json_file: str = DEFAULT_ALL_JSON - errors_json_file: str = DEFAULT_ERRORS_JSON - link_json_file: str = DEFAULT_LINK_JSON - - -@dataclass(slots=True) -class ApplicationConfig: - """Root application configuration assembled from the YAML file.""" - - spider_settings: SpiderSettings - proxy_settings: ProxySettings - merge_settings: MergeSettings - link_check: LinkCheckConfig - email_push: EmailPushConfig - rss_subscribe: RssSubscribeConfig - smtp: SmtpConfig - specific_rss: list[dict] - runtime_paths: RuntimePaths = field(default_factory=RuntimePaths) - future_article_tolerance_days: int = 2 - - @classmethod - def from_dict(cls, data: dict) -> "ApplicationConfig": - """Create a typed config object from the raw YAML dictionary.""" - spider_raw = data.get("spider_settings", {}) - proxy_raw = data.get("proxy_settings", {}) - merge_raw = data.get("merge_settings", {}) - link_check_raw = data.get("link_check", {}) - email_push_raw = data.get("email_push", {}) - rss_subscribe_raw = data.get("rss_subscribe", {}) - website_info_raw = rss_subscribe_raw.get("website_info", {}) - smtp_raw = data.get("smtp", {}) - runtime_raw = data.get("runtime_paths", {}) - - return cls( - spider_settings=SpiderSettings( - enable=bool(spider_raw.get("enable", True)), - json_url=str(spider_raw.get("json_url", "")).strip(), - article_count=int(spider_raw.get("article_count", 5)), - ), - proxy_settings=ProxySettings( - proxy_url=os.getenv("PROXY_URL") or str(proxy_raw.get("proxy_url", "")).strip(), - ), - merge_settings=MergeSettings( - enable=bool(merge_raw.get("enable", False)), - remote_base_url=str(merge_raw.get("remote_base_url", "")).strip(), - merge_article_data=bool(merge_raw.get("merge_article_data", True)), - merge_link_check_data=bool(merge_raw.get("merge_link_check_data", True)), - ), - link_check=LinkCheckConfig( - enable=bool(link_check_raw.get("enable", True)), - max_age_hours=int(link_check_raw.get("max_age_hours", 24)), - timeout=int(link_check_raw.get("timeout", 15)), - max_workers=int(link_check_raw.get("max_workers", 10)), - status_api_url=str(link_check_raw.get("status_api_url", "https://v2.xxapi.cn/api/status?url={url}")).strip(), - enable_backlink_check=bool(link_check_raw.get("enable_backlink_check", False)), - author_url=str(link_check_raw.get("author_url", "")).strip(), - ), - email_push=EmailPushConfig( - enable=bool(email_push_raw.get("enable", False)), - to_email=str(email_push_raw.get("to_email", "")).strip(), - subject=str(email_push_raw.get("subject", "")).strip(), - body_template=str(email_push_raw.get("body_template", "")).strip(), - ), - rss_subscribe=RssSubscribeConfig( - enable=bool(rss_subscribe_raw.get("enable", False)), - github_username=str(rss_subscribe_raw.get("github_username", "")).strip(), - github_repo=str(rss_subscribe_raw.get("github_repo", "")).strip(), - your_blog_url=str(rss_subscribe_raw.get("your_blog_url", "")).strip(), - email_template=str(rss_subscribe_raw.get("email_template", "")).strip(), - website_info=WebsiteInfo( - title=str(website_info_raw.get("title", "")).strip(), - ), - ), - smtp=SmtpConfig( - email=str(smtp_raw.get("email", "")).strip(), - server=str(smtp_raw.get("server", "")).strip(), - port=int(smtp_raw.get("port", 0) or 0), - use_tls=bool(smtp_raw.get("use_tls", True)), - ), - specific_rss=list(data.get("specific_RSS", []) or []), - runtime_paths=RuntimePaths( - cache_file=str(runtime_raw.get("cache_file", DEFAULT_CACHE_FILE)).strip() or DEFAULT_CACHE_FILE, - all_json_file=str(runtime_raw.get("all_json_file", DEFAULT_ALL_JSON)).strip() or DEFAULT_ALL_JSON, - errors_json_file=str(runtime_raw.get("errors_json_file", DEFAULT_ERRORS_JSON)).strip() or DEFAULT_ERRORS_JSON, - link_json_file=str(runtime_raw.get("link_json_file", DEFAULT_LINK_JSON)).strip() or DEFAULT_LINK_JSON, - ), - ) - - -@dataclass(slots=True) -class MailRuntime: - """Runtime SMTP credentials resolved from configuration and environment.""" - - sender_email: str - smtp_server: str - port: int - password: str - use_tls: bool - - @property - def is_ready(self) -> bool: - """Whether enough information is available to send email.""" - return bool(self.sender_email and self.smtp_server and self.port and self.password) +from friend_circle_lite.config.models import * # noqa: F401,F403 diff --git a/friend_circle_lite/application.py b/friend_circle_lite/application.py index 930f64828bb..38a630b36a2 100644 --- a/friend_circle_lite/application.py +++ b/friend_circle_lite/application.py @@ -1,211 +1,6 @@ -"""Top-level application orchestration. +"""Backward-compatible application entrypoint exports. -This module keeps the main script very small by moving the end-to-end workflow -into focused orchestration methods. +New code should import from `friend_circle_lite.cli`. """ -from __future__ import annotations - -import logging -import os -import sys - -from friend_circle_lite.all_friends import ( - deal_with_large_data, - fetch_and_process_data, - marge_data_from_json_url, - marge_errors_from_json_url, -) -from friend_circle_lite.app_config import ApplicationConfig, MailRuntime -from friend_circle_lite.config_printer import print_startup_config -from friend_circle_lite.single_friend import get_latest_articles_from_link -from friend_circle_lite.utils.github import extract_emails_from_issues -from friend_circle_lite.utils.json import write_json -from friend_circle_lite.utils.mail import send_emails - - -class FriendCircleLiteApplication: - """Application service coordinating crawl and notification workflows.""" - - def __init__(self, config: ApplicationConfig): - self.config = config - - def run(self) -> None: - """Execute the enabled application features in a stable order.""" - print_startup_config(self.config) - self.run_crawler_if_enabled() - mail_runtime = self.prepare_mail_runtime() - self.run_email_push_if_enabled(mail_runtime) - self.run_rss_subscription_if_enabled(mail_runtime) - - def run_crawler_if_enabled(self) -> None: - """Run the article crawl and persist public output files when enabled.""" - spider_settings = self.config.spider_settings - if not spider_settings.enable: - logging.info("⏭️ 爬虫未启用，跳过抓取流程") - return - - logging.info("✅ 爬虫已启用") - logging.info( - f"📥 正在从 {spider_settings.json_url} 获取数据，每个博客获取 {spider_settings.article_count} 篇文章" - ) - - crawl_result = fetch_and_process_data( - json_url=spider_settings.json_url, - specific_RSS=self.config.specific_rss, - count=spider_settings.article_count, - cache_file=self.config.runtime_paths.cache_file, - link_check_config=self.config.link_check, - proxy_settings=self.config.proxy_settings, - ) - if crawl_result is None: - logging.error("❌ 抓取流程失败，未生成任何输出文件") - return - - result, lost_friends, link_payload = crawl_result - result, lost_friends, link_payload = self._merge_remote_results_if_enabled(result, lost_friends, link_payload) - - article_count = len(result.get("article_data", [])) - logging.info(f"📦 数据获取完毕，共有 {article_count} 篇文章，正在处理数据") - - result = deal_with_large_data( - result, - future_tolerance_days=self.config.future_article_tolerance_days, - ) - write_json(self.config.runtime_paths.all_json_file, result) - write_json(self.config.runtime_paths.errors_json_file, lost_friends) - write_json(self.config.runtime_paths.link_json_file, link_payload) - - def prepare_mail_runtime(self) -> MailRuntime: - """Build SMTP runtime credentials from config and environment variables.""" - if not (self.config.email_push.enable or self.config.rss_subscribe.enable): - return MailRuntime(sender_email="", smtp_server="", port=0, password="", use_tls=False) - - logging.info("📨 推送功能已启用，正在准备中...") - smtp_conf = self.config.smtp - mail_runtime = MailRuntime( - sender_email=smtp_conf.email, - smtp_server=smtp_conf.server, - port=smtp_conf.port, - password=os.getenv("SMTP_PWD", ""), - use_tls=smtp_conf.use_tls, - ) - - logging.info(f"📡 SMTP 服务器：{mail_runtime.smtp_server}:{mail_runtime.port}") - if mail_runtime.is_ready: - logging.info(f"🔐 密码(部分)：{mail_runtime.password[:3]}*****") - else: - logging.error("❌ SMTP 信息不完整或环境变量 SMTP_PWD 未设置，无法发送邮件") - return mail_runtime - - def run_email_push_if_enabled(self, mail_runtime: MailRuntime) -> None: - """Keep the reserved email push entrypoint behavior unchanged.""" - if self.config.email_push.enable and mail_runtime.is_ready: - logging.info("📧 邮件推送已启用") - logging.info("⚠️ 抱歉，目前尚未实现邮件推送功能") - - def run_rss_subscription_if_enabled(self, mail_runtime: MailRuntime) -> None: - """Send subscription emails for newly discovered posts when enabled.""" - if not self.config.rss_subscribe.enable: - return - if not mail_runtime.is_ready: - logging.info("⏭️ RSS 订阅推送未执行，因为 SMTP 尚未就绪") - return - - logging.info("📰 RSS 订阅推送已启用") - github_username, github_repo = self._resolve_github_repo() - logging.info(f"👤 GitHub 用户名：{github_username}") - logging.info(f"📁 GitHub 仓库：{github_repo}") - - latest_articles = get_latest_articles_from_link( - url=self.config.rss_subscribe.your_blog_url, - count=10, - last_articles_path=self.config.runtime_paths.cache_file, - ) - if not latest_articles: - logging.info("📭 无新文章，无需推送") - return - - logging.info(f"🆕 获取到的最新文章：{latest_articles}") - email_list = self._load_subscriber_emails(github_username, github_repo) - if not email_list: - logging.info("⚠️ 无订阅邮箱，请检查格式或是否有订阅者") - sys.exit(0) - - logging.info(f"📬 获取到邮箱列表：{email_list}") - for article in latest_articles: - template_data = self._build_email_template_data(article, github_username, github_repo) - send_emails( - emails=email_list["emails"], - sender_email=mail_runtime.sender_email, - smtp_server=mail_runtime.smtp_server, - port=mail_runtime.port, - password=mail_runtime.password, - subject=f"{self.config.rss_subscribe.website_info.title} の最新文章：{article['title']}", - body=self._build_plaintext_mail_body(article), - template_path=self.config.rss_subscribe.email_template, - template_data=template_data, - use_tls=mail_runtime.use_tls, - ) - - def _merge_remote_results_if_enabled( - self, result: dict, lost_friends: list[list[str]], link_payload: dict - ) -> tuple[dict, list[list[str]], dict]: - """Merge remote outputs when the merge option is enabled.""" - merge_settings = self.config.merge_settings - if not merge_settings.enable: - return result, lost_friends, link_payload - - remote_url = merge_settings.remote_base_url - logging.info(f"🔀 合并功能开启，从 {remote_url} 获取外部数据") - - if merge_settings.merge_article_data: - result = marge_data_from_json_url(result, f"{remote_url}/all.json") - lost_friends = marge_errors_from_json_url(lost_friends, f"{remote_url}/errors.json") - - if merge_settings.merge_link_check_data: - from friend_circle_lite.all_friends import merge_link_data_from_json_url - link_payload = merge_link_data_from_json_url(link_payload, f"{remote_url}/link.json") - - return result, lost_friends, link_payload - - def _resolve_github_repo(self) -> tuple[str, str]: - """Resolve repository coordinates from env override or config.""" - fcl_repo = os.getenv("FCL_REPO") - if fcl_repo: - return tuple(fcl_repo.split("/", 1)) - return self.config.rss_subscribe.github_username, self.config.rss_subscribe.github_repo - - @staticmethod - def _load_subscriber_emails(github_username: str, github_repo: str) -> dict | None: - """Load subscriber emails from GitHub closed issues.""" - github_api_url = ( - f"https://api.github.com/repos/{github_username}/{github_repo}/issues" - f"?state=closed&label=subscribed&per_page=200" - ) - logging.info(f"🔎 正在从 GitHub 获取订阅邮箱：{github_api_url}") - return extract_emails_from_issues(github_api_url) - - def _build_email_template_data(self, article: dict, github_username: str, github_repo: str) -> dict[str, str]: - """Assemble template variables for one outbound notification email.""" - return { - "title": article["title"], - "summary": article["summary"], - "published": article["published"], - "link": article["link"], - "website_title": self.config.rss_subscribe.website_info.title, - "github_issue_url": ( - f"https://github.com/{github_username}/{github_repo}" - "/issues?q=is%3Aissue+is%3Aclosed" - ), - } - - @staticmethod - def _build_plaintext_mail_body(article: dict) -> str: - """Build the plain-text fallback body for one notification email.""" - return ( - f"📄 文章标题：{article['title']}\n" - f"🔗 链接：{article['link']}\n" - f"📝 简介：{article['summary']}\n" - f"🕒 发布时间：{article['published']}" - ) +from friend_circle_lite.cli import FriendCircleLiteApplication # noqa: F401 diff --git a/friend_circle_lite/cache_store.py b/friend_circle_lite/cache_store.py index cfa1c997f93..2cfdded6c2d 100644 --- a/friend_circle_lite/cache_store.py +++ b/friend_circle_lite/cache_store.py @@ -1,486 +1,6 @@ -"""Persistent RSS cache and article tracking storage. +"""Backward-compatible storage exports. -SQLite is used for both feed cache and article tracking because it is more robust -than hand-edited text formats for internal state: - -- schema is explicit and stable; -- writes are transactional; -- corruption risk from accidental manual edits is lower; -- Python ships with `sqlite3`, so no extra dependency is required. - -For smooth upgrades, this store can also migrate legacy cache data from the old -JSON cache file and the intermediate YAML cache file if they exist. +New code should import from `friend_circle_lite.storage.sqlite_store`. """ -from __future__ import annotations - -import json -import logging -import sqlite3 -from datetime import datetime -from pathlib import Path - -import yaml - -from friend_circle_lite.models import Article, CacheRecord, LinkCheckRecord, LinkMethodStatus - - -class FeedCacheStore: - """Persist and load discovered RSS endpoints using SQLite.""" - - def __init__(self, cache_path: str | Path | None): - self.cache_path = Path(cache_path) if cache_path else None - - def load_records(self) -> list[CacheRecord]: - """Load cache records from SQLite, migrating legacy formats if needed.""" - if not self.cache_path: - return [] - - if self.cache_path.exists(): - return self._load_from_sqlite() - - migrated_records = self._load_legacy_records() - if migrated_records: - if self.save_records(migrated_records): - logging.info(f"已从旧格式迁移 {len(migrated_records)} 条 RSS 缓存到 SQLite") - return migrated_records - - logging.info(f"RSS 缓存文件不存在，将在首次抓取后自动创建") - return [] - - def save_records(self, records: list[CacheRecord]) -> bool: - """Persist cache records to the SQLite database.""" - if not self.cache_path: - return True - - try: - self.cache_path.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(self.cache_path) as connection: - self._ensure_schema(connection) - connection.execute("DELETE FROM feed_cache") - connection.executemany( - "INSERT INTO feed_cache(name, url, source) VALUES (?, ?, ?)", - [(record.name, record.url, record.source) for record in sorted(records, key=lambda item: item.name)], - ) - connection.commit() - logging.info(f"RSS 缓存已保存（{len(records)} 条）") - return True - except Exception as exc: - logging.error(f"保存 RSS 缓存失败: {exc}") - return False - - def _load_from_sqlite(self) -> list[CacheRecord]: - """Load records from the current SQLite cache file.""" - try: - with sqlite3.connect(self.cache_path) as connection: - self._ensure_schema(connection) - rows = connection.execute( - "SELECT name, url, source FROM feed_cache ORDER BY name" - ).fetchall() - except Exception as exc: - logging.warning(f"读取 RSS 缓存失败: {exc}") - return [] - - return [ - CacheRecord(name=name, url=url, source=source or "cache") - for name, url, source in rows - if name and url - ] - - @staticmethod - def _ensure_schema(connection: sqlite3.Connection) -> None: - """Create the cache table when it does not exist yet.""" - connection.execute( - """ - CREATE TABLE IF NOT EXISTS feed_cache ( - name TEXT PRIMARY KEY, - url TEXT NOT NULL, - source TEXT NOT NULL DEFAULT 'cache' - ) - """ - ) - - def _load_legacy_records(self) -> list[CacheRecord]: - """Read old cache formats for seamless upgrades.""" - json_records = self._load_legacy_json_cache() - if json_records: - return json_records - - yaml_records = self._load_legacy_yaml_cache() - if yaml_records: - return yaml_records - - return [] - - def _load_legacy_json_cache(self) -> list[CacheRecord]: - """Read the previous JSON cache file format.""" - if not self.cache_path: - return [] - - legacy_path = self.cache_path.with_name("cache.json") - if not legacy_path.exists(): - return [] - - try: - with open(legacy_path, "r", encoding="utf-8") as file: - payload = json.load(file) - except Exception as exc: - logging.warning(f"读取旧 JSON 缓存失败: {exc}") - return [] - - if not isinstance(payload, list): - return [] - - return self._normalize_legacy_items(payload) - - def _load_legacy_yaml_cache(self) -> list[CacheRecord]: - """Read the temporary YAML cache format used during refactoring.""" - if not self.cache_path: - return [] - - legacy_path = self.cache_path.with_name("feed_cache.yaml") - if not legacy_path.exists(): - return [] - - try: - with open(legacy_path, "r", encoding="utf-8") as file: - payload = yaml.safe_load(file) or {} - except Exception as exc: - logging.warning(f"读取旧 YAML 缓存失败: {exc}") - return [] - - items = payload.get("feeds", []) if isinstance(payload, dict) else [] - return self._normalize_legacy_items(items) - - @staticmethod - def _normalize_legacy_items(items: list[object]) -> list[CacheRecord]: - """Normalize legacy cache items into typed cache records.""" - records: list[CacheRecord] = [] - for item in items: - if not isinstance(item, dict): - continue - name = str(item.get("name", "")).strip() - url = str(item.get("url", "")).strip() - source = str(item.get("source", "cache")).strip() or "cache" - if name and url: - records.append(CacheRecord(name=name, url=url, source=source)) - return records - - -class ArticleTrackingStore: - """Persist and load article tracking data using SQLite.""" - - def __init__(self, storage_path: str | Path | None, max_tracked_articles: int = 10): - self.storage_path = Path(storage_path) if storage_path else None - self.max_tracked_articles = max_tracked_articles - - def load_articles(self) -> list[Article]: - """Load tracked articles from SQLite, migrating from legacy JSON if needed.""" - if not self.storage_path: - return [] - - if self.storage_path.exists(): - return self._load_from_sqlite() - - # Try to migrate from legacy JSON format - migrated_articles = self._load_legacy_json() - if migrated_articles: - if self.save_articles(migrated_articles): - logging.info(f"已从旧 JSON 格式迁移 {len(migrated_articles)} 篇文章记录到 SQLite") - return migrated_articles - - logging.info(f"文章追踪数据不存在，这是首次运行") - return [] - - def save_articles(self, articles: list[Article]) -> bool: - """Persist articles to SQLite, keeping only the most recent max_tracked_articles.""" - if not self.storage_path: - return True - - try: - # Sort by date and keep only the most recent articles - valid_articles = [article for article in articles if article.published] - valid_articles.sort( - key=lambda item: datetime.strptime(item.published, "%Y-%m-%d %H:%M"), - reverse=True - ) - articles_to_save = valid_articles[:self.max_tracked_articles] - - self.storage_path.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(self.storage_path) as connection: - self._ensure_schema(connection) - connection.execute("DELETE FROM article_tracking") - connection.executemany( - """INSERT INTO article_tracking(title, author, link, published, summary, content) - VALUES (?, ?, ?, ?, ?, ?)""", - [ - ( - article.title, - article.author, - article.link, - article.published, - article.summary, - article.content, - ) - for article in articles_to_save - ], - ) - connection.commit() - return True - except Exception as exc: - logging.error(f"保存文章追踪数据失败: {exc}") - return False - - def _load_from_sqlite(self) -> list[Article]: - """Load articles from the SQLite database.""" - try: - with sqlite3.connect(self.storage_path) as connection: - self._ensure_schema(connection) - rows = connection.execute( - """SELECT title, author, link, published, summary, content - FROM article_tracking - ORDER BY published DESC""" - ).fetchall() - except Exception as exc: - logging.warning(f"读取文章追踪数据失败: {exc}") - return [] - - return [ - Article( - title=title or "", - author=author or "", - link=link or "", - published=published or "", - summary=summary or "", - content=content or "", - ) - for title, author, link, published, summary, content in rows - ] - - @staticmethod - def _ensure_schema(connection: sqlite3.Connection) -> None: - """Create the article tracking table when it does not exist yet.""" - connection.execute( - """ - CREATE TABLE IF NOT EXISTS article_tracking ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - title TEXT NOT NULL, - author TEXT NOT NULL, - link TEXT NOT NULL, - published TEXT NOT NULL, - summary TEXT, - content TEXT - ) - """ - ) - - def _load_legacy_json(self) -> list[Article]: - """Read the old JSON format for seamless upgrades.""" - if not self.storage_path: - return [] - - legacy_path = self.storage_path.with_name("newest_posts.json") - if not legacy_path.exists(): - return [] - - try: - with open(legacy_path, "r", encoding="utf-8") as file: - payload = json.load(file) - except Exception as exc: - logging.warning(f"读取旧 JSON 文章追踪文件失败: {exc}") - return [] - - articles_data = payload.get("articles", []) if isinstance(payload, dict) else [] - articles: list[Article] = [] - for item in articles_data: - if not isinstance(item, dict): - continue - articles.append( - Article( - title=item.get("title", ""), - author=item.get("author", ""), - link=item.get("link", ""), - published=item.get("published", ""), - summary=item.get("summary", ""), - content=item.get("content", ""), - ) - ) - return articles - - -class LinkCheckStore: - """Persist friend link reachability checks using SQLite.""" - - def __init__(self, cache_path: str | Path | None): - self.cache_path = Path(cache_path) if cache_path else None - - def load_records(self, urls: list[str] | None = None) -> dict[str, LinkCheckRecord]: - if not self.cache_path or not self.cache_path.exists(): - return {} - - try: - with sqlite3.connect(self.cache_path) as connection: - self._ensure_schema(connection) - rows = connection.execute( - """ - SELECT url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, - best_method, best_latency, fail_count, backlink_checked, has_author_link, - rss_crawl_reason, direct_success, direct_status_code, direct_latency, - proxy_success, proxy_status_code, proxy_latency, api_success, - api_status_code, api_latency - FROM link_check_state - """ - ).fetchall() - except Exception as exc: - logging.warning(f"读取友链检测缓存失败: {exc}") - return {} - - allowed_urls = set(urls or []) - records: dict[str, LinkCheckRecord] = {} - for row in rows: - ( - url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, - best_method, best_latency, fail_count, backlink_checked, has_author_link, - rss_crawl_reason, direct_success, direct_status_code, direct_latency, - proxy_success, proxy_status_code, proxy_latency, api_success, - api_status_code, api_latency, - ) = row - if allowed_urls and url not in allowed_urls: - continue - records[url] = LinkCheckRecord( - name=name or "", - url=url or "", - avatar=avatar or "", - linkpage=linkpage or "", - checked_at=checked_at or "", - reachable=bool(reachable), - crawl_allowed=bool(crawl_allowed), - best_method=best_method or "none", - best_latency=best_latency if best_latency is not None else -1, - fail_count=fail_count or 0, - backlink_checked=bool(backlink_checked), - has_author_link=bool(has_author_link), - rss_crawl_reason=rss_crawl_reason or "blocked_unreachable", - direct=LinkMethodStatus(bool(direct_success), direct_status_code, direct_latency if direct_latency is not None else -1), - proxy=LinkMethodStatus(bool(proxy_success), proxy_status_code, proxy_latency if proxy_latency is not None else -1), - api=LinkMethodStatus(bool(api_success), api_status_code, api_latency if api_latency is not None else -1), - ) - return records - - def save_records(self, records: list[LinkCheckRecord]) -> bool: - if not self.cache_path: - return True - - try: - self.cache_path.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(self.cache_path) as connection: - self._ensure_schema(connection) - connection.executemany( - """ - INSERT INTO link_check_state( - url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, - best_method, best_latency, fail_count, backlink_checked, has_author_link, - rss_crawl_reason, direct_success, direct_status_code, direct_latency, - proxy_success, proxy_status_code, proxy_latency, api_success, - api_status_code, api_latency - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(url) DO UPDATE SET - name = excluded.name, - avatar = excluded.avatar, - linkpage = excluded.linkpage, - checked_at = excluded.checked_at, - reachable = excluded.reachable, - crawl_allowed = excluded.crawl_allowed, - best_method = excluded.best_method, - best_latency = excluded.best_latency, - fail_count = excluded.fail_count, - backlink_checked = excluded.backlink_checked, - has_author_link = excluded.has_author_link, - rss_crawl_reason = excluded.rss_crawl_reason, - direct_success = excluded.direct_success, - direct_status_code = excluded.direct_status_code, - direct_latency = excluded.direct_latency, - proxy_success = excluded.proxy_success, - proxy_status_code = excluded.proxy_status_code, - proxy_latency = excluded.proxy_latency, - api_success = excluded.api_success, - api_status_code = excluded.api_status_code, - api_latency = excluded.api_latency - """, - [self._record_to_row(record) for record in records], - ) - connection.commit() - logging.info(f"友链检测缓存已保存（{len(records)} 条）") - return True - except Exception as exc: - logging.error(f"保存友链检测缓存失败: {exc}") - return False - - @staticmethod - def is_fresh(record: LinkCheckRecord, max_age_hours: int) -> bool: - if not record.checked_at: - return False - try: - checked_at = datetime.strptime(record.checked_at, "%Y-%m-%d %H:%M:%S") - except ValueError: - return False - age_seconds = (datetime.now() - checked_at).total_seconds() - return age_seconds < max_age_hours * 3600 - - @staticmethod - def _record_to_row(record: LinkCheckRecord) -> tuple: - return ( - record.url, - record.name, - record.avatar, - record.linkpage, - record.checked_at, - int(record.reachable), - int(record.crawl_allowed), - record.best_method, - record.best_latency, - record.fail_count, - int(record.backlink_checked), - int(record.has_author_link), - record.rss_crawl_reason, - int(record.direct.success), - record.direct.status_code, - record.direct.latency, - int(record.proxy.success), - record.proxy.status_code, - record.proxy.latency, - int(record.api.success), - record.api.status_code, - record.api.latency, - ) - - @staticmethod - def _ensure_schema(connection: sqlite3.Connection) -> None: - connection.execute( - """ - CREATE TABLE IF NOT EXISTS link_check_state ( - url TEXT PRIMARY KEY, - name TEXT NOT NULL, - avatar TEXT DEFAULT '', - linkpage TEXT DEFAULT '', - checked_at TEXT NOT NULL, - reachable INTEGER NOT NULL DEFAULT 0, - crawl_allowed INTEGER NOT NULL DEFAULT 0, - best_method TEXT NOT NULL DEFAULT 'none', - best_latency REAL DEFAULT -1, - fail_count INTEGER NOT NULL DEFAULT 0, - backlink_checked INTEGER NOT NULL DEFAULT 0, - has_author_link INTEGER NOT NULL DEFAULT 0, - rss_crawl_reason TEXT NOT NULL DEFAULT '', - direct_success INTEGER NOT NULL DEFAULT 0, - direct_status_code INTEGER, - direct_latency REAL DEFAULT -1, - proxy_success INTEGER NOT NULL DEFAULT 0, - proxy_status_code INTEGER, - proxy_latency REAL DEFAULT -1, - api_success INTEGER NOT NULL DEFAULT 0, - api_status_code INTEGER, - api_latency REAL DEFAULT -1 - ) - """ - ) +from friend_circle_lite.storage.sqlite_store import * # noqa: F401,F403 diff --git a/friend_circle_lite/cli.py b/friend_circle_lite/cli.py new file mode 100644 index 00000000000..3b712014f61 --- /dev/null +++ b/friend_circle_lite/cli.py @@ -0,0 +1,211 @@ +"""Top-level application orchestration. + +This module keeps the main script very small by moving the end-to-end workflow +into focused orchestration methods. +""" + +from __future__ import annotations + +import logging +import os +import sys + +from friend_circle_lite.config.models import ApplicationConfig, MailRuntime +from friend_circle_lite.config.printer import print_startup_config +from friend_circle_lite.crawler.single_site_legacy import get_latest_articles_from_link +from friend_circle_lite.notifications.github import extract_emails_from_issues +from friend_circle_lite.notifications.mail import send_emails +from friend_circle_lite.outputs.legacy_api import ( + deal_with_large_data, + fetch_and_process_data, + merge_data_from_json_url, + merge_errors_from_json_url, + merge_link_data_from_json_url, +) +from friend_circle_lite.utils.json import write_json + + +class FriendCircleLiteApplication: + """Application service coordinating crawl and notification workflows.""" + + def __init__(self, config: ApplicationConfig): + self.config = config + + def run(self) -> None: + """Execute the enabled application features in a stable order.""" + print_startup_config(self.config) + self.run_crawler_if_enabled() + mail_runtime = self.prepare_mail_runtime() + self.run_email_push_if_enabled(mail_runtime) + self.run_rss_subscription_if_enabled(mail_runtime) + + def run_crawler_if_enabled(self) -> None: + """Run the article crawl and persist public output files when enabled.""" + spider_settings = self.config.spider_settings + if not spider_settings.enable: + logging.info("⏭️ 爬虫未启用，跳过抓取流程") + return + + logging.info("✅ 爬虫已启用") + logging.info( + f"📥 正在从 {spider_settings.json_url} 获取数据，每个博客获取 {spider_settings.article_count} 篇文章" + ) + + crawl_result = fetch_and_process_data( + json_url=spider_settings.json_url, + specific_RSS=self.config.specific_rss, + count=spider_settings.article_count, + cache_file=self.config.runtime_paths.cache_file, + link_check_config=self.config.link_check, + proxy_settings=self.config.proxy_settings, + ) + if crawl_result is None: + logging.error("❌ 抓取流程失败，未生成任何输出文件") + return + + result, lost_friends, link_payload = crawl_result + result, lost_friends, link_payload = self._merge_remote_results_if_enabled(result, lost_friends, link_payload) + + article_count = len(result.get("article_data", [])) + logging.info(f"📦 数据获取完毕，共有 {article_count} 篇文章，正在处理数据") + + result = deal_with_large_data( + result, + future_tolerance_days=self.config.future_article_tolerance_days, + ) + write_json(self.config.runtime_paths.all_json_file, result) + write_json(self.config.runtime_paths.errors_json_file, lost_friends) + write_json(self.config.runtime_paths.link_json_file, link_payload) + + def prepare_mail_runtime(self) -> MailRuntime: + """Build SMTP runtime credentials from config and environment variables.""" + if not (self.config.email_push.enable or self.config.rss_subscribe.enable): + return MailRuntime(sender_email="", smtp_server="", port=0, password="", use_tls=False) + + logging.info("📨 推送功能已启用，正在准备中...") + smtp_conf = self.config.smtp + mail_runtime = MailRuntime( + sender_email=smtp_conf.email, + smtp_server=smtp_conf.server, + port=smtp_conf.port, + password=os.getenv("SMTP_PWD", ""), + use_tls=smtp_conf.use_tls, + ) + + logging.info(f"📡 SMTP 服务器：{mail_runtime.smtp_server}:{mail_runtime.port}") + if mail_runtime.is_ready: + logging.info(f"🔐 密码(部分)：{mail_runtime.password[:3]}*****") + else: + logging.error("❌ SMTP 信息不完整或环境变量 SMTP_PWD 未设置，无法发送邮件") + return mail_runtime + + def run_email_push_if_enabled(self, mail_runtime: MailRuntime) -> None: + """Keep the reserved email push entrypoint behavior unchanged.""" + if self.config.email_push.enable and mail_runtime.is_ready: + logging.info("📧 邮件推送已启用") + logging.info("⚠️ 抱歉，目前尚未实现邮件推送功能") + + def run_rss_subscription_if_enabled(self, mail_runtime: MailRuntime) -> None: + """Send subscription emails for newly discovered posts when enabled.""" + if not self.config.rss_subscribe.enable: + return + if not mail_runtime.is_ready: + logging.info("⏭️ RSS 订阅推送未执行，因为 SMTP 尚未就绪") + return + + logging.info("📰 RSS 订阅推送已启用") + github_username, github_repo = self._resolve_github_repo() + logging.info(f"👤 GitHub 用户名：{github_username}") + logging.info(f"📁 GitHub 仓库：{github_repo}") + + latest_articles = get_latest_articles_from_link( + url=self.config.rss_subscribe.your_blog_url, + count=10, + last_articles_path=self.config.runtime_paths.cache_file, + ) + if not latest_articles: + logging.info("📭 无新文章，无需推送") + return + + logging.info(f"🆕 获取到的最新文章：{latest_articles}") + email_list = self._load_subscriber_emails(github_username, github_repo) + if not email_list: + logging.info("⚠️ 无订阅邮箱，请检查格式或是否有订阅者") + sys.exit(0) + + logging.info(f"📬 获取到邮箱列表：{email_list}") + for article in latest_articles: + template_data = self._build_email_template_data(article, github_username, github_repo) + send_emails( + emails=email_list["emails"], + sender_email=mail_runtime.sender_email, + smtp_server=mail_runtime.smtp_server, + port=mail_runtime.port, + password=mail_runtime.password, + subject=f"{self.config.rss_subscribe.website_info.title} の最新文章：{article['title']}", + body=self._build_plaintext_mail_body(article), + template_path=self.config.rss_subscribe.email_template, + template_data=template_data, + use_tls=mail_runtime.use_tls, + ) + + def _merge_remote_results_if_enabled( + self, result: dict, lost_friends: list[list[str]], link_payload: dict + ) -> tuple[dict, list[list[str]], dict]: + """Merge remote outputs when the merge option is enabled.""" + merge_settings = self.config.merge_settings + if not merge_settings.enable: + return result, lost_friends, link_payload + + remote_url = merge_settings.remote_base_url + logging.info(f"🔀 合并功能开启，从 {remote_url} 获取外部数据") + + if merge_settings.merge_article_data: + result = merge_data_from_json_url(result, f"{remote_url}/all.json") + lost_friends = merge_errors_from_json_url(lost_friends, f"{remote_url}/errors.json") + + if merge_settings.merge_link_check_data: + link_payload = merge_link_data_from_json_url(link_payload, f"{remote_url}/link.json") + + return result, lost_friends, link_payload + + def _resolve_github_repo(self) -> tuple[str, str]: + """Resolve repository coordinates from env override or config.""" + fcl_repo = os.getenv("FCL_REPO") + if fcl_repo: + return tuple(fcl_repo.split("/", 1)) + return self.config.rss_subscribe.github_username, self.config.rss_subscribe.github_repo + + @staticmethod + def _load_subscriber_emails(github_username: str, github_repo: str) -> dict | None: + """Load subscriber emails from GitHub closed issues.""" + github_api_url = ( + f"https://api.github.com/repos/{github_username}/{github_repo}/issues" + f"?state=closed&label=subscribed&per_page=200" + ) + logging.info(f"🔎 正在从 GitHub 获取订阅邮箱：{github_api_url}") + return extract_emails_from_issues(github_api_url) + + def _build_email_template_data(self, article: dict, github_username: str, github_repo: str) -> dict[str, str]: + """Assemble template variables for one outbound notification email.""" + return { + "title": article["title"], + "summary": article["summary"], + "published": article["published"], + "link": article["link"], + "website_title": self.config.rss_subscribe.website_info.title, + "github_issue_url": ( + f"https://github.com/{github_username}/{github_repo}" + "/issues?q=is%3Aissue+is%3Aclosed" + ), + } + + @staticmethod + def _build_plaintext_mail_body(article: dict) -> str: + """Build the plain-text fallback body for one notification email.""" + return ( + f"📄 文章标题：{article['title']}\n" + f"🔗 链接：{article['link']}\n" + f"📝 简介：{article['summary']}\n" + f"🕒 发布时间：{article['published']}" + ) diff --git a/friend_circle_lite/config/__init__.py b/friend_circle_lite/config/__init__.py new file mode 100644 index 00000000000..df04cf33002 --- /dev/null +++ b/friend_circle_lite/config/__init__.py @@ -0,0 +1,3 @@ +"""Configuration loading and typed config models.""" + +from friend_circle_lite.config.models import * # noqa: F401,F403 diff --git a/friend_circle_lite/config/models.py b/friend_circle_lite/config/models.py new file mode 100644 index 00000000000..7635506cc74 --- /dev/null +++ b/friend_circle_lite/config/models.py @@ -0,0 +1,208 @@ +"""Application configuration models. + +This module converts the raw YAML structure into typed configuration objects so +that the rest of the application can depend on explicit fields instead of a +loosely typed nested dictionary. + +The external YAML keys are preserved for backward compatibility. Internally, +snake_case names are used consistently. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field + + +DEFAULT_CACHE_FILE = "./temp/cache.sqlite3" +DEFAULT_ALL_JSON = "./all.json" +DEFAULT_ERRORS_JSON = "./errors.json" +DEFAULT_LINK_JSON = "./link.json" + + +@dataclass(slots=True) +class MergeSettings: + """Options for merging local crawl results with remote data sources.""" + + enable: bool = False + remote_base_url: str = "" + merge_article_data: bool = True + merge_link_check_data: bool = True + + +@dataclass(slots=True) +class ProxySettings: + """Proxy configuration for both link checking and RSS crawling.""" + + proxy_url: str = "" + + +@dataclass(slots=True) +class SpiderSettings: + """Crawler settings controlling source list and output density.""" + + enable: bool = True + json_url: str = "" + article_count: int = 5 + + +@dataclass(slots=True) +class LinkCheckConfig: + """Settings for friend link reachability checks.""" + + enable: bool = True + max_age_hours: int = 24 + timeout: int = 15 + max_workers: int = 10 + status_api_url: str = "https://v2.xxapi.cn/api/status?url={url}" + enable_backlink_check: bool = False + author_url: str = "" + + +@dataclass(slots=True) +class EmailPushConfig: + """Reserved configuration for the not-yet-implemented email push feature.""" + + enable: bool = False + to_email: str = "" + subject: str = "" + body_template: str = "" + + +@dataclass(slots=True) +class WebsiteInfo: + """Display metadata for outbound notifications.""" + + title: str = "" + + +@dataclass(slots=True) +class RssSubscribeConfig: + """Configuration for GitHub issue based email subscriptions.""" + + enable: bool = False + github_username: str = "" + github_repo: str = "" + your_blog_url: str = "" + email_template: str = "" + website_info: WebsiteInfo = field(default_factory=WebsiteInfo) + + +@dataclass(slots=True) +class SmtpConfig: + """SMTP connection settings used by all mail sending features.""" + + email: str = "" + server: str = "" + port: int = 0 + use_tls: bool = True + + +@dataclass(slots=True) +class RuntimePaths: + """Filesystem locations used by the runtime.""" + + cache_file: str = DEFAULT_CACHE_FILE + all_json_file: str = DEFAULT_ALL_JSON + errors_json_file: str = DEFAULT_ERRORS_JSON + link_json_file: str = DEFAULT_LINK_JSON + + +@dataclass(slots=True) +class ApplicationConfig: + """Root application configuration assembled from the YAML file.""" + + spider_settings: SpiderSettings + proxy_settings: ProxySettings + merge_settings: MergeSettings + link_check: LinkCheckConfig + email_push: EmailPushConfig + rss_subscribe: RssSubscribeConfig + smtp: SmtpConfig + specific_rss: list[dict] + runtime_paths: RuntimePaths = field(default_factory=RuntimePaths) + future_article_tolerance_days: int = 2 + + @classmethod + def from_dict(cls, data: dict) -> "ApplicationConfig": + """Create a typed config object from the raw YAML dictionary.""" + spider_raw = data.get("spider_settings", {}) + proxy_raw = data.get("proxy_settings", {}) + merge_raw = data.get("merge_settings", {}) + link_check_raw = data.get("link_check", {}) + email_push_raw = data.get("email_push", {}) + rss_subscribe_raw = data.get("rss_subscribe", {}) + website_info_raw = rss_subscribe_raw.get("website_info", {}) + smtp_raw = data.get("smtp", {}) + runtime_raw = data.get("runtime_paths", {}) + + return cls( + spider_settings=SpiderSettings( + enable=bool(spider_raw.get("enable", True)), + json_url=str(spider_raw.get("json_url", "")).strip(), + article_count=int(spider_raw.get("article_count", 5)), + ), + proxy_settings=ProxySettings( + proxy_url=os.getenv("PROXY_URL") or str(proxy_raw.get("proxy_url", "")).strip(), + ), + merge_settings=MergeSettings( + enable=bool(merge_raw.get("enable", False)), + remote_base_url=str(merge_raw.get("remote_base_url", "")).strip(), + merge_article_data=bool(merge_raw.get("merge_article_data", True)), + merge_link_check_data=bool(merge_raw.get("merge_link_check_data", True)), + ), + link_check=LinkCheckConfig( + enable=bool(link_check_raw.get("enable", True)), + max_age_hours=int(link_check_raw.get("max_age_hours", 24)), + timeout=int(link_check_raw.get("timeout", 15)), + max_workers=int(link_check_raw.get("max_workers", 10)), + status_api_url=str(link_check_raw.get("status_api_url", "https://v2.xxapi.cn/api/status?url={url}")).strip(), + enable_backlink_check=bool(link_check_raw.get("enable_backlink_check", False)), + author_url=str(link_check_raw.get("author_url", "")).strip(), + ), + email_push=EmailPushConfig( + enable=bool(email_push_raw.get("enable", False)), + to_email=str(email_push_raw.get("to_email", "")).strip(), + subject=str(email_push_raw.get("subject", "")).strip(), + body_template=str(email_push_raw.get("body_template", "")).strip(), + ), + rss_subscribe=RssSubscribeConfig( + enable=bool(rss_subscribe_raw.get("enable", False)), + github_username=str(rss_subscribe_raw.get("github_username", "")).strip(), + github_repo=str(rss_subscribe_raw.get("github_repo", "")).strip(), + your_blog_url=str(rss_subscribe_raw.get("your_blog_url", "")).strip(), + email_template=str(rss_subscribe_raw.get("email_template", "")).strip(), + website_info=WebsiteInfo( + title=str(website_info_raw.get("title", "")).strip(), + ), + ), + smtp=SmtpConfig( + email=str(smtp_raw.get("email", "")).strip(), + server=str(smtp_raw.get("server", "")).strip(), + port=int(smtp_raw.get("port", 0) or 0), + use_tls=bool(smtp_raw.get("use_tls", True)), + ), + specific_rss=list(data.get("specific_RSS", []) or []), + runtime_paths=RuntimePaths( + cache_file=str(runtime_raw.get("cache_file", DEFAULT_CACHE_FILE)).strip() or DEFAULT_CACHE_FILE, + all_json_file=str(runtime_raw.get("all_json_file", DEFAULT_ALL_JSON)).strip() or DEFAULT_ALL_JSON, + errors_json_file=str(runtime_raw.get("errors_json_file", DEFAULT_ERRORS_JSON)).strip() or DEFAULT_ERRORS_JSON, + link_json_file=str(runtime_raw.get("link_json_file", DEFAULT_LINK_JSON)).strip() or DEFAULT_LINK_JSON, + ), + ) + + +@dataclass(slots=True) +class MailRuntime: + """Runtime SMTP credentials resolved from configuration and environment.""" + + sender_email: str + smtp_server: str + port: int + password: str + use_tls: bool + + @property + def is_ready(self) -> bool: + """Whether enough information is available to send email.""" + return bool(self.sender_email and self.smtp_server and self.port and self.password) diff --git a/friend_circle_lite/config/printer.py b/friend_circle_lite/config/printer.py new file mode 100644 index 00000000000..e87c82a4214 --- /dev/null +++ b/friend_circle_lite/config/printer.py @@ -0,0 +1,54 @@ +"""Configuration printer for startup diagnostics.""" + +import logging + + +def print_startup_config(config): + """Print all configuration settings at startup for debugging.""" + logging.info("=" * 60) + logging.info("🚀 Friend-Circle-Lite 启动配置") + logging.info("=" * 60) + + # Spider settings + logging.info("📡 爬虫配置:") + logging.info(f" - 启用状态: {'✅ 已启用' if config.spider_settings.enable else '❌ 已禁用'}") + if config.spider_settings.enable: + logging.info(f" - 数据源: {config.spider_settings.json_url}") + logging.info(f" - 每站文章数: {config.spider_settings.article_count}") + + # Proxy settings + logging.info("🔀 代理配置:") + if config.proxy_settings.proxy_url: + logging.info(f" - 代理地址: {config.proxy_settings.proxy_url}") + logging.info(f" - 用途: 友链检测 + RSS 抓取") + else: + logging.info(f" - 代理状态: ❌ 未配置") + + # Link check settings + logging.info("🔍 友链检测配置:") + logging.info(f" - 启用状态: {'✅ 已启用' if config.link_check.enable else '❌ 已禁用'}") + if config.link_check.enable: + logging.info(f" - 缓存时间: {config.link_check.max_age_hours} 小时") + logging.info(f" - 超时时间: {config.link_check.timeout} 秒") + logging.info(f" - 并发数: {config.link_check.max_workers}") + logging.info(f" - 反链检测: {'✅ 已启用' if config.link_check.enable_backlink_check else '❌ 已禁用'}") + if config.link_check.enable_backlink_check: + logging.info(f" - 站点域名: {config.link_check.author_url}") + + # Merge settings + logging.info("🔗 数据合并配置:") + logging.info(f" - 启用状态: {'✅ 已启用' if config.merge_settings.enable else '❌ 已禁用'}") + if config.merge_settings.enable: + logging.info(f" - 远程数据源: {config.merge_settings.remote_base_url}") + logging.info(f" - 合并文章数据: {'✅ 是' if config.merge_settings.merge_article_data else '❌ 否'}") + logging.info(f" - 合并友链数据: {'✅ 是' if config.merge_settings.merge_link_check_data else '❌ 否'}") + + # Email push settings + logging.info("📧 邮件推送配置:") + logging.info(f" - 启用状态: {'✅ 已启用' if config.email_push.enable else '❌ 已禁用'}") + + # RSS subscribe settings + logging.info("📮 RSS 订阅配置:") + logging.info(f" - 启用状态: {'✅ 已启用' if config.rss_subscribe.enable else '❌ 已禁用'}") + + logging.info("=" * 60) diff --git a/friend_circle_lite/config_printer.py b/friend_circle_lite/config_printer.py index e87c82a4214..c5f367d5ff8 100644 --- a/friend_circle_lite/config_printer.py +++ b/friend_circle_lite/config_printer.py @@ -1,54 +1,6 @@ -"""Configuration printer for startup diagnostics.""" +"""Backward-compatible startup config printer export. -import logging +New code should import from `friend_circle_lite.config.printer`. +""" - -def print_startup_config(config): - """Print all configuration settings at startup for debugging.""" - logging.info("=" * 60) - logging.info("🚀 Friend-Circle-Lite 启动配置") - logging.info("=" * 60) - - # Spider settings - logging.info("📡 爬虫配置:") - logging.info(f" - 启用状态: {'✅ 已启用' if config.spider_settings.enable else '❌ 已禁用'}") - if config.spider_settings.enable: - logging.info(f" - 数据源: {config.spider_settings.json_url}") - logging.info(f" - 每站文章数: {config.spider_settings.article_count}") - - # Proxy settings - logging.info("🔀 代理配置:") - if config.proxy_settings.proxy_url: - logging.info(f" - 代理地址: {config.proxy_settings.proxy_url}") - logging.info(f" - 用途: 友链检测 + RSS 抓取") - else: - logging.info(f" - 代理状态: ❌ 未配置") - - # Link check settings - logging.info("🔍 友链检测配置:") - logging.info(f" - 启用状态: {'✅ 已启用' if config.link_check.enable else '❌ 已禁用'}") - if config.link_check.enable: - logging.info(f" - 缓存时间: {config.link_check.max_age_hours} 小时") - logging.info(f" - 超时时间: {config.link_check.timeout} 秒") - logging.info(f" - 并发数: {config.link_check.max_workers}") - logging.info(f" - 反链检测: {'✅ 已启用' if config.link_check.enable_backlink_check else '❌ 已禁用'}") - if config.link_check.enable_backlink_check: - logging.info(f" - 站点域名: {config.link_check.author_url}") - - # Merge settings - logging.info("🔗 数据合并配置:") - logging.info(f" - 启用状态: {'✅ 已启用' if config.merge_settings.enable else '❌ 已禁用'}") - if config.merge_settings.enable: - logging.info(f" - 远程数据源: {config.merge_settings.remote_base_url}") - logging.info(f" - 合并文章数据: {'✅ 是' if config.merge_settings.merge_article_data else '❌ 否'}") - logging.info(f" - 合并友链数据: {'✅ 是' if config.merge_settings.merge_link_check_data else '❌ 否'}") - - # Email push settings - logging.info("📧 邮件推送配置:") - logging.info(f" - 启用状态: {'✅ 已启用' if config.email_push.enable else '❌ 已禁用'}") - - # RSS subscribe settings - logging.info("📮 RSS 订阅配置:") - logging.info(f" - 启用状态: {'✅ 已启用' if config.rss_subscribe.enable else '❌ 已禁用'}") - - logging.info("=" * 60) +from friend_circle_lite.config.printer import print_startup_config # noqa: F401 diff --git a/friend_circle_lite/crawler/__init__.py b/friend_circle_lite/crawler/__init__.py new file mode 100644 index 00000000000..be66f0adf1d --- /dev/null +++ b/friend_circle_lite/crawler/__init__.py @@ -0,0 +1,4 @@ +"""Crawler services for feed discovery, parsing, and aggregation.""" + +from friend_circle_lite.crawler.feed_service import FeedDiscoveryService, FeedParserService, LatestArticleTracker +from friend_circle_lite.crawler.service import FeedResolver, FriendCircleCrawlService, SingleSiteCrawler diff --git a/friend_circle_lite/crawler/feed_service.py b/friend_circle_lite/crawler/feed_service.py new file mode 100644 index 00000000000..3f6736f8e77 --- /dev/null +++ b/friend_circle_lite/crawler/feed_service.py @@ -0,0 +1,256 @@ +"""Feed discovery, parsing, and incremental tracking services.""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime +from pathlib import Path +from urllib.parse import urlparse + +import feedparser +import requests + +from friend_circle_lite import HEADERS_XML, timeout +from friend_circle_lite.domain.models import Article, FeedEndpoint, Website +from friend_circle_lite.utils.time import format_published_time +from friend_circle_lite.utils.url import replace_non_domain + + +class FeedDiscoveryService: + """Discover an RSS or Atom endpoint for a website.""" + + POSSIBLE_FEEDS = [ + ("rss1", "/feed"), # WordPress / 最常见 + ("rss2", "/feed/"), # WordPress 兼容写法 + ("rss3", "/rss.xml"), # 很多传统站点 + ("rss4", "/atom.xml"), # 静态博客常见（Hugo / Jekyll） + ("rss5", "/feed.xml"), # 通用型 + ("rss6", "/index.xml"), # Hugo / 一些静态站 + ("rss7", "/feed.atom"), # Atom 明确路径 + ("rss8", "/rss2.xml"), # 老系统遗留 + ("rss9", "/rss/feed.xml"),# 少见但存在 + ("rss10", "/rss.php"), # 老 PHP 程序 + ("rss11", "/feed.php"), # 同上 + ] + + def __init__(self, session: requests.Session): + self.session = session + + def discover(self, website_url: str) -> FeedEndpoint | None: + """Try common feed endpoints and return the first valid match.""" + for feed_type, path in self.POSSIBLE_FEEDS: + feed_url = website_url.rstrip("/") + path + try: + response = self.session.get(feed_url, headers=HEADERS_XML, timeout=timeout) + except requests.RequestException: + continue + + if response.status_code != 200: + continue + + content_type = response.headers.get("Content-Type", "").lower() + if "xml" in content_type or "rss" in content_type or "atom" in content_type: + return FeedEndpoint(url=feed_url, feed_type=feed_type, source="auto") + + text_head = response.text[:1000].lower() + if " list[Article]: + """Parse a feed URL and return the newest `count` articles. + + The returned articles are normalized to the project's internal domain + model, while preserving the original public output fields. + """ + try: + response = self.session.get(feed_url, headers=HEADERS_XML, timeout=timeout) + # 强制使用 UTF-8 编码，因为 apparent_encoding 可能检测错误 + response.encoding = "utf-8" + feed = feedparser.parse(response.text) + except Exception as exc: + logging.error(f"解析 RSS 失败：{feed_url}，错误: {exc}") + return [] + + default_author = feed.feed.author if "author" in feed.feed else "" + articles: list[Article] = [] + + for entry in feed.entries: + published = self._extract_published_time(entry) + article_link = replace_non_domain(entry.link, blog_url) if "link" in entry else "" + article = Article( + title=entry.title if "title" in entry else "", + author=default_author, + link=article_link, + published=published, + summary=entry.summary if "summary" in entry else "", + content=entry.content[0].value if "content" in entry and entry.content else entry.description if "description" in entry else "", + ) + articles.append(article) + + valid_articles = [article for article in articles if article.published] + + # 过滤掉无法解析的日期格式 + def safe_parse_date(article): + try: + return datetime.strptime(article.published, "%Y-%m-%d %H:%M") + except ValueError: + logging.warning(f"文章 {article.title} 的发布时间格式异常: {article.published}，已跳过") + return None + + # 只保留能成功解析日期的文章 + valid_articles_with_dates = [] + for article in valid_articles: + parsed_date = safe_parse_date(article) + if parsed_date: + valid_articles_with_dates.append((article, parsed_date)) + + # 按日期排序 + valid_articles_with_dates.sort(key=lambda item: item[1], reverse=True) + sorted_articles = [item[0] for item in valid_articles_with_dates] + + return sorted_articles[:count] if count < len(sorted_articles) else sorted_articles + + @staticmethod + def _extract_published_time(entry) -> str: + """Extract a normalized publish time from a feed entry.""" + import time + + def convert_time_to_string(time_value): + """Convert various time formats to string.""" + if isinstance(time_value, str): + return time_value + elif isinstance(time_value, time.struct_time): + # 检查年份是否异常 + if time_value.tm_year < 1900: + logging.warning(f"文章 {entry.get('title', 'Unknown')} 的时间年份异常: {time_value.tm_year}，已跳过") + return "" + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time_value) + else: + logging.warning(f"文章 {entry.get('title', 'Unknown')} 的时间格式未知: {type(time_value)}，已跳过") + return "" + + if "published" in entry: + time_str = convert_time_to_string(entry.published) + if not time_str: + return "" + return format_published_time(time_str) + if "updated" in entry: + time_str = convert_time_to_string(entry.updated) + if not time_str: + return "" + published = format_published_time(time_str) + logging.warning(f"文章 {entry.title} 未包含发布时间，已使用更新时间 {published}") + return published + + logging.warning(f"文章 {entry.title} 未包含任何时间信息, 请检查原文, 跳过该文章") + return "" + + +class LatestArticleTracker: + """Track whether a website published new posts since the last crawl.""" + + def __init__(self, storage_path: str | Path, max_tracked_articles: int = 10): + from friend_circle_lite.storage.sqlite_store import ArticleTrackingStore + self.store = ArticleTrackingStore(storage_path, max_tracked_articles) + + def diff_and_persist(self, latest_articles: list[Article]) -> list[dict] | None: + """Return newly seen articles and update the local storage. + + Returns None if: + - This is the first run (no previous data exists) + - No new articles are found + - New articles exist but are not newer than the most recent tracked article + """ + previous_articles = self.store.load_articles() + + # First run: no previous data exists, skip sending to prevent sending old articles + if not previous_articles: + logging.info(f"首次运行：跳过推送以防止发送旧文章") + self.store.save_articles(latest_articles) + return None + + previous_latest_date = self._get_latest_date(previous_articles) + + # Find articles that are truly new (check only: link, title, published) + new_articles = [] + for article in latest_articles: + if self._is_truly_new_article(article, previous_articles): + new_articles.append(article) + + if not new_articles: + self.store.save_articles(latest_articles) + return None + + # Filter new articles: only keep those newer than the previous latest date + truly_new_articles = [] + for article in new_articles: + if not article.published: + continue + try: + article_date = datetime.strptime(article.published, "%Y-%m-%d %H:%M") + if previous_latest_date is None or article_date > previous_latest_date: + truly_new_articles.append(article) + except Exception as exc: + logging.warning(f"解析文章日期失败: {article.title}, 日期: {article.published}, 错误: {exc}") + continue + + self.store.save_articles(latest_articles) + + if truly_new_articles: + logging.info(f"发现 {len(truly_new_articles)} 篇新文章（日期比之前更新）") + return [article.to_tracking_dict() for article in truly_new_articles] + else: + logging.info(f"发现 {len(new_articles)} 篇新文章，但日期不够新，跳过推送") + return None + + @staticmethod + def _is_truly_new_article(article: Article, previous_articles: list[Article]) -> bool: + """Check if an article is truly new by comparing link, title, and published date. + + An article is considered new only if its link, title, and published date + do not match any previous article (empty values are skipped). + """ + for prev in previous_articles: + # Check link, title, and published: if any non-empty field matches, it's not new + if article.link and article.link == prev.link: + return False + if article.title and article.title == prev.title: + return False + if article.published and article.published == prev.published: + return False + + return True + + @staticmethod + def _get_latest_date(articles: list[Article]) -> datetime | None: + """Find the latest publish date from a list of articles.""" + latest_date = None + for article in articles: + if not article.published: + continue + try: + article_date = datetime.strptime(article.published, "%Y-%m-%d %H:%M") + if latest_date is None or article_date > latest_date: + latest_date = article_date + except Exception: + continue + return latest_date + + +def extract_blog_origin(url: str) -> str: + """Return a normalized origin for display or author profile links.""" + parsed = urlparse(url) + if not parsed.scheme or not parsed.netloc: + return url + return f"{parsed.scheme}://{parsed.netloc}" diff --git a/friend_circle_lite/crawler/service.py b/friend_circle_lite/crawler/service.py new file mode 100644 index 00000000000..1efb487171c --- /dev/null +++ b/friend_circle_lite/crawler/service.py @@ -0,0 +1,375 @@ +"""High-level crawler orchestration. + +This module contains the system-level services that coordinate website loading, +RSS discovery, parsing, cache updates, result aggregation, and legacy output +formatting. +""" + +from __future__ import annotations + +import logging +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +import requests + +from friend_circle_lite import HEADERS_JSON, timeout +from friend_circle_lite.config.models import LinkCheckConfig, ProxySettings +from friend_circle_lite.crawler.feed_service import FeedDiscoveryService, FeedParserService +from friend_circle_lite.domain.models import Article, CacheRecord, CacheUpdate, CrawlResult, CrawlStatistics, FeedEndpoint, LinkCheckRecord, Website +from friend_circle_lite.link_checker.service import LinkReachabilityService +from friend_circle_lite.storage.sqlite_store import FeedCacheStore, LinkCheckStore + + +class FeedResolver: + """Resolve which feed endpoint should be used for a website. + + Resolution order is kept compatible with the previous implementation: + manual configuration first, then cache, and finally automatic discovery. + """ + + def __init__(self, discovery_service: FeedDiscoveryService, configured_feeds: list[CacheRecord]): + self.discovery_service = discovery_service + self.feed_lookup = {item.name: item for item in configured_feeds} + + def resolve(self, website: Website) -> FeedEndpoint | None: + configured = self.feed_lookup.get(website.name) + if configured: + if configured.source == 'manual': + logging.info(f"'{website.name}' 使用预设 RSS 源：{configured.url}") + elif configured.source == 'cache': + logging.info(f"'{website.name}' 使用缓存 RSS 源：{configured.url}") + else: + logging.info(f"'{website.name}' 使用 RSS 源：{configured.url} (来源: {configured.source})") + return FeedEndpoint(url=configured.url, feed_type="specific", source=configured.source) + + discovered = self.discovery_service.discover(website.url) + if discovered: + logging.info(f"'{website.name}' 自动探测到 RSS：{discovered.url}") + return discovered + + +class SingleSiteCrawler: + """Crawl one website and produce a normalized result.""" + + def __init__(self, parser_service: FeedParserService, resolver: FeedResolver): + self.parser_service = parser_service + self.resolver = resolver + + def crawl(self, website: Website, count: int) -> CrawlResult: + """Crawl one website while preserving legacy cache repair behavior.""" + endpoint = self.resolver.resolve(website) + cache_update = CacheUpdate(action="none", name=website.name) + + if endpoint and endpoint.source == "auto": + cache_update = CacheUpdate(action="set", name=website.name, url=endpoint.url, reason="auto_discovered") + + articles = self._parse_articles(endpoint, website, count) + parse_error = endpoint is not None and not articles + + if parse_error and endpoint and endpoint.source in ("cache", "unknown"): + logging.warning(f"'{website.name}' 缓存的 RSS 源无效，尝试重新探测...") + rediscovered = self.resolver.discovery_service.discover(website.url) + if rediscovered: + articles = self._parse_articles(rediscovered, website, count) + if articles: + endpoint = rediscovered + cache_update = CacheUpdate(action="set", name=website.name, url=rediscovered.url, reason="repair_cache") + logging.info(f"'{website.name}' 重新探测成功，更新缓存：{rediscovered.url}") + else: + endpoint = None + cache_update = CacheUpdate(action="delete", name=website.name, url=None, reason="remove_invalid") + logging.warning(f"'{website.name}' 重新探测失败，删除无效缓存") + else: + endpoint = None + cache_update = CacheUpdate(action="delete", name=website.name, url=None, reason="remove_invalid") + logging.warning(f"'{website.name}' 未找到有效 RSS，删除无效缓存") + + status = "active" if articles else "error" + if not articles: + if endpoint is None: + logging.warning(f"'{website.name}' 的博客 {website.url} 未找到有效 RSS ") + else: + logging.warning(f"'{website.name}' 的 RSS {endpoint.url} 未解析出文章 ") + + return CrawlResult( + website=website, + status=status, + articles=articles, + feed_url=endpoint.url if endpoint else None, + feed_type=endpoint.feed_type if endpoint else "none", + source_used=endpoint.source if endpoint else "none", + cache_update=cache_update, + ) + + def _parse_articles(self, endpoint: FeedEndpoint | None, website: Website, count: int) -> list[Article]: + if endpoint is None: + return [] + + articles = self.parser_service.parse(endpoint.url, count=count, blog_url=website.url) + for article in articles: + article.author = website.name + article.avatar = website.avatar + logging.info(f"{website.name} 发布了新文章：{article.title}，时间：{article.published}，链接：{article.link}") + return articles + + +class FriendCircleCrawlService: + """System-level orchestrator for crawling all configured websites.""" + + def __init__( + self, + json_url: str, + count: int, + specific_rss: list[dict] | None = None, + cache_file: str | None = None, + link_check_config: LinkCheckConfig | None = None, + proxy_settings: ProxySettings | None = None, + ): + self.json_url = json_url + self.count = count + self.specific_rss = specific_rss or [] + self.cache_store = FeedCacheStore(cache_file) + self.link_check_config = link_check_config or LinkCheckConfig(enable=False) + self.proxy_settings = proxy_settings or ProxySettings() + self.link_check_store = LinkCheckStore(cache_file) + + def run(self) -> tuple[dict, list[list[str]]] | None: + """Fetch website list, crawl all websites, and build public outputs.""" + session = requests.Session() + websites = self._load_websites(session) + if websites is None: + return None + + link_check_records = self._check_links(websites) + link_check_map = {record.url: record for record in link_check_records} + crawlable_websites = [website for website in websites if link_check_map.get(website.url, LinkCheckRecord.unchecked(website)).crawl_allowed] + skipped_count = len(websites) - len(crawlable_websites) + if skipped_count: + logging.info(f"🔎 根据友链可达性检测跳过 {skipped_count} 个不可抓取站点") + + cache_records = self.cache_store.load_records() + manual_records = self._build_manual_records() + merged_records = self._merge_feed_records(cache_records, manual_records) + manual_names = {record.name for record in manual_records} + + discovery_service = FeedDiscoveryService(session) + parser_service = FeedParserService(session) + resolver = FeedResolver(discovery_service=discovery_service, configured_feeds=merged_records) + crawler = SingleSiteCrawler(parser_service=parser_service, resolver=resolver) + + crawl_results: list[CrawlResult] = [] + with ThreadPoolExecutor(max_workers=10) as executor: + future_to_website = { + executor.submit(crawler.crawl, website, self.count): website + for website in crawlable_websites + } + for future in as_completed(future_to_website): + website = future_to_website[future] + try: + crawl_results.append(future.result()) + except Exception as exc: + logging.error(f"处理 {website.to_error_payload()} 时发生错误: {exc}", exc_info=True) + crawl_results.append(CrawlResult(website=website, status="error")) + + self._apply_cache_updates(cache_records, crawl_results, manual_names) + + active_results = [result for result in crawl_results if result.status == "active"] + unreachable_results = [record for record in link_check_records if not record.reachable] + crawl_error_results = [result.website.to_error_payload() for result in crawl_results if result.status != "active"] + error_results = [[record.name, record.url, record.avatar] for record in unreachable_results] + all_articles = [article.to_public_dict() for result in active_results for article in result.articles] + + statistics = CrawlStatistics.create( + friends_num=len(websites), + active_num=len(active_results), + error_num=len(websites) - len(active_results), + article_num=len(all_articles), + ) + stats_payload = statistics.to_dict() + stats_payload.update(self._build_link_statistics(link_check_records)) + result = { + "statistical_data": stats_payload, + "article_data": all_articles, + } + link_payload = self._build_link_payload(link_check_records) + logging.info( + f"数据处理完成，总共有 {len(websites)} 位朋友，其中 {len(active_results)} 位博客可抓取到文章，" + f"{len(crawl_error_results)} 位博客 RSS 抓取失败，{len(unreachable_results)} 位友链不可达。" + ) + return result, error_results, link_payload + + def _check_links(self, websites: list[Website]) -> list[LinkCheckRecord]: + service = LinkReachabilityService(config=self.link_check_config, proxy_settings=self.proxy_settings, store=self.link_check_store) + return service.check_websites(websites) + + @staticmethod + def _build_link_statistics(records: list[LinkCheckRecord]) -> dict[str, int | str]: + reachable = [record for record in records if record.reachable] + crawl_allowed = [record for record in records if record.crawl_allowed] + api_only = [record for record in records if record.best_method == "api"] + has_author_link = [record for record in records if record.has_author_link] + checked_times = [record.checked_at for record in records if record.checked_at] + return { + "link_total_num": len(records), + "link_reachable_num": len(reachable), + "link_unreachable_num": len(records) - len(reachable), + "crawl_allowed_num": len(crawl_allowed), + "api_only_num": len(api_only), + "has_author_link_num": len(has_author_link), + "link_last_checked_time": max(checked_times) if checked_times else "", + } + + @staticmethod + def _build_link_payload(records: list[LinkCheckRecord]) -> dict[str, object]: + return { + "statistical_data": FriendCircleCrawlService._build_link_statistics(records), + "link_data": [record.to_link_dict() for record in records], + } + + @staticmethod + def _build_friend_data( + websites: list[Website], + crawl_results: list[CrawlResult], + link_check_map: dict[str, LinkCheckRecord], + ) -> list[dict[str, object]]: + crawl_result_map = {result.website.url: result for result in crawl_results} + friend_data: list[dict[str, object]] = [] + for website in websites: + link_record = link_check_map.get(website.url) or LinkCheckRecord.unchecked(website) + crawl_result = crawl_result_map.get(website.url) + friend_data.append({ + "name": website.name, + "url": website.url, + "avatar": website.avatar, + "linkpage": website.linkpage, + "reachable": link_record.reachable, + "crawl_allowed": link_record.crawl_allowed, + "best_method": link_record.best_method, + "best_latency": link_record.best_latency, + "fail_count": link_record.fail_count, + "backlink_checked": link_record.backlink_checked, + "has_author_link": link_record.has_author_link, + "rss_crawl_reason": link_record.rss_crawl_reason, + "feed_status": crawl_result.status if crawl_result else "skipped", + "feed_url": crawl_result.feed_url if crawl_result else None, + "feed_type": crawl_result.feed_type if crawl_result else "none", + "article_count": len(crawl_result.articles) if crawl_result else 0, + }) + return friend_data + + def _load_websites(self, session: requests.Session) -> list[Website] | None: + try: + response = session.get(self.json_url, headers=HEADERS_JSON, timeout=timeout) + response.raise_for_status() + friends_data = response.json() + except Exception as exc: + logging.error(f"无法获取链接：{self.json_url} ：{exc}", exc_info=True) + return None + + websites: list[Website] = [] + for friend in friends_data.get("friends", []): + try: + websites.append(Website.from_friend_item(friend)) + except Exception: + logging.warning(f"发现格式异常的友链数据，已跳过: {friend!r}") + return websites + + def _build_manual_records(self) -> list[CacheRecord]: + manual_records: list[CacheRecord] = [] + for item in self.specific_rss: + if isinstance(item, dict) and item.get("name") and item.get("url"): + manual_records.append(CacheRecord(name=item["name"], url=item["url"], source="manual")) + return manual_records + + @staticmethod + def _merge_feed_records(cache_records: list[CacheRecord], manual_records: list[CacheRecord]) -> list[CacheRecord]: + merged = {record.name: record for record in cache_records} + for record in manual_records: + merged[record.name] = record + return list(merged.values()) + + def _apply_cache_updates(self, cache_records: list[CacheRecord], crawl_results: list[CrawlResult], manual_names: set[str]) -> None: + cache_map = {record.name: record for record in cache_records} + unique_updates: dict[str, CacheUpdate] = {} + + for result in crawl_results: + update = result.cache_update + if not update.name or update.action == "none" or update.name in manual_names: + continue + if update.action == "set" and update.url: + unique_updates[update.name] = update + elif update.action == "delete": + unique_updates[update.name] = update + + for name, update in unique_updates.items(): + if update.action == "set" and update.url: + cache_map[name] = CacheRecord(name=name, url=update.url, source="cache") + if update.reason == "auto_discovered": + logging.info(f"💾 缓存新增：{name} -> {update.url} (自动探测)") + elif update.reason == "repair_cache": + logging.info(f"💾 缓存修复：{name} -> {update.url} (重新探测)") + else: + logging.info(f"💾 缓存更新：{name} -> {update.url} ({update.reason})") + elif update.action == "delete" and name in cache_map: + cache_map.pop(name) + logging.info(f"🗑️ 缓存删除：{name} (RSS 源失效)") + + self.cache_store.save_records(list(cache_map.values())) + + +def sort_articles_by_time(data: dict, future_tolerance_days: int = 2) -> dict: + """Sort article payloads by time and remove far-future timestamps.""" + for article in data.get("article_data", []): + if not article.get("created"): + article["created"] = "2024-01-01 00:00" + logging.warning(f"文章 {article['title']} 未包含时间信息，已设置为默认时间 2024-01-01 00:00") + + now = datetime.now(ZoneInfo("Asia/Shanghai")).replace(tzinfo=None) + max_allowed_time = now + timedelta(days=future_tolerance_days) + filtered_articles = [] + removed_count = 0 + + for article in data.get("article_data", []): + article_time = datetime.strptime(article["created"], "%Y-%m-%d %H:%M") + if article_time > max_allowed_time: + removed_count += 1 + logging.warning( + f"文章 {article['title']} 的时间 {article['created']} 超出当前时间 {future_tolerance_days} 天以上，已跳过显示" + ) + continue + filtered_articles.append(article) + + filtered_articles.sort(key=lambda item: datetime.strptime(item["created"], "%Y-%m-%d %H:%M"), reverse=True) + data["article_data"] = filtered_articles + if removed_count: + logging.info(f"已过滤 {removed_count} 篇未来时间异常的文章") + return data + + +def limit_large_dataset(result: dict, future_tolerance_days: int = 2) -> dict: + """Keep the existing data trimming strategy for very large datasets.""" + result = sort_articles_by_time(result, future_tolerance_days=future_tolerance_days) + article_data = result.get("article_data", []) + result["statistical_data"]["article_num"] = len(article_data) + + max_articles = 150 + if len(article_data) > max_articles: + logging.info("数据量较大，开始进行处理...") + top_authors = {article["author"] for article in article_data[:max_articles]} + filtered_articles = article_data[:max_articles] + [ + article for article in article_data[max_articles:] + if article["author"] in top_authors + ] + result["article_data"] = filtered_articles + result["statistical_data"]["article_num"] = len(filtered_articles) + logging.info(f"数据处理完成，保留 {len(filtered_articles)} 篇文章") + + return result + + +# Backward-compatible class names kept for legacy imports. +WebsiteFeedResolver = FeedResolver +WebsiteCrawler = SingleSiteCrawler +FriendCircleCrawler = FriendCircleCrawlService diff --git a/friend_circle_lite/crawler/single_site_legacy.py b/friend_circle_lite/crawler/single_site_legacy.py new file mode 100644 index 00000000000..91e7765be05 --- /dev/null +++ b/friend_circle_lite/crawler/single_site_legacy.py @@ -0,0 +1,141 @@ +"""Legacy-compatible single website helpers. + +The project now uses dedicated domain models and services, but these helpers are +kept as a compatibility layer because existing entrypoints still import them. +""" + +from __future__ import annotations + +import logging + +import requests + +from friend_circle_lite.crawler.feed_service import FeedDiscoveryService, FeedParserService, LatestArticleTracker +from friend_circle_lite.domain.models import CacheUpdate, Website + +def check_feed(blog_url, session): + """Return the discovered feed type and URL in the historical tuple format.""" + endpoint = FeedDiscoveryService(session).discover(blog_url) + if endpoint is None: + return ["none", blog_url] + return [endpoint.feed_type, endpoint.url] + +def parse_feed(url, session, count=5, blog_url=''): + """Parse a feed and return the historical dictionary structure.""" + articles = FeedParserService(session).parse(url, count=count, blog_url=blog_url) + return { + 'website_name': '', + 'author': articles[0].author if articles else '', + 'link': '', + 'articles': [article.to_tracking_dict() for article in articles], + } + +def process_friend(friend, session: requests.Session, count: int, specific_and_cache=None): + """Crawl one friend entry and return the historical result shape.""" + if specific_and_cache is None: + specific_and_cache = [] + + try: + website = Website.from_friend_item(friend) + except Exception: + logging.error(f"friend 数据格式不正确: {friend!r}") + return { + 'name': None, + 'status': 'error', + 'articles': [], + 'feed_url': None, + 'feed_type': 'none', + 'cache_update': CacheUpdate(action='none', name=None, url=None, reason='bad_friend_data').to_dict(), + 'source_used': 'none', + } + + rss_lookup = {entry['name']: entry for entry in specific_and_cache if entry.get('name') and entry.get('url')} + entry = rss_lookup.get(website.name) + + endpoint = None + cache_update = CacheUpdate(action='none', name=website.name) + if entry: + source = entry.get('source', 'unknown') + endpoint = {'feed_type': 'specific', 'url': entry['url'], 'source': source} + if source == 'manual': + logging.info(f"'{website.name}' 使用预设 RSS 源：{entry['url']}") + elif source == 'cache': + logging.info(f"'{website.name}' 使用缓存 RSS 源：{entry['url']}") + else: + logging.info(f"'{website.name}' 使用 RSS 源：{entry['url']} (来源: {source})") + else: + feed_type, feed_url = check_feed(website.url, session) + if feed_type != 'none' and feed_url: + endpoint = {'feed_type': feed_type, 'url': feed_url, 'source': 'auto'} + cache_update = CacheUpdate(action='set', name=website.name, url=feed_url, reason='auto_discovered') + logging.info(f"'{website.name}' 自动探测到 RSS：{feed_url}") + + articles = [] + parse_error = endpoint is not None + if endpoint: + parsed = FeedParserService(session).parse(endpoint['url'], count=count, blog_url=website.url) + articles = [ + { + 'title': article.title, + 'created': article.published, + 'link': article.link, + 'author': website.name, + 'avatar': website.avatar, + } + for article in parsed + ] + parse_error = not articles + + if parse_error and endpoint and endpoint['source'] in ('cache', 'unknown'): + logging.warning(f"'{website.name}' 缓存的 RSS 源无效，尝试重新探测...") + new_feed_type, new_feed_url = check_feed(website.url, session) + if new_feed_type != 'none' and new_feed_url: + reparsed = FeedParserService(session).parse(new_feed_url, count=count, blog_url=website.url) + articles = [ + { + 'title': article.title, + 'created': article.published, + 'link': article.link, + 'author': website.name, + 'avatar': website.avatar, + } + for article in reparsed + ] + if articles: + endpoint = {'feed_type': new_feed_type, 'url': new_feed_url, 'source': 'auto'} + cache_update = CacheUpdate(action='set', name=website.name, url=new_feed_url, reason='repair_cache') + logging.info(f"'{website.name}' 重新探测成功，更新缓存：{new_feed_url}") + else: + endpoint = None + cache_update = CacheUpdate(action='delete', name=website.name, url=None, reason='remove_invalid') + logging.warning(f"'{website.name}' 重新探测失败，删除无效缓存") + else: + endpoint = None + cache_update = CacheUpdate(action='delete', name=website.name, url=None, reason='remove_invalid') + logging.warning(f"'{website.name}' 未找到有效 RSS，删除无效缓存") + + return { + 'name': website.name, + 'status': 'active' if articles else 'error', + 'articles': articles, + 'feed_url': endpoint['url'] if endpoint else None, + 'feed_type': endpoint['feed_type'] if endpoint else 'none', + 'cache_update': cache_update.to_dict(), + 'source_used': endpoint['source'] if endpoint else 'none', + } + +def get_latest_articles_from_link(url, count=5, last_articles_path="./temp/newest_posts.json"): + """Return newly published articles relative to the last local snapshot.""" + session = requests.Session() + feed_type, feed_url = check_feed(url, session) + if feed_type == 'none': + logging.error(f"无法获取 {url} 的文章数据") + return None + + latest_articles = FeedParserService(session).parse(feed_url, count=count, blog_url=url) + updated_articles = LatestArticleTracker(last_articles_path).diff_and_persist(latest_articles) + logging.info( + f"从 {url} 获取到 {len(latest_articles)} 篇文章，其中 {0 if updated_articles is None else len(updated_articles)} 篇为新文章" + ) + return updated_articles + diff --git a/friend_circle_lite/crawler_service.py b/friend_circle_lite/crawler_service.py index 5a215920f96..b81044a1c77 100644 --- a/friend_circle_lite/crawler_service.py +++ b/friend_circle_lite/crawler_service.py @@ -1,369 +1,6 @@ -"""High-level crawler orchestration. +"""Backward-compatible crawler service exports. -This module contains the system-level services that coordinate website loading, -RSS discovery, parsing, cache updates, result aggregation, and legacy output -formatting. +New code should import from `friend_circle_lite.crawler.service`. """ -from __future__ import annotations - -import logging -from concurrent.futures import ThreadPoolExecutor, as_completed -from datetime import datetime, timedelta -from zoneinfo import ZoneInfo - -import requests - -from friend_circle_lite import HEADERS_JSON, timeout -from friend_circle_lite.app_config import LinkCheckConfig, ProxySettings -from friend_circle_lite.cache_store import FeedCacheStore, LinkCheckStore -from friend_circle_lite.feed_service import FeedDiscoveryService, FeedParserService -from friend_circle_lite.link_check_service import LinkCheckService -from friend_circle_lite.models import Article, CacheRecord, CacheUpdate, CrawlResult, CrawlStatistics, FeedEndpoint, LinkCheckRecord, Website - - -class WebsiteFeedResolver: - """Resolve which feed endpoint should be used for a website. - - Resolution order is kept compatible with the previous implementation: - manual configuration first, then cache, and finally automatic discovery. - """ - - def __init__(self, discovery_service: FeedDiscoveryService, configured_feeds: list[CacheRecord]): - self.discovery_service = discovery_service - self.feed_lookup = {item.name: item for item in configured_feeds} - - def resolve(self, website: Website) -> FeedEndpoint | None: - configured = self.feed_lookup.get(website.name) - if configured: - if configured.source == 'manual': - logging.info(f"'{website.name}' 使用预设 RSS 源：{configured.url}") - elif configured.source == 'cache': - logging.info(f"'{website.name}' 使用缓存 RSS 源：{configured.url}") - else: - logging.info(f"'{website.name}' 使用 RSS 源：{configured.url} (来源: {configured.source})") - return FeedEndpoint(url=configured.url, feed_type="specific", source=configured.source) - - discovered = self.discovery_service.discover(website.url) - if discovered: - logging.info(f"'{website.name}' 自动探测到 RSS：{discovered.url}") - return discovered - - -class WebsiteCrawler: - """Crawl one website and produce a normalized result.""" - - def __init__(self, parser_service: FeedParserService, resolver: WebsiteFeedResolver): - self.parser_service = parser_service - self.resolver = resolver - - def crawl(self, website: Website, count: int) -> CrawlResult: - """Crawl one website while preserving legacy cache repair behavior.""" - endpoint = self.resolver.resolve(website) - cache_update = CacheUpdate(action="none", name=website.name) - - if endpoint and endpoint.source == "auto": - cache_update = CacheUpdate(action="set", name=website.name, url=endpoint.url, reason="auto_discovered") - - articles = self._parse_articles(endpoint, website, count) - parse_error = endpoint is not None and not articles - - if parse_error and endpoint and endpoint.source in ("cache", "unknown"): - logging.warning(f"'{website.name}' 缓存的 RSS 源无效，尝试重新探测...") - rediscovered = self.resolver.discovery_service.discover(website.url) - if rediscovered: - articles = self._parse_articles(rediscovered, website, count) - if articles: - endpoint = rediscovered - cache_update = CacheUpdate(action="set", name=website.name, url=rediscovered.url, reason="repair_cache") - logging.info(f"'{website.name}' 重新探测成功，更新缓存：{rediscovered.url}") - else: - endpoint = None - cache_update = CacheUpdate(action="delete", name=website.name, url=None, reason="remove_invalid") - logging.warning(f"'{website.name}' 重新探测失败，删除无效缓存") - else: - endpoint = None - cache_update = CacheUpdate(action="delete", name=website.name, url=None, reason="remove_invalid") - logging.warning(f"'{website.name}' 未找到有效 RSS，删除无效缓存") - - status = "active" if articles else "error" - if not articles: - if endpoint is None: - logging.warning(f"'{website.name}' 的博客 {website.url} 未找到有效 RSS ") - else: - logging.warning(f"'{website.name}' 的 RSS {endpoint.url} 未解析出文章 ") - - return CrawlResult( - website=website, - status=status, - articles=articles, - feed_url=endpoint.url if endpoint else None, - feed_type=endpoint.feed_type if endpoint else "none", - source_used=endpoint.source if endpoint else "none", - cache_update=cache_update, - ) - - def _parse_articles(self, endpoint: FeedEndpoint | None, website: Website, count: int) -> list[Article]: - if endpoint is None: - return [] - - articles = self.parser_service.parse(endpoint.url, count=count, blog_url=website.url) - for article in articles: - article.author = website.name - article.avatar = website.avatar - logging.info(f"{website.name} 发布了新文章：{article.title}，时间：{article.published}，链接：{article.link}") - return articles - - -class FriendCircleCrawler: - """System-level orchestrator for crawling all configured websites.""" - - def __init__( - self, - json_url: str, - count: int, - specific_rss: list[dict] | None = None, - cache_file: str | None = None, - link_check_config: LinkCheckConfig | None = None, - proxy_settings: ProxySettings | None = None, - ): - self.json_url = json_url - self.count = count - self.specific_rss = specific_rss or [] - self.cache_store = FeedCacheStore(cache_file) - self.link_check_config = link_check_config or LinkCheckConfig(enable=False) - self.proxy_settings = proxy_settings or ProxySettings() - self.link_check_store = LinkCheckStore(cache_file) - - def run(self) -> tuple[dict, list[list[str]]] | None: - """Fetch website list, crawl all websites, and build public outputs.""" - session = requests.Session() - websites = self._load_websites(session) - if websites is None: - return None - - link_check_records = self._check_links(websites) - link_check_map = {record.url: record for record in link_check_records} - crawlable_websites = [website for website in websites if link_check_map.get(website.url, LinkCheckRecord.unchecked(website)).crawl_allowed] - skipped_count = len(websites) - len(crawlable_websites) - if skipped_count: - logging.info(f"🔎 根据友链可达性检测跳过 {skipped_count} 个不可抓取站点") - - cache_records = self.cache_store.load_records() - manual_records = self._build_manual_records() - merged_records = self._merge_feed_records(cache_records, manual_records) - manual_names = {record.name for record in manual_records} - - discovery_service = FeedDiscoveryService(session) - parser_service = FeedParserService(session) - resolver = WebsiteFeedResolver(discovery_service=discovery_service, configured_feeds=merged_records) - crawler = WebsiteCrawler(parser_service=parser_service, resolver=resolver) - - crawl_results: list[CrawlResult] = [] - with ThreadPoolExecutor(max_workers=10) as executor: - future_to_website = { - executor.submit(crawler.crawl, website, self.count): website - for website in crawlable_websites - } - for future in as_completed(future_to_website): - website = future_to_website[future] - try: - crawl_results.append(future.result()) - except Exception as exc: - logging.error(f"处理 {website.to_error_payload()} 时发生错误: {exc}", exc_info=True) - crawl_results.append(CrawlResult(website=website, status="error")) - - self._apply_cache_updates(cache_records, crawl_results, manual_names) - - active_results = [result for result in crawl_results if result.status == "active"] - unreachable_results = [record for record in link_check_records if not record.reachable] - crawl_error_results = [result.website.to_error_payload() for result in crawl_results if result.status != "active"] - error_results = [[record.name, record.url, record.avatar] for record in unreachable_results] - all_articles = [article.to_public_dict() for result in active_results for article in result.articles] - - statistics = CrawlStatistics.create( - friends_num=len(websites), - active_num=len(active_results), - error_num=len(websites) - len(active_results), - article_num=len(all_articles), - ) - stats_payload = statistics.to_dict() - stats_payload.update(self._build_link_statistics(link_check_records)) - result = { - "statistical_data": stats_payload, - "article_data": all_articles, - } - link_payload = self._build_link_payload(link_check_records) - logging.info( - f"数据处理完成，总共有 {len(websites)} 位朋友，其中 {len(active_results)} 位博客可抓取到文章，" - f"{len(crawl_error_results)} 位博客 RSS 抓取失败，{len(unreachable_results)} 位友链不可达。" - ) - return result, error_results, link_payload - - def _check_links(self, websites: list[Website]) -> list[LinkCheckRecord]: - service = LinkCheckService(config=self.link_check_config, proxy_settings=self.proxy_settings, store=self.link_check_store) - return service.check_websites(websites) - - @staticmethod - def _build_link_statistics(records: list[LinkCheckRecord]) -> dict[str, int | str]: - reachable = [record for record in records if record.reachable] - crawl_allowed = [record for record in records if record.crawl_allowed] - api_only = [record for record in records if record.best_method == "api"] - has_author_link = [record for record in records if record.has_author_link] - checked_times = [record.checked_at for record in records if record.checked_at] - return { - "link_total_num": len(records), - "link_reachable_num": len(reachable), - "link_unreachable_num": len(records) - len(reachable), - "crawl_allowed_num": len(crawl_allowed), - "api_only_num": len(api_only), - "has_author_link_num": len(has_author_link), - "link_last_checked_time": max(checked_times) if checked_times else "", - } - - @staticmethod - def _build_link_payload(records: list[LinkCheckRecord]) -> dict[str, object]: - return { - "statistical_data": FriendCircleCrawler._build_link_statistics(records), - "link_data": [record.to_link_dict() for record in records], - } - - @staticmethod - def _build_friend_data( - websites: list[Website], - crawl_results: list[CrawlResult], - link_check_map: dict[str, LinkCheckRecord], - ) -> list[dict[str, object]]: - crawl_result_map = {result.website.url: result for result in crawl_results} - friend_data: list[dict[str, object]] = [] - for website in websites: - link_record = link_check_map.get(website.url) or LinkCheckRecord.unchecked(website) - crawl_result = crawl_result_map.get(website.url) - friend_data.append({ - "name": website.name, - "url": website.url, - "avatar": website.avatar, - "linkpage": website.linkpage, - "reachable": link_record.reachable, - "crawl_allowed": link_record.crawl_allowed, - "best_method": link_record.best_method, - "best_latency": link_record.best_latency, - "fail_count": link_record.fail_count, - "backlink_checked": link_record.backlink_checked, - "has_author_link": link_record.has_author_link, - "rss_crawl_reason": link_record.rss_crawl_reason, - "feed_status": crawl_result.status if crawl_result else "skipped", - "feed_url": crawl_result.feed_url if crawl_result else None, - "feed_type": crawl_result.feed_type if crawl_result else "none", - "article_count": len(crawl_result.articles) if crawl_result else 0, - }) - return friend_data - - def _load_websites(self, session: requests.Session) -> list[Website] | None: - try: - response = session.get(self.json_url, headers=HEADERS_JSON, timeout=timeout) - response.raise_for_status() - friends_data = response.json() - except Exception as exc: - logging.error(f"无法获取链接：{self.json_url} ：{exc}", exc_info=True) - return None - - websites: list[Website] = [] - for friend in friends_data.get("friends", []): - try: - websites.append(Website.from_friend_item(friend)) - except Exception: - logging.warning(f"发现格式异常的友链数据，已跳过: {friend!r}") - return websites - - def _build_manual_records(self) -> list[CacheRecord]: - manual_records: list[CacheRecord] = [] - for item in self.specific_rss: - if isinstance(item, dict) and item.get("name") and item.get("url"): - manual_records.append(CacheRecord(name=item["name"], url=item["url"], source="manual")) - return manual_records - - @staticmethod - def _merge_feed_records(cache_records: list[CacheRecord], manual_records: list[CacheRecord]) -> list[CacheRecord]: - merged = {record.name: record for record in cache_records} - for record in manual_records: - merged[record.name] = record - return list(merged.values()) - - def _apply_cache_updates(self, cache_records: list[CacheRecord], crawl_results: list[CrawlResult], manual_names: set[str]) -> None: - cache_map = {record.name: record for record in cache_records} - unique_updates: dict[str, CacheUpdate] = {} - - for result in crawl_results: - update = result.cache_update - if not update.name or update.action == "none" or update.name in manual_names: - continue - if update.action == "set" and update.url: - unique_updates[update.name] = update - elif update.action == "delete": - unique_updates[update.name] = update - - for name, update in unique_updates.items(): - if update.action == "set" and update.url: - cache_map[name] = CacheRecord(name=name, url=update.url, source="cache") - if update.reason == "auto_discovered": - logging.info(f"💾 缓存新增：{name} -> {update.url} (自动探测)") - elif update.reason == "repair_cache": - logging.info(f"💾 缓存修复：{name} -> {update.url} (重新探测)") - else: - logging.info(f"💾 缓存更新：{name} -> {update.url} ({update.reason})") - elif update.action == "delete" and name in cache_map: - cache_map.pop(name) - logging.info(f"🗑️ 缓存删除：{name} (RSS 源失效)") - - self.cache_store.save_records(list(cache_map.values())) - - -def sort_articles_by_time(data: dict, future_tolerance_days: int = 2) -> dict: - """Sort article payloads by time and remove far-future timestamps.""" - for article in data.get("article_data", []): - if not article.get("created"): - article["created"] = "2024-01-01 00:00" - logging.warning(f"文章 {article['title']} 未包含时间信息，已设置为默认时间 2024-01-01 00:00") - - now = datetime.now(ZoneInfo("Asia/Shanghai")).replace(tzinfo=None) - max_allowed_time = now + timedelta(days=future_tolerance_days) - filtered_articles = [] - removed_count = 0 - - for article in data.get("article_data", []): - article_time = datetime.strptime(article["created"], "%Y-%m-%d %H:%M") - if article_time > max_allowed_time: - removed_count += 1 - logging.warning( - f"文章 {article['title']} 的时间 {article['created']} 超出当前时间 {future_tolerance_days} 天以上，已跳过显示" - ) - continue - filtered_articles.append(article) - - filtered_articles.sort(key=lambda item: datetime.strptime(item["created"], "%Y-%m-%d %H:%M"), reverse=True) - data["article_data"] = filtered_articles - if removed_count: - logging.info(f"已过滤 {removed_count} 篇未来时间异常的文章") - return data - - -def limit_large_dataset(result: dict, future_tolerance_days: int = 2) -> dict: - """Keep the existing data trimming strategy for very large datasets.""" - result = sort_articles_by_time(result, future_tolerance_days=future_tolerance_days) - article_data = result.get("article_data", []) - result["statistical_data"]["article_num"] = len(article_data) - - max_articles = 150 - if len(article_data) > max_articles: - logging.info("数据量较大，开始进行处理...") - top_authors = {article["author"] for article in article_data[:max_articles]} - filtered_articles = article_data[:max_articles] + [ - article for article in article_data[max_articles:] - if article["author"] in top_authors - ] - result["article_data"] = filtered_articles - result["statistical_data"]["article_num"] = len(filtered_articles) - logging.info(f"数据处理完成，保留 {len(filtered_articles)} 篇文章") - - return result +from friend_circle_lite.crawler.service import * # noqa: F401,F403 diff --git a/friend_circle_lite/domain/__init__.py b/friend_circle_lite/domain/__init__.py new file mode 100644 index 00000000000..a9dec7356c1 --- /dev/null +++ b/friend_circle_lite/domain/__init__.py @@ -0,0 +1,3 @@ +"""Domain models used by crawlers, storage, and output builders.""" + +from friend_circle_lite.domain.models import * # noqa: F401,F403 diff --git a/friend_circle_lite/domain/models.py b/friend_circle_lite/domain/models.py new file mode 100644 index 00000000000..b8ab4390232 --- /dev/null +++ b/friend_circle_lite/domain/models.py @@ -0,0 +1,267 @@ +"""Domain models for Friend-Circle-Lite. + +These models centralize the core concepts used across the crawler so that the +transport layer, parsing logic, cache logic, and output formatting can evolve +independently. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from zoneinfo import ZoneInfo + + +SHANGHAI_TZ = ZoneInfo("Asia/Shanghai") + + +@dataclass(slots=True) +class Website: + """Represents a friend website entry from the upstream friend list.""" + + name: str + url: str + avatar: str = "" + linkpage: str = "" + + @classmethod + def from_friend_item(cls, raw_friend: list | tuple | dict) -> "Website": + """Create a website from common friend link structures.""" + if isinstance(raw_friend, dict): + return cls( + name=str(raw_friend.get("name", "")).strip(), + url=str(raw_friend.get("link") or raw_friend.get("url") or "").strip(), + avatar=str(raw_friend.get("avatar", "")).strip(), + linkpage=str(raw_friend.get("linkpage", "")).strip(), + ) + + name = raw_friend[0] + url = raw_friend[1] + if len(raw_friend) > 3: + linkpage = raw_friend[2] + avatar = raw_friend[3] + else: + linkpage = "" + avatar = raw_friend[2] if len(raw_friend) > 2 else "" + return cls(name=str(name).strip(), url=str(url).strip(), avatar=str(avatar or "").strip(), linkpage=str(linkpage or "").strip()) + + def to_error_payload(self) -> list[str]: + """Return the legacy structure used by `errors.json`.""" + return [self.name, self.url, self.avatar] + + def to_public_dict(self) -> dict[str, str]: + return { + "name": self.name, + "url": self.url, + "avatar": self.avatar, + "linkpage": self.linkpage, + } + + +@dataclass(slots=True) +class LinkMethodStatus: + """Status for one link-check method.""" + + success: bool = False + status_code: int | None = None + latency: float = -1 + + def to_dict(self) -> dict[str, bool | int | float | None]: + return { + "success": self.success, + "status_code": self.status_code, + "latency": self.latency, + } + + +@dataclass(slots=True) +class LinkCheckRecord: + """Reachability status for one friend website.""" + + name: str + url: str + avatar: str = "" + linkpage: str = "" + checked_at: str = "" + reachable: bool = False + crawl_allowed: bool = False + best_method: str = "none" + best_latency: float = -1 + fail_count: int = 0 + backlink_checked: bool = False + has_author_link: bool = False + rss_crawl_reason: str = "blocked_unreachable" + direct: LinkMethodStatus = field(default_factory=LinkMethodStatus) + proxy: LinkMethodStatus = field(default_factory=LinkMethodStatus) + api: LinkMethodStatus = field(default_factory=LinkMethodStatus) + + @classmethod + def unchecked(cls, website: Website, checked_at: str = "") -> "LinkCheckRecord": + return cls(name=website.name, url=website.url, avatar=website.avatar, linkpage=website.linkpage, checked_at=checked_at) + + def to_public_dict(self) -> dict[str, object]: + return { + "name": self.name, + "url": self.url, + "avatar": self.avatar, + "linkpage": self.linkpage, + "checked_at": self.checked_at, + "reachable": self.reachable, + "crawl_allowed": self.crawl_allowed, + "best_method": self.best_method, + "best_latency": self.best_latency, + "fail_count": self.fail_count, + "backlink_checked": self.backlink_checked, + "has_author_link": self.has_author_link, + "rss_crawl_reason": self.rss_crawl_reason, + "methods": { + "direct": self.direct.to_dict(), + "proxy": self.proxy.to_dict(), + "api": self.api.to_dict(), + }, + } + def to_link_dict(self) -> dict[str, object]: + return { + "name": self.name, + "link": self.url, + "link_page": self.linkpage, + "avatar": self.avatar, + "reachable": self.reachable, + "crawlable": self.crawl_allowed, + "method": self.best_method, + "latency": self.best_latency, + "fail_count": self.fail_count, + "checked_at": self.checked_at, + "has_backlink": self.has_author_link if self.backlink_checked else None, + "reason": self.rss_crawl_reason, + } + + +@dataclass(slots=True) +class Article: + """Represents one crawled article belonging to a website.""" + + title: str + author: str + link: str + published: str + summary: str = "" + content: str = "" + avatar: str = "" + + def to_public_dict(self) -> dict[str, str]: + """Return the legacy public article schema used by `all.json`.""" + return { + "title": self.title, + "created": self.published, + "link": self.link, + "author": self.author, + "avatar": self.avatar, + } + + def to_tracking_dict(self) -> dict[str, str]: + """Return the article schema used by the latest article tracker.""" + return { + "title": self.title, + "author": self.author, + "link": self.link, + "published": self.published, + "summary": self.summary, + "content": self.content, + } + + +@dataclass(slots=True) +class FeedEndpoint: + """Represents a concrete feed endpoint and how it was found.""" + + url: str + feed_type: str + source: str + + +@dataclass(slots=True) +class CacheRecord: + """Represents one cached RSS endpoint mapping for a website.""" + + name: str + url: str + source: str = "cache" + + def to_dict(self) -> dict[str, str]: + return { + "name": self.name, + "url": self.url, + } + + +@dataclass(slots=True) +class CacheUpdate: + """Describes how a crawl should update the persisted RSS cache.""" + + action: str = "none" + name: str | None = None + url: str | None = None + reason: str = "" + + def to_dict(self) -> dict[str, str | None]: + return { + "action": self.action, + "name": self.name, + "url": self.url, + "reason": self.reason, + } + + +@dataclass(slots=True) +class CrawlResult: + """Represents the crawl result for a single website.""" + + website: Website + status: str + articles: list[Article] = field(default_factory=list) + feed_url: str | None = None + feed_type: str = "none" + source_used: str = "none" + cache_update: CacheUpdate = field(default_factory=CacheUpdate) + + def to_legacy_dict(self) -> dict[str, object]: + return { + "name": self.website.name, + "status": self.status, + "articles": [article.to_public_dict() for article in self.articles], + "feed_url": self.feed_url, + "feed_type": self.feed_type, + "cache_update": self.cache_update.to_dict(), + "source_used": self.source_used, + } + + +@dataclass(slots=True) +class CrawlStatistics: + """Aggregated crawl statistics for the generated `all.json` output.""" + + friends_num: int = 0 + active_num: int = 0 + error_num: int = 0 + article_num: int = 0 + last_updated_time: str = "" + + @classmethod + def create(cls, friends_num: int, active_num: int, error_num: int, article_num: int) -> "CrawlStatistics": + return cls( + friends_num=friends_num, + active_num=active_num, + error_num=error_num, + article_num=article_num, + last_updated_time=datetime.now(SHANGHAI_TZ).strftime("%Y-%m-%d %H:%M:%S"), + ) + + def to_dict(self) -> dict[str, int | str]: + return { + "friends_num": self.friends_num, + "active_num": self.active_num, + "error_num": self.error_num, + "article_num": self.article_num, + "last_updated_time": self.last_updated_time, + } diff --git a/friend_circle_lite/feed_service.py b/friend_circle_lite/feed_service.py index 7714e7682c1..870a7e8c1e3 100644 --- a/friend_circle_lite/feed_service.py +++ b/friend_circle_lite/feed_service.py @@ -1,256 +1,6 @@ -"""Feed discovery, parsing, and incremental tracking services.""" +"""Backward-compatible feed service exports. -from __future__ import annotations +New code should import from `friend_circle_lite.crawler.feed_service`. +""" -import json -import logging -from datetime import datetime -from pathlib import Path -from urllib.parse import urlparse - -import feedparser -import requests - -from friend_circle_lite import HEADERS_XML, timeout -from friend_circle_lite.models import Article, FeedEndpoint, Website -from friend_circle_lite.utils.time import format_published_time -from friend_circle_lite.utils.url import replace_non_domain - - -class FeedDiscoveryService: - """Discover an RSS or Atom endpoint for a website.""" - - POSSIBLE_FEEDS = [ - ("rss1", "/feed"), # WordPress / 最常见 - ("rss2", "/feed/"), # WordPress 兼容写法 - ("rss3", "/rss.xml"), # 很多传统站点 - ("rss4", "/atom.xml"), # 静态博客常见（Hugo / Jekyll） - ("rss5", "/feed.xml"), # 通用型 - ("rss6", "/index.xml"), # Hugo / 一些静态站 - ("rss7", "/feed.atom"), # Atom 明确路径 - ("rss8", "/rss2.xml"), # 老系统遗留 - ("rss9", "/rss/feed.xml"),# 少见但存在 - ("rss10", "/rss.php"), # 老 PHP 程序 - ("rss11", "/feed.php"), # 同上 - ] - - def __init__(self, session: requests.Session): - self.session = session - - def discover(self, website_url: str) -> FeedEndpoint | None: - """Try common feed endpoints and return the first valid match.""" - for feed_type, path in self.POSSIBLE_FEEDS: - feed_url = website_url.rstrip("/") + path - try: - response = self.session.get(feed_url, headers=HEADERS_XML, timeout=timeout) - except requests.RequestException: - continue - - if response.status_code != 200: - continue - - content_type = response.headers.get("Content-Type", "").lower() - if "xml" in content_type or "rss" in content_type or "atom" in content_type: - return FeedEndpoint(url=feed_url, feed_type=feed_type, source="auto") - - text_head = response.text[:1000].lower() - if " list[Article]: - """Parse a feed URL and return the newest `count` articles. - - The returned articles are normalized to the project's internal domain - model, while preserving the original public output fields. - """ - try: - response = self.session.get(feed_url, headers=HEADERS_XML, timeout=timeout) - # 强制使用 UTF-8 编码，因为 apparent_encoding 可能检测错误 - response.encoding = "utf-8" - feed = feedparser.parse(response.text) - except Exception as exc: - logging.error(f"解析 RSS 失败：{feed_url}，错误: {exc}") - return [] - - default_author = feed.feed.author if "author" in feed.feed else "" - articles: list[Article] = [] - - for entry in feed.entries: - published = self._extract_published_time(entry) - article_link = replace_non_domain(entry.link, blog_url) if "link" in entry else "" - article = Article( - title=entry.title if "title" in entry else "", - author=default_author, - link=article_link, - published=published, - summary=entry.summary if "summary" in entry else "", - content=entry.content[0].value if "content" in entry and entry.content else entry.description if "description" in entry else "", - ) - articles.append(article) - - valid_articles = [article for article in articles if article.published] - - # 过滤掉无法解析的日期格式 - def safe_parse_date(article): - try: - return datetime.strptime(article.published, "%Y-%m-%d %H:%M") - except ValueError: - logging.warning(f"文章 {article.title} 的发布时间格式异常: {article.published}，已跳过") - return None - - # 只保留能成功解析日期的文章 - valid_articles_with_dates = [] - for article in valid_articles: - parsed_date = safe_parse_date(article) - if parsed_date: - valid_articles_with_dates.append((article, parsed_date)) - - # 按日期排序 - valid_articles_with_dates.sort(key=lambda item: item[1], reverse=True) - sorted_articles = [item[0] for item in valid_articles_with_dates] - - return sorted_articles[:count] if count < len(sorted_articles) else sorted_articles - - @staticmethod - def _extract_published_time(entry) -> str: - """Extract a normalized publish time from a feed entry.""" - import time - - def convert_time_to_string(time_value): - """Convert various time formats to string.""" - if isinstance(time_value, str): - return time_value - elif isinstance(time_value, time.struct_time): - # 检查年份是否异常 - if time_value.tm_year < 1900: - logging.warning(f"文章 {entry.get('title', 'Unknown')} 的时间年份异常: {time_value.tm_year}，已跳过") - return "" - return time.strftime('%Y-%m-%dT%H:%M:%SZ', time_value) - else: - logging.warning(f"文章 {entry.get('title', 'Unknown')} 的时间格式未知: {type(time_value)}，已跳过") - return "" - - if "published" in entry: - time_str = convert_time_to_string(entry.published) - if not time_str: - return "" - return format_published_time(time_str) - if "updated" in entry: - time_str = convert_time_to_string(entry.updated) - if not time_str: - return "" - published = format_published_time(time_str) - logging.warning(f"文章 {entry.title} 未包含发布时间，已使用更新时间 {published}") - return published - - logging.warning(f"文章 {entry.title} 未包含任何时间信息, 请检查原文, 跳过该文章") - return "" - - -class LatestArticleTracker: - """Track whether a website published new posts since the last crawl.""" - - def __init__(self, storage_path: str | Path, max_tracked_articles: int = 10): - from friend_circle_lite.cache_store import ArticleTrackingStore - self.store = ArticleTrackingStore(storage_path, max_tracked_articles) - - def diff_and_persist(self, latest_articles: list[Article]) -> list[dict] | None: - """Return newly seen articles and update the local storage. - - Returns None if: - - This is the first run (no previous data exists) - - No new articles are found - - New articles exist but are not newer than the most recent tracked article - """ - previous_articles = self.store.load_articles() - - # First run: no previous data exists, skip sending to prevent sending old articles - if not previous_articles: - logging.info(f"首次运行：跳过推送以防止发送旧文章") - self.store.save_articles(latest_articles) - return None - - previous_latest_date = self._get_latest_date(previous_articles) - - # Find articles that are truly new (check only: link, title, published) - new_articles = [] - for article in latest_articles: - if self._is_truly_new_article(article, previous_articles): - new_articles.append(article) - - if not new_articles: - self.store.save_articles(latest_articles) - return None - - # Filter new articles: only keep those newer than the previous latest date - truly_new_articles = [] - for article in new_articles: - if not article.published: - continue - try: - article_date = datetime.strptime(article.published, "%Y-%m-%d %H:%M") - if previous_latest_date is None or article_date > previous_latest_date: - truly_new_articles.append(article) - except Exception as exc: - logging.warning(f"解析文章日期失败: {article.title}, 日期: {article.published}, 错误: {exc}") - continue - - self.store.save_articles(latest_articles) - - if truly_new_articles: - logging.info(f"发现 {len(truly_new_articles)} 篇新文章（日期比之前更新）") - return [article.to_tracking_dict() for article in truly_new_articles] - else: - logging.info(f"发现 {len(new_articles)} 篇新文章，但日期不够新，跳过推送") - return None - - @staticmethod - def _is_truly_new_article(article: Article, previous_articles: list[Article]) -> bool: - """Check if an article is truly new by comparing link, title, and published date. - - An article is considered new only if its link, title, and published date - do not match any previous article (empty values are skipped). - """ - for prev in previous_articles: - # Check link, title, and published: if any non-empty field matches, it's not new - if article.link and article.link == prev.link: - return False - if article.title and article.title == prev.title: - return False - if article.published and article.published == prev.published: - return False - - return True - - @staticmethod - def _get_latest_date(articles: list[Article]) -> datetime | None: - """Find the latest publish date from a list of articles.""" - latest_date = None - for article in articles: - if not article.published: - continue - try: - article_date = datetime.strptime(article.published, "%Y-%m-%d %H:%M") - if latest_date is None or article_date > latest_date: - latest_date = article_date - except Exception: - continue - return latest_date - - -def extract_blog_origin(url: str) -> str: - """Return a normalized origin for display or author profile links.""" - parsed = urlparse(url) - if not parsed.scheme or not parsed.netloc: - return url - return f"{parsed.scheme}://{parsed.netloc}" +from friend_circle_lite.crawler.feed_service import * # noqa: F401,F403 diff --git a/friend_circle_lite/link_check_service.py b/friend_circle_lite/link_check_service.py index 3319b2b0136..02d19695318 100644 --- a/friend_circle_lite/link_check_service.py +++ b/friend_circle_lite/link_check_service.py @@ -1,284 +1,6 @@ -"""Friend link reachability checks used before RSS crawling.""" +"""Backward-compatible link checker exports. -from __future__ import annotations +New code should import from `friend_circle_lite.link_checker.service`. +""" -import logging -import time -from concurrent.futures import ThreadPoolExecutor, as_completed -from datetime import datetime -from urllib.parse import quote, urlparse - -import requests - -from friend_circle_lite.app_config import LinkCheckConfig, ProxySettings -from friend_circle_lite.cache_store import LinkCheckStore -from friend_circle_lite.models import LinkCheckRecord, LinkMethodStatus, Website - - -LINK_CHECK_HEADERS = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/123.0.0.0 Safari/537.36 " - "(Friend-Circle-Lite/2.0; +https://github.com/willow-god/Friend-Circle-Lite)" - ), - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "zh-CN,zh;q=0.9", - "Connection": "keep-alive", - "X-Friend-Circle-Link-Check": "1.0", -} - -RAW_HEADERS = { - "User-Agent": LINK_CHECK_HEADERS["User-Agent"], - "X-Friend-Circle-Link-Check": "1.0", -} - - -class LinkCheckService: - """Check friend homepage reachability and cache results.""" - - def __init__(self, config: LinkCheckConfig, proxy_settings: ProxySettings, store: LinkCheckStore): - self.config = config - self.proxy_settings = proxy_settings - self.store = store - - def check_websites(self, websites: list[Website]) -> list[LinkCheckRecord]: - if not self.config.enable: - now = self._now_text() - return [self._build_disabled_record(website, now) for website in websites] - - cached_records = self.store.load_records([website.url for website in websites]) - records_by_url: dict[str, LinkCheckRecord] = {} - websites_to_check: list[Website] = [] - - for website in websites: - cached = cached_records.get(website.url) - if cached and self._can_reuse_cached_record(cached, website): - records_by_url[website.url] = self._refresh_cached_metadata(cached, website) - else: - websites_to_check.append(website) - - if websites_to_check: - logging.info(f"🔎 开始检测 {len(websites_to_check)} 个友链可达性") - checked_records = self._check_fresh_websites(websites_to_check, cached_records) - self.store.save_records(checked_records) - for record in checked_records: - records_by_url[record.url] = record - else: - logging.info("🔎 友链可达性检测缓存仍有效，本次复用缓存结果") - - return [records_by_url.get(website.url) or LinkCheckRecord.unchecked(website) for website in websites] - - def _check_fresh_websites(self, websites: list[Website], cached_records: dict[str, LinkCheckRecord]) -> list[LinkCheckRecord]: - records: list[LinkCheckRecord] = [] - with requests.Session() as session: - with ThreadPoolExecutor(max_workers=max(1, self.config.max_workers)) as executor: - future_to_website = { - executor.submit(self._check_website, session, website, cached_records.get(website.url)): website - for website in websites - } - for future in as_completed(future_to_website): - website = future_to_website[future] - try: - records.append(future.result()) - except Exception as exc: - logging.warning(f"友链 {website.name} 检测失败: {exc}") - records.append(self._build_failed_record(website, cached_records.get(website.url))) - return records - - def _check_website(self, session: requests.Session, website: Website, cached: LinkCheckRecord | None) -> LinkCheckRecord: - direct = self._request_method(session, website.url, "直接访问") - proxy = LinkMethodStatus() - api = LinkMethodStatus() - - if not direct.success: - proxy_url = self._build_proxy_url(website.url) - if proxy_url: - proxy = self._request_method(session, proxy_url, "代理访问") - - if not direct.success and not proxy.success: - api = self._request_api(session, website.url) - time.sleep(0.2) - - record = self._compose_record(website, cached, direct, proxy, api) - if record.reachable and self.config.enable_backlink_check and self.config.author_url and website.linkpage: - record.backlink_checked = True - record.has_author_link = self._check_author_link_in_page(session, website.linkpage) - return record - - def _request_method(self, session: requests.Session, url: str, desc: str) -> LinkMethodStatus: - if not self._is_url(url): - return LinkMethodStatus() - - response, latency = self._request_url(session, url, headers=LINK_CHECK_HEADERS, desc=desc) - if response is None: - return LinkMethodStatus(success=False, status_code=None, latency=latency) - success = response.status_code == 200 - if success: - logging.info(f"[{desc}] 成功访问: {url}，延迟 {latency} 秒") - else: - logging.warning(f"[{desc}] 状态码异常: {url} -> {response.status_code}") - return LinkMethodStatus(success=success, status_code=response.status_code, latency=latency) - - def _request_api(self, session: requests.Session, url: str) -> LinkMethodStatus: - if not self.config.status_api_url: - return LinkMethodStatus() - - api_url = self.config.status_api_url.format(url=quote(url, safe="")) - response, latency = self._request_url(session, api_url, headers=RAW_HEADERS, desc="API 检查", timeout=30) - if response is None: - return LinkMethodStatus(success=False, status_code=None, latency=latency) - - try: - payload = response.json() - status_code = int(payload.get("data", 0)) - success = int(payload.get("code", 0)) == 200 and status_code == 200 - if success: - logging.info(f"[API] 成功访问: {url}，状态码 200") - else: - logging.warning(f"[API] 状态异常: {url} -> [{payload.get('code')}, {payload.get('data')}]") - return LinkMethodStatus(success=success, status_code=status_code, latency=latency) - except Exception as exc: - logging.warning(f"[API] 解析响应失败: {url}，错误: {exc}") - return LinkMethodStatus(success=False, status_code=response.status_code, latency=latency) - - def _compose_record( - self, - website: Website, - cached: LinkCheckRecord | None, - direct: LinkMethodStatus, - proxy: LinkMethodStatus, - api: LinkMethodStatus, - ) -> LinkCheckRecord: - reachable = direct.success or proxy.success or api.success - crawl_allowed = direct.success or proxy.success - if direct.success: - best_method = "direct" - best_latency = direct.latency - reason = "allowed_by_direct" - elif proxy.success: - best_method = "proxy" - best_latency = proxy.latency - reason = "allowed_by_proxy" - elif api.success: - best_method = "api" - best_latency = api.latency - reason = "blocked_api_only" - else: - best_method = "none" - best_latency = -1 - reason = "blocked_unreachable" - - fail_count = 0 if reachable else ((cached.fail_count if cached else 0) + 1) - return LinkCheckRecord( - name=website.name, - url=website.url, - avatar=website.avatar, - linkpage=website.linkpage, - checked_at=self._now_text(), - reachable=reachable, - crawl_allowed=crawl_allowed, - best_method=best_method, - best_latency=best_latency, - fail_count=fail_count, - rss_crawl_reason=reason, - direct=direct, - proxy=proxy, - api=api, - ) - - def _check_author_link_in_page(self, session: requests.Session, linkpage_url: str) -> bool: - response, _ = self._request_url(session, linkpage_url, headers=RAW_HEADERS, desc="友链页面检测") - if not response: - return False - - author_url = self.config.author_url - if not author_url.startswith(("http://", "https://")): - author_url = "https://" + author_url - - variants = { - author_url, - author_url.replace("https://", "http://"), - author_url.replace("https://", "//"), - author_url.replace("https://", ""), - self.config.author_url, - "//" + self.config.author_url, - "https://" + self.config.author_url, - "http://" + self.config.author_url, - } - content = response.text - for variant in variants: - if ( - f'href="{variant}"' in content - or f"href='{variant}'" in content - or f'href="{variant}/"' in content - or f"href='{variant}/'" in content - or variant in content - ): - return True - return False - - def _request_url( - self, - session: requests.Session, - url: str, - headers: dict[str, str], - desc: str, - timeout: int | None = None, - ) -> tuple[requests.Response | None, float]: - try: - start_time = time.time() - response = session.get(url, headers=headers, timeout=timeout or self.config.timeout) - return response, round(time.time() - start_time, 2) - except requests.RequestException as exc: - logging.warning(f"[{desc}] 请求失败: {url}，错误: {exc}") - return None, -1 - - def _can_reuse_cached_record(self, cached: LinkCheckRecord, website: Website) -> bool: - if self.config.enable_backlink_check and cached.linkpage != website.linkpage: - return False - return self.store.is_fresh(cached, self.config.max_age_hours) - - @staticmethod - def _refresh_cached_metadata(cached: LinkCheckRecord, website: Website) -> LinkCheckRecord: - cached.name = website.name - cached.avatar = website.avatar - cached.linkpage = website.linkpage - return cached - - def _build_failed_record(self, website: Website, cached: LinkCheckRecord | None) -> LinkCheckRecord: - record = LinkCheckRecord.unchecked(website, self._now_text()) - record.fail_count = (cached.fail_count if cached else 0) + 1 - return record - - @staticmethod - def _build_disabled_record(website: Website, checked_at: str) -> LinkCheckRecord: - return LinkCheckRecord( - name=website.name, - url=website.url, - avatar=website.avatar, - linkpage=website.linkpage, - checked_at=checked_at, - reachable=True, - crawl_allowed=True, - best_method="disabled", - best_latency=-1, - rss_crawl_reason="link_check_disabled", - ) - - def _build_proxy_url(self, url: str) -> str: - if not self.proxy_settings.proxy_url: - return "" - if "{}" in self.proxy_settings.proxy_url: - return self.proxy_settings.proxy_url.format(url) - if "{url}" in self.proxy_settings.proxy_url: - return self.proxy_settings.proxy_url.format(url=url) - return f"{self.proxy_settings.proxy_url}{url}" - - @staticmethod - def _is_url(path: str) -> bool: - return urlparse(path).scheme in ("http", "https") - - @staticmethod - def _now_text() -> str: - return datetime.now().strftime("%Y-%m-%d %H:%M:%S") +from friend_circle_lite.link_checker.service import * # noqa: F401,F403 diff --git a/friend_circle_lite/link_checker/__init__.py b/friend_circle_lite/link_checker/__init__.py new file mode 100644 index 00000000000..b607c3b2a24 --- /dev/null +++ b/friend_circle_lite/link_checker/__init__.py @@ -0,0 +1,3 @@ +"""Friend link reachability checks.""" + +from friend_circle_lite.link_checker.service import LinkReachabilityService diff --git a/friend_circle_lite/link_checker/service.py b/friend_circle_lite/link_checker/service.py new file mode 100644 index 00000000000..e6e21bd8c3e --- /dev/null +++ b/friend_circle_lite/link_checker/service.py @@ -0,0 +1,288 @@ +"""Friend link reachability checks used before RSS crawling.""" + +from __future__ import annotations + +import logging +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +from urllib.parse import quote, urlparse + +import requests + +from friend_circle_lite.config.models import LinkCheckConfig, ProxySettings +from friend_circle_lite.domain.models import LinkCheckRecord, LinkMethodStatus, Website +from friend_circle_lite.storage.sqlite_store import LinkCheckStore + + +LINK_CHECK_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/123.0.0.0 Safari/537.36 " + "(Friend-Circle-Lite/2.0; +https://github.com/willow-god/Friend-Circle-Lite)" + ), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "zh-CN,zh;q=0.9", + "Connection": "keep-alive", + "X-Friend-Circle-Link-Check": "1.0", +} + +RAW_HEADERS = { + "User-Agent": LINK_CHECK_HEADERS["User-Agent"], + "X-Friend-Circle-Link-Check": "1.0", +} + + +class LinkReachabilityService: + """Check friend homepage reachability and cache results.""" + + def __init__(self, config: LinkCheckConfig, proxy_settings: ProxySettings, store: LinkCheckStore): + self.config = config + self.proxy_settings = proxy_settings + self.store = store + + def check_websites(self, websites: list[Website]) -> list[LinkCheckRecord]: + if not self.config.enable: + now = self._now_text() + return [self._build_disabled_record(website, now) for website in websites] + + cached_records = self.store.load_records([website.url for website in websites]) + records_by_url: dict[str, LinkCheckRecord] = {} + websites_to_check: list[Website] = [] + + for website in websites: + cached = cached_records.get(website.url) + if cached and self._can_reuse_cached_record(cached, website): + records_by_url[website.url] = self._refresh_cached_metadata(cached, website) + else: + websites_to_check.append(website) + + if websites_to_check: + logging.info(f"🔎 开始检测 {len(websites_to_check)} 个友链可达性") + checked_records = self._check_fresh_websites(websites_to_check, cached_records) + self.store.save_records(checked_records) + for record in checked_records: + records_by_url[record.url] = record + else: + logging.info("🔎 友链可达性检测缓存仍有效，本次复用缓存结果") + + return [records_by_url.get(website.url) or LinkCheckRecord.unchecked(website) for website in websites] + + def _check_fresh_websites(self, websites: list[Website], cached_records: dict[str, LinkCheckRecord]) -> list[LinkCheckRecord]: + records: list[LinkCheckRecord] = [] + with requests.Session() as session: + with ThreadPoolExecutor(max_workers=max(1, self.config.max_workers)) as executor: + future_to_website = { + executor.submit(self._check_website, session, website, cached_records.get(website.url)): website + for website in websites + } + for future in as_completed(future_to_website): + website = future_to_website[future] + try: + records.append(future.result()) + except Exception as exc: + logging.warning(f"友链 {website.name} 检测失败: {exc}") + records.append(self._build_failed_record(website, cached_records.get(website.url))) + return records + + def _check_website(self, session: requests.Session, website: Website, cached: LinkCheckRecord | None) -> LinkCheckRecord: + direct = self._request_method(session, website.url, "直接访问") + proxy = LinkMethodStatus() + api = LinkMethodStatus() + + if not direct.success: + proxy_url = self._build_proxy_url(website.url) + if proxy_url: + proxy = self._request_method(session, proxy_url, "代理访问") + + if not direct.success and not proxy.success: + api = self._request_api(session, website.url) + time.sleep(0.2) + + record = self._compose_record(website, cached, direct, proxy, api) + if record.reachable and self.config.enable_backlink_check and self.config.author_url and website.linkpage: + record.backlink_checked = True + record.has_author_link = self._check_author_link_in_page(session, website.linkpage) + return record + + def _request_method(self, session: requests.Session, url: str, desc: str) -> LinkMethodStatus: + if not self._is_url(url): + return LinkMethodStatus() + + response, latency = self._request_url(session, url, headers=LINK_CHECK_HEADERS, desc=desc) + if response is None: + return LinkMethodStatus(success=False, status_code=None, latency=latency) + success = response.status_code == 200 + if success: + logging.info(f"[{desc}] 成功访问: {url}，延迟 {latency} 秒") + else: + logging.warning(f"[{desc}] 状态码异常: {url} -> {response.status_code}") + return LinkMethodStatus(success=success, status_code=response.status_code, latency=latency) + + def _request_api(self, session: requests.Session, url: str) -> LinkMethodStatus: + if not self.config.status_api_url: + return LinkMethodStatus() + + api_url = self.config.status_api_url.format(url=quote(url, safe="")) + response, latency = self._request_url(session, api_url, headers=RAW_HEADERS, desc="API 检查", timeout=30) + if response is None: + return LinkMethodStatus(success=False, status_code=None, latency=latency) + + try: + payload = response.json() + status_code = int(payload.get("data", 0)) + success = int(payload.get("code", 0)) == 200 and status_code == 200 + if success: + logging.info(f"[API] 成功访问: {url}，状态码 200") + else: + logging.warning(f"[API] 状态异常: {url} -> [{payload.get('code')}, {payload.get('data')}]") + return LinkMethodStatus(success=success, status_code=status_code, latency=latency) + except Exception as exc: + logging.warning(f"[API] 解析响应失败: {url}，错误: {exc}") + return LinkMethodStatus(success=False, status_code=response.status_code, latency=latency) + + def _compose_record( + self, + website: Website, + cached: LinkCheckRecord | None, + direct: LinkMethodStatus, + proxy: LinkMethodStatus, + api: LinkMethodStatus, + ) -> LinkCheckRecord: + reachable = direct.success or proxy.success or api.success + crawl_allowed = direct.success or proxy.success + if direct.success: + best_method = "direct" + best_latency = direct.latency + reason = "allowed_by_direct" + elif proxy.success: + best_method = "proxy" + best_latency = proxy.latency + reason = "allowed_by_proxy" + elif api.success: + best_method = "api" + best_latency = api.latency + reason = "blocked_api_only" + else: + best_method = "none" + best_latency = -1 + reason = "blocked_unreachable" + + fail_count = 0 if reachable else ((cached.fail_count if cached else 0) + 1) + return LinkCheckRecord( + name=website.name, + url=website.url, + avatar=website.avatar, + linkpage=website.linkpage, + checked_at=self._now_text(), + reachable=reachable, + crawl_allowed=crawl_allowed, + best_method=best_method, + best_latency=best_latency, + fail_count=fail_count, + rss_crawl_reason=reason, + direct=direct, + proxy=proxy, + api=api, + ) + + def _check_author_link_in_page(self, session: requests.Session, linkpage_url: str) -> bool: + response, _ = self._request_url(session, linkpage_url, headers=RAW_HEADERS, desc="友链页面检测") + if not response: + return False + + author_url = self.config.author_url + if not author_url.startswith(("http://", "https://")): + author_url = "https://" + author_url + + variants = { + author_url, + author_url.replace("https://", "http://"), + author_url.replace("https://", "//"), + author_url.replace("https://", ""), + self.config.author_url, + "//" + self.config.author_url, + "https://" + self.config.author_url, + "http://" + self.config.author_url, + } + content = response.text + for variant in variants: + if ( + f'href="{variant}"' in content + or f"href='{variant}'" in content + or f'href="{variant}/"' in content + or f"href='{variant}/'" in content + or variant in content + ): + return True + return False + + def _request_url( + self, + session: requests.Session, + url: str, + headers: dict[str, str], + desc: str, + timeout: int | None = None, + ) -> tuple[requests.Response | None, float]: + try: + start_time = time.time() + response = session.get(url, headers=headers, timeout=timeout or self.config.timeout) + return response, round(time.time() - start_time, 2) + except requests.RequestException as exc: + logging.warning(f"[{desc}] 请求失败: {url}，错误: {exc}") + return None, -1 + + def _can_reuse_cached_record(self, cached: LinkCheckRecord, website: Website) -> bool: + if self.config.enable_backlink_check and cached.linkpage != website.linkpage: + return False + return self.store.is_fresh(cached, self.config.max_age_hours) + + @staticmethod + def _refresh_cached_metadata(cached: LinkCheckRecord, website: Website) -> LinkCheckRecord: + cached.name = website.name + cached.avatar = website.avatar + cached.linkpage = website.linkpage + return cached + + def _build_failed_record(self, website: Website, cached: LinkCheckRecord | None) -> LinkCheckRecord: + record = LinkCheckRecord.unchecked(website, self._now_text()) + record.fail_count = (cached.fail_count if cached else 0) + 1 + return record + + @staticmethod + def _build_disabled_record(website: Website, checked_at: str) -> LinkCheckRecord: + return LinkCheckRecord( + name=website.name, + url=website.url, + avatar=website.avatar, + linkpage=website.linkpage, + checked_at=checked_at, + reachable=True, + crawl_allowed=True, + best_method="disabled", + best_latency=-1, + rss_crawl_reason="link_check_disabled", + ) + + def _build_proxy_url(self, url: str) -> str: + if not self.proxy_settings.proxy_url: + return "" + if "{}" in self.proxy_settings.proxy_url: + return self.proxy_settings.proxy_url.format(url) + if "{url}" in self.proxy_settings.proxy_url: + return self.proxy_settings.proxy_url.format(url=url) + return f"{self.proxy_settings.proxy_url}{url}" + + @staticmethod + def _is_url(path: str) -> bool: + return urlparse(path).scheme in ("http", "https") + + @staticmethod + def _now_text() -> str: + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + +# Backward-compatible class name kept for legacy imports. +LinkCheckService = LinkReachabilityService diff --git a/friend_circle_lite/models.py b/friend_circle_lite/models.py index b8ab4390232..80e41043c64 100644 --- a/friend_circle_lite/models.py +++ b/friend_circle_lite/models.py @@ -1,267 +1,6 @@ -"""Domain models for Friend-Circle-Lite. +"""Backward-compatible domain model exports. -These models centralize the core concepts used across the crawler so that the -transport layer, parsing logic, cache logic, and output formatting can evolve -independently. +New code should import from `friend_circle_lite.domain.models`. """ -from __future__ import annotations - -from dataclasses import dataclass, field -from datetime import datetime -from zoneinfo import ZoneInfo - - -SHANGHAI_TZ = ZoneInfo("Asia/Shanghai") - - -@dataclass(slots=True) -class Website: - """Represents a friend website entry from the upstream friend list.""" - - name: str - url: str - avatar: str = "" - linkpage: str = "" - - @classmethod - def from_friend_item(cls, raw_friend: list | tuple | dict) -> "Website": - """Create a website from common friend link structures.""" - if isinstance(raw_friend, dict): - return cls( - name=str(raw_friend.get("name", "")).strip(), - url=str(raw_friend.get("link") or raw_friend.get("url") or "").strip(), - avatar=str(raw_friend.get("avatar", "")).strip(), - linkpage=str(raw_friend.get("linkpage", "")).strip(), - ) - - name = raw_friend[0] - url = raw_friend[1] - if len(raw_friend) > 3: - linkpage = raw_friend[2] - avatar = raw_friend[3] - else: - linkpage = "" - avatar = raw_friend[2] if len(raw_friend) > 2 else "" - return cls(name=str(name).strip(), url=str(url).strip(), avatar=str(avatar or "").strip(), linkpage=str(linkpage or "").strip()) - - def to_error_payload(self) -> list[str]: - """Return the legacy structure used by `errors.json`.""" - return [self.name, self.url, self.avatar] - - def to_public_dict(self) -> dict[str, str]: - return { - "name": self.name, - "url": self.url, - "avatar": self.avatar, - "linkpage": self.linkpage, - } - - -@dataclass(slots=True) -class LinkMethodStatus: - """Status for one link-check method.""" - - success: bool = False - status_code: int | None = None - latency: float = -1 - - def to_dict(self) -> dict[str, bool | int | float | None]: - return { - "success": self.success, - "status_code": self.status_code, - "latency": self.latency, - } - - -@dataclass(slots=True) -class LinkCheckRecord: - """Reachability status for one friend website.""" - - name: str - url: str - avatar: str = "" - linkpage: str = "" - checked_at: str = "" - reachable: bool = False - crawl_allowed: bool = False - best_method: str = "none" - best_latency: float = -1 - fail_count: int = 0 - backlink_checked: bool = False - has_author_link: bool = False - rss_crawl_reason: str = "blocked_unreachable" - direct: LinkMethodStatus = field(default_factory=LinkMethodStatus) - proxy: LinkMethodStatus = field(default_factory=LinkMethodStatus) - api: LinkMethodStatus = field(default_factory=LinkMethodStatus) - - @classmethod - def unchecked(cls, website: Website, checked_at: str = "") -> "LinkCheckRecord": - return cls(name=website.name, url=website.url, avatar=website.avatar, linkpage=website.linkpage, checked_at=checked_at) - - def to_public_dict(self) -> dict[str, object]: - return { - "name": self.name, - "url": self.url, - "avatar": self.avatar, - "linkpage": self.linkpage, - "checked_at": self.checked_at, - "reachable": self.reachable, - "crawl_allowed": self.crawl_allowed, - "best_method": self.best_method, - "best_latency": self.best_latency, - "fail_count": self.fail_count, - "backlink_checked": self.backlink_checked, - "has_author_link": self.has_author_link, - "rss_crawl_reason": self.rss_crawl_reason, - "methods": { - "direct": self.direct.to_dict(), - "proxy": self.proxy.to_dict(), - "api": self.api.to_dict(), - }, - } - def to_link_dict(self) -> dict[str, object]: - return { - "name": self.name, - "link": self.url, - "link_page": self.linkpage, - "avatar": self.avatar, - "reachable": self.reachable, - "crawlable": self.crawl_allowed, - "method": self.best_method, - "latency": self.best_latency, - "fail_count": self.fail_count, - "checked_at": self.checked_at, - "has_backlink": self.has_author_link if self.backlink_checked else None, - "reason": self.rss_crawl_reason, - } - - -@dataclass(slots=True) -class Article: - """Represents one crawled article belonging to a website.""" - - title: str - author: str - link: str - published: str - summary: str = "" - content: str = "" - avatar: str = "" - - def to_public_dict(self) -> dict[str, str]: - """Return the legacy public article schema used by `all.json`.""" - return { - "title": self.title, - "created": self.published, - "link": self.link, - "author": self.author, - "avatar": self.avatar, - } - - def to_tracking_dict(self) -> dict[str, str]: - """Return the article schema used by the latest article tracker.""" - return { - "title": self.title, - "author": self.author, - "link": self.link, - "published": self.published, - "summary": self.summary, - "content": self.content, - } - - -@dataclass(slots=True) -class FeedEndpoint: - """Represents a concrete feed endpoint and how it was found.""" - - url: str - feed_type: str - source: str - - -@dataclass(slots=True) -class CacheRecord: - """Represents one cached RSS endpoint mapping for a website.""" - - name: str - url: str - source: str = "cache" - - def to_dict(self) -> dict[str, str]: - return { - "name": self.name, - "url": self.url, - } - - -@dataclass(slots=True) -class CacheUpdate: - """Describes how a crawl should update the persisted RSS cache.""" - - action: str = "none" - name: str | None = None - url: str | None = None - reason: str = "" - - def to_dict(self) -> dict[str, str | None]: - return { - "action": self.action, - "name": self.name, - "url": self.url, - "reason": self.reason, - } - - -@dataclass(slots=True) -class CrawlResult: - """Represents the crawl result for a single website.""" - - website: Website - status: str - articles: list[Article] = field(default_factory=list) - feed_url: str | None = None - feed_type: str = "none" - source_used: str = "none" - cache_update: CacheUpdate = field(default_factory=CacheUpdate) - - def to_legacy_dict(self) -> dict[str, object]: - return { - "name": self.website.name, - "status": self.status, - "articles": [article.to_public_dict() for article in self.articles], - "feed_url": self.feed_url, - "feed_type": self.feed_type, - "cache_update": self.cache_update.to_dict(), - "source_used": self.source_used, - } - - -@dataclass(slots=True) -class CrawlStatistics: - """Aggregated crawl statistics for the generated `all.json` output.""" - - friends_num: int = 0 - active_num: int = 0 - error_num: int = 0 - article_num: int = 0 - last_updated_time: str = "" - - @classmethod - def create(cls, friends_num: int, active_num: int, error_num: int, article_num: int) -> "CrawlStatistics": - return cls( - friends_num=friends_num, - active_num=active_num, - error_num=error_num, - article_num=article_num, - last_updated_time=datetime.now(SHANGHAI_TZ).strftime("%Y-%m-%d %H:%M:%S"), - ) - - def to_dict(self) -> dict[str, int | str]: - return { - "friends_num": self.friends_num, - "active_num": self.active_num, - "error_num": self.error_num, - "article_num": self.article_num, - "last_updated_time": self.last_updated_time, - } +from friend_circle_lite.domain.models import * # noqa: F401,F403 diff --git a/friend_circle_lite/notifications/__init__.py b/friend_circle_lite/notifications/__init__.py new file mode 100644 index 00000000000..738310168dc --- /dev/null +++ b/friend_circle_lite/notifications/__init__.py @@ -0,0 +1,4 @@ +"""Notification integrations for GitHub issue subscriptions and email.""" + +from friend_circle_lite.notifications.github import extract_emails_from_issues +from friend_circle_lite.notifications.mail import send_emails diff --git a/friend_circle_lite/notifications/github.py b/friend_circle_lite/notifications/github.py new file mode 100644 index 00000000000..2cbdc50a196 --- /dev/null +++ b/friend_circle_lite/notifications/github.py @@ -0,0 +1,39 @@ +import logging +import requests +import re +from friend_circle_lite import HEADERS_JSON + +def extract_emails_from_issues(api_url): + """ + 从GitHub issues API中提取以[e-mail]开头的title中的邮箱地址。 + + 参数： + api_url (str): GitHub issues API的URL。 + + 返回： + dict: 包含所有提取的邮箱地址的字典。 + { + "emails": [ + "3162475700@qq.com" + ] + } + """ + try: + response = requests.get(api_url, headers=HEADERS_JSON, timeout=10) + response.raise_for_status() + issues = response.json() + except Exception as e: + logging.error(f"无法获取 GitHub issues 数据，错误信息: {e}") + return None + + email_pattern = re.compile(r'^\[邮箱订阅\](.+)$') + emails = [] + + for issue in issues: + title = issue.get("title", "") + match = email_pattern.match(title) + if match: + email = match.group(1).strip() + emails.append(email) + + return {"emails": emails} \ No newline at end of file diff --git a/friend_circle_lite/notifications/mail.py b/friend_circle_lite/notifications/mail.py new file mode 100644 index 00000000000..bb7df1e8b92 --- /dev/null +++ b/friend_circle_lite/notifications/mail.py @@ -0,0 +1,255 @@ +import logging +import smtplib +import time +import os +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +from email.utils import formatdate, make_msgid, parseaddr +from jinja2 import Environment, FileSystemLoader + +# ============================================================ +# 内部工具 +# ============================================================ + +def _render_message( + target_email, + sender_email, + subject, + body, + template_path=None, + template_data=None, +): + """ + 构建 MIME 邮件对象，支持纯文本 + 可选 HTML。 + """ + msg = MIMEMultipart("alternative") + msg["From"] = sender_email + msg["To"] = target_email + msg["Subject"] = subject + msg["Date"] = formatdate(localtime=True) + domain = sender_email.split("@")[-1] if "@" in sender_email else "localhost" + msg["Message-ID"] = make_msgid(domain=domain) + + # 纯文本内容 + msg.attach(MIMEText(body or "", "plain", "utf-8")) + + # HTML 模板内容 + if template_path and template_data: + env = Environment(loader=FileSystemLoader(os.path.dirname(template_path))) + template = env.get_template(os.path.basename(template_path)) + html_content = template.render(template_data) + msg.attach(MIMEText(html_content, "html", "utf-8")) + + return msg + + +def _smtp_connect(smtp_server, port, sender_email, password, use_tls=True, timeout=30): + """ + 智能 SMTP 连接： + - use_tls=True: 优先尝试 SMTP_SSL，失败则回退到 SMTP + STARTTLS。 + - use_tls=False: 明文连接。 + """ + try: + if use_tls: + try: + server = smtplib.SMTP_SSL(smtp_server, port, timeout=timeout) + except Exception as e_ssl: + logging.warning(f"SMTP_SSL 连接失败，尝试 STARTTLS: {e_ssl}") + server = smtplib.SMTP(smtp_server, port, timeout=timeout) + server.starttls() + else: + server = smtplib.SMTP(smtp_server, port, timeout=timeout) + + server.login(sender_email, password) + return server + except Exception as e: + logging.error(f"SMTP 连接失败: {e}") + raise + + +def _validate_email(addr: str) -> bool: + """ + 基础 email 格式检查。 + """ + if not addr: + return False + name, email = parseaddr(addr) + if "@" not in email or email.count("@") != 1: + return False + local, domain = email.rsplit("@", 1) + if not local or not domain or "." not in domain: + return False + return True + + +# ============================================================ +# 单封邮件发送 +# ============================================================ + +def email_sender( + target_email, + sender_email, + smtp_server, + port, + password, + subject, + body, + template_path=None, + template_data=None, + use_tls=True, +): + """ + 发送单封邮件。 + """ + msg = _render_message( + target_email=target_email, + sender_email=sender_email, + subject=subject, + body=body, + template_path=template_path, + template_data=template_data, + ) + + try: + server = _smtp_connect(smtp_server, port, sender_email, password, use_tls=use_tls) + server.sendmail(sender_email, [target_email], msg.as_string()) + server.quit() + print(f"邮件已发送到 {target_email}") + except Exception as e: + logging.error(f"邮件发送失败，目标地址: {target_email}，错误信息: {e}") + + +# ============================================================ +# 批量邮件发送 +# ============================================================ + +def send_emails( + emails, + sender_email, + smtp_server, + port, + password, + subject, + body, + template_path=None, + template_data=None, + use_tls=True, +): + """ + 批量发送邮件： + - 分批（默认100封为一批，可通过 EMAIL_BATCH_SIZE 环境变量调整） + - 单封发送，防止泄露邮箱 + - SMTP 连接复用，失败隔离 + - 返回 summary + """ + batch_size = int(os.getenv("EMAIL_BATCH_SIZE", "100")) + sleep_between_batches = float(os.getenv("EMAIL_BATCH_SLEEP", "0")) + validate_strict = os.getenv("EMAIL_VALIDATE_STRICT", "1") not in ("0", "false", "False") + + # 去重 & 校验 + seen = set() + cleaned, invalid = [], [] + for addr in emails: + addr = addr.strip() + if not addr or addr in seen: + continue + seen.add(addr) + if validate_strict and not _validate_email(addr): + invalid.append(addr) + logging.warning(f"无效邮箱: {addr}") + continue + cleaned.append(addr) + + total = len(cleaned) + logging.info(f"准备发送 {total} 封邮件 (原始 {len(emails)}, 无效 {len(invalid)})") + + if total == 0: + return { + "total_requested": len(emails), + "total_valid": 0, + "sent_success": 0, + "sent_failed": 0, + "invalid": invalid, + "failed": [], + } + + # 预渲染 HTML 模板 + html_cache = None + if template_path and template_data: + env = Environment(loader=FileSystemLoader(os.path.dirname(template_path))) + template = env.get_template(os.path.basename(template_path)) + html_cache = template.render(template_data) + + def build_msg_for(to_addr): + msg = MIMEMultipart("alternative") + msg["From"] = sender_email + msg["To"] = to_addr + msg["Subject"] = subject + msg["Date"] = formatdate(localtime=True) + domain = sender_email.split("@")[-1] if "@" in sender_email else "localhost" + msg["Message-ID"] = make_msgid(domain=domain) + msg.attach(MIMEText(body or "", "plain", "utf-8")) + if html_cache: + msg.attach(MIMEText(html_cache, "html", "utf-8")) + return msg + + try: + server = _smtp_connect(smtp_server, port, sender_email, password, use_tls=use_tls) + except Exception: + return { + "total_requested": len(emails), + "total_valid": total, + "sent_success": 0, + "sent_failed": total, + "invalid": invalid, + "failed": cleaned, + } + + successes, failures = [], [] + + for i in range(0, total, batch_size): + batch = cleaned[i:i + batch_size] + logging.info(f"发送批次 {i // batch_size + 1}: {len(batch)} 封") + + for addr in batch: + msg = build_msg_for(addr) + try: + refused = server.sendmail(sender_email, [addr], msg.as_string()) + if refused: + failures.append(addr) + logging.error(f"发送被拒绝: {addr} - {refused}") + else: + successes.append(addr) + except (smtplib.SMTPServerDisconnected, smtplib.SMTPConnectError): + # 尝试重连一次 + try: + logging.warning("SMTP 连接断开，尝试重连...") + server = _smtp_connect(smtp_server, port, sender_email, password, use_tls=use_tls) + server.sendmail(sender_email, [addr], msg.as_string()) + successes.append(addr) + except Exception as e: + failures.append(addr) + logging.error(f"重连后发送失败: {addr} - {e}") + except Exception as e: + failures.append(addr) + logging.error(f"发送失败: {addr} - {e}") + + if sleep_between_batches > 0 and i + batch_size < total: + time.sleep(sleep_between_batches) + + try: + server.quit() + except Exception: + pass + + summary = { + "total_requested": len(emails), + "total_valid": total, + "sent_success": len(successes), + "sent_failed": len(failures), + "invalid": invalid, + "success": successes, + "failed": failures, + } + logging.info(f"批量发送完成: 成功 {summary['sent_success']} / {summary['total_valid']}") + return summary diff --git a/friend_circle_lite/outputs/__init__.py b/friend_circle_lite/outputs/__init__.py new file mode 100644 index 00000000000..0db718bfaa4 --- /dev/null +++ b/friend_circle_lite/outputs/__init__.py @@ -0,0 +1,3 @@ +"""Output assembly and legacy-compatible JSON helpers.""" + +from friend_circle_lite.outputs.legacy_api import * # noqa: F401,F403 diff --git a/friend_circle_lite/outputs/legacy_api.py b/friend_circle_lite/outputs/legacy_api.py new file mode 100644 index 00000000000..b9e9d669cea --- /dev/null +++ b/friend_circle_lite/outputs/legacy_api.py @@ -0,0 +1,239 @@ +"""Legacy-compatible crawl entrypoints. + +The internal implementation is now delegated to `crawler_service`, but these +functions keep the existing public API stable for `run.py` and external users. +""" + +import logging + +import requests + +from friend_circle_lite import HEADERS_JSON, timeout +from friend_circle_lite.crawler.service import ( + FriendCircleCrawlService, + limit_large_dataset as _limit_large_dataset, + sort_articles_by_time as _sort_articles_by_time, +) + +def fetch_and_process_data( + json_url: str, + specific_RSS: list = None, + count: int = 5, + cache_file: str = None, + link_check_config=None, + proxy_settings=None, +): + """Legacy wrapper around the new crawler orchestration service.""" + return FriendCircleCrawlService( + json_url=json_url, + count=count, + specific_rss=specific_RSS, + cache_file=cache_file, + link_check_config=link_check_config, + proxy_settings=proxy_settings, + ).run() + +def sort_articles_by_time(data, future_tolerance_days=2): + """Legacy wrapper around the refactored sort helper.""" + return _sort_articles_by_time(data, future_tolerance_days=future_tolerance_days) + +def marge_data_from_json_url(data, marge_json_url): + """ + 从另一个 JSON 文件中获取数据并合并到原数据中。 + + 参数： + data (dict): 包含文章信息的字典 + marge_json_url (str): 包含另一个文章信息的 JSON 文件的 URL。 + + 返回： + dict: 合并后的文章信息字典，已去重处理 + """ + try: + response = requests.get(marge_json_url, headers=HEADERS_JSON, timeout=timeout) + marge_data = response.json() + except Exception as e: + logging.error(f"无法获取链接：{marge_json_url} ，出现的问题为：{e}", exc_info=True) + return data + + if 'article_data' in marge_data: + logging.info(f"开始合并文章数据，原数据共有 {len(data['article_data'])} 篇文章，第三方数据共有 {len(marge_data['article_data'])} 篇文章") + data['article_data'].extend(marge_data['article_data']) + data['article_data'] = list({v['link']:v for v in data['article_data']}.values()) + logging.info(f"合并文章数据完成，现在共有 {len(data['article_data'])} 篇文章") + return data + + +def merge_link_data_from_json_url(link_data, merge_json_url): + """ + 从另一个 link.json 文件中获取友链可达性数据并智能合并。 + + 合并策略： + - 可达性优先级：direct > proxy > api > none + - 延迟取最优（最小值） + - 反链取并集（任一为 true 则为 true） + - 失败次数取最小值 + + 参数： + link_data (dict): 本地友链数据，包含 statistical_data 和 link_data + merge_json_url (str): 远程 link.json 的 URL + + 返回： + dict: 合并后的友链数据 + """ + try: + response = requests.get(merge_json_url, headers=HEADERS_JSON, timeout=timeout) + remote_data = response.json() + except Exception as e: + logging.warning(f"无法获取友链数据：{merge_json_url} ，跳过友链数据合并。错误：{e}") + return link_data + + if 'link_data' not in remote_data: + logging.warning(f"远程数据不包含 link_data 字段，跳过友链数据合并") + return link_data + + local_links = link_data.get('link_data', []) + remote_links = remote_data.get('link_data', []) + + logging.info(f"开始合并友链数据，本地 {len(local_links)} 条，远程 {len(remote_links)} 条") + + # 按 URL 建立索引 + link_map = {link['link']: link for link in local_links} + + for remote_link in remote_links: + url = remote_link['link'] + if url not in link_map: + # 新友链，直接添加 + link_map[url] = remote_link + else: + # 已存在，智能合并 + local_link = link_map[url] + link_map[url] = _merge_single_link(local_link, remote_link) + + merged_links = list(link_map.values()) + logging.info(f"合并友链数据完成，共有 {len(merged_links)} 条友链") + + # 重新计算统计数据 + merged_stats = _recalculate_link_statistics(merged_links) + + return { + 'statistical_data': merged_stats, + 'link_data': merged_links, + } + + +def _merge_single_link(local, remote): + """ + 合并单条友链数据，优先选择更好的检测结果。 + + 优先级： + 1. 可达性：direct > proxy > api > none + 2. 延迟：取最小值 + 3. 反链：任一为 true 则为 true + 4. 失败次数：取最小值 + """ + method_priority = {'direct': 4, 'proxy': 3, 'api': 2, 'disabled': 1, 'none': 0, '': 0} + + local_priority = method_priority.get(local.get('method', ''), 0) + remote_priority = method_priority.get(remote.get('method', ''), 0) + + # 选择优先级更高的作为基础 + if remote_priority > local_priority: + base = remote.copy() + alt = local + elif remote_priority < local_priority: + base = local.copy() + alt = remote + else: + # 优先级相同，选择延迟更低的 + local_latency = local.get('latency', 999) + remote_latency = remote.get('latency', 999) + if remote_latency >= 0 and (local_latency < 0 or remote_latency < local_latency): + base = remote.copy() + alt = local + else: + base = local.copy() + alt = remote + + # 反链取并集 + local_backlink = local.get('has_backlink') + remote_backlink = remote.get('has_backlink') + if local_backlink is True or remote_backlink is True: + base['has_backlink'] = True + elif local_backlink is False and remote_backlink is False: + base['has_backlink'] = False + # 否则保持 base 的值 + + # 失败次数取最小值 + local_fail = local.get('fail_count', 0) + remote_fail = remote.get('fail_count', 0) + base['fail_count'] = min(local_fail, remote_fail) + + # 检测时间取最新 + local_checked = local.get('checked_at', '') + remote_checked = remote.get('checked_at', '') + if remote_checked > local_checked: + base['checked_at'] = remote_checked + + return base + + +def _recalculate_link_statistics(links): + """重新计算合并后的友链统计数据。""" + reachable = [link for link in links if link.get('reachable')] + crawl_allowed = [link for link in links if link.get('crawlable')] + api_only = [link for link in links if link.get('method') == 'api'] + has_backlink = [link for link in links if link.get('has_backlink') is True] + checked_times = [link.get('checked_at', '') for link in links if link.get('checked_at')] + + return { + 'link_total_num': len(links), + 'link_reachable_num': len(reachable), + 'link_unreachable_num': len(links) - len(reachable), + 'crawl_allowed_num': len(crawl_allowed), + 'api_only_num': len(api_only), + 'has_author_link_num': len(has_backlink), + 'link_last_checked_time': max(checked_times) if checked_times else '', + } + + +def marge_errors_from_json_url(errors, marge_json_url): + """ + 从另一个网络 JSON 文件中获取错误信息并遍历，删除在errors中， + 不存在于marge_errors中的友链信息。 + + 参数： + errors (list): 包含错误信息的列表 + marge_json_url (str): 包含另一个错误信息的 JSON 文件的 URL。 + + 返回： + list: 合并后的错误信息列表 + """ + try: + response = requests.get(marge_json_url, timeout=10) # 设置请求超时时间 + marge_errors = response.json() + except Exception as e: + logging.error(f"无法获取链接：{marge_json_url} ，出现的问题为：{e}", exc_info=True) + return errors + + # 提取 marge_errors 中的 URL + marge_urls = {item[1] for item in marge_errors} + + # 使用过滤器保留 errors 中在 marge_errors 中出现的 URL + filtered_errors = [error for error in errors if error[1] in marge_urls] + + logging.info(f"合并错误信息完成，合并后共有 {len(filtered_errors)} 位朋友") + return filtered_errors + +def deal_with_large_data(result, future_tolerance_days=2): + """Legacy wrapper around the refactored dataset trimming helper.""" + return _limit_large_dataset(result, future_tolerance_days=future_tolerance_days) + + +def merge_data_from_json_url(data, merge_json_url): + """Correctly spelled wrapper for the legacy article merge helper.""" + return marge_data_from_json_url(data, merge_json_url) + + +def merge_errors_from_json_url(errors, merge_json_url): + """Correctly spelled wrapper for the legacy error merge helper.""" + return marge_errors_from_json_url(errors, merge_json_url) diff --git a/friend_circle_lite/single_friend.py b/friend_circle_lite/single_friend.py index 6e240e18e0d..677fc18b65a 100644 --- a/friend_circle_lite/single_friend.py +++ b/friend_circle_lite/single_friend.py @@ -1,141 +1,6 @@ -"""Legacy-compatible single website helpers. +"""Backward-compatible single-site crawl helpers. -The project now uses dedicated domain models and services, but these helpers are -kept as a compatibility layer because existing entrypoints still import them. +New code should import from `friend_circle_lite.crawler.single_site_legacy`. """ -from __future__ import annotations - -import logging - -import requests - -from friend_circle_lite.feed_service import FeedDiscoveryService, FeedParserService, LatestArticleTracker -from friend_circle_lite.models import CacheUpdate, Website - -def check_feed(blog_url, session): - """Return the discovered feed type and URL in the historical tuple format.""" - endpoint = FeedDiscoveryService(session).discover(blog_url) - if endpoint is None: - return ["none", blog_url] - return [endpoint.feed_type, endpoint.url] - -def parse_feed(url, session, count=5, blog_url=''): - """Parse a feed and return the historical dictionary structure.""" - articles = FeedParserService(session).parse(url, count=count, blog_url=blog_url) - return { - 'website_name': '', - 'author': articles[0].author if articles else '', - 'link': '', - 'articles': [article.to_tracking_dict() for article in articles], - } - -def process_friend(friend, session: requests.Session, count: int, specific_and_cache=None): - """Crawl one friend entry and return the historical result shape.""" - if specific_and_cache is None: - specific_and_cache = [] - - try: - website = Website.from_friend_item(friend) - except Exception: - logging.error(f"friend 数据格式不正确: {friend!r}") - return { - 'name': None, - 'status': 'error', - 'articles': [], - 'feed_url': None, - 'feed_type': 'none', - 'cache_update': CacheUpdate(action='none', name=None, url=None, reason='bad_friend_data').to_dict(), - 'source_used': 'none', - } - - rss_lookup = {entry['name']: entry for entry in specific_and_cache if entry.get('name') and entry.get('url')} - entry = rss_lookup.get(website.name) - - endpoint = None - cache_update = CacheUpdate(action='none', name=website.name) - if entry: - source = entry.get('source', 'unknown') - endpoint = {'feed_type': 'specific', 'url': entry['url'], 'source': source} - if source == 'manual': - logging.info(f"'{website.name}' 使用预设 RSS 源：{entry['url']}") - elif source == 'cache': - logging.info(f"'{website.name}' 使用缓存 RSS 源：{entry['url']}") - else: - logging.info(f"'{website.name}' 使用 RSS 源：{entry['url']} (来源: {source})") - else: - feed_type, feed_url = check_feed(website.url, session) - if feed_type != 'none' and feed_url: - endpoint = {'feed_type': feed_type, 'url': feed_url, 'source': 'auto'} - cache_update = CacheUpdate(action='set', name=website.name, url=feed_url, reason='auto_discovered') - logging.info(f"'{website.name}' 自动探测到 RSS：{feed_url}") - - articles = [] - parse_error = endpoint is not None - if endpoint: - parsed = FeedParserService(session).parse(endpoint['url'], count=count, blog_url=website.url) - articles = [ - { - 'title': article.title, - 'created': article.published, - 'link': article.link, - 'author': website.name, - 'avatar': website.avatar, - } - for article in parsed - ] - parse_error = not articles - - if parse_error and endpoint and endpoint['source'] in ('cache', 'unknown'): - logging.warning(f"'{website.name}' 缓存的 RSS 源无效，尝试重新探测...") - new_feed_type, new_feed_url = check_feed(website.url, session) - if new_feed_type != 'none' and new_feed_url: - reparsed = FeedParserService(session).parse(new_feed_url, count=count, blog_url=website.url) - articles = [ - { - 'title': article.title, - 'created': article.published, - 'link': article.link, - 'author': website.name, - 'avatar': website.avatar, - } - for article in reparsed - ] - if articles: - endpoint = {'feed_type': new_feed_type, 'url': new_feed_url, 'source': 'auto'} - cache_update = CacheUpdate(action='set', name=website.name, url=new_feed_url, reason='repair_cache') - logging.info(f"'{website.name}' 重新探测成功，更新缓存：{new_feed_url}") - else: - endpoint = None - cache_update = CacheUpdate(action='delete', name=website.name, url=None, reason='remove_invalid') - logging.warning(f"'{website.name}' 重新探测失败，删除无效缓存") - else: - endpoint = None - cache_update = CacheUpdate(action='delete', name=website.name, url=None, reason='remove_invalid') - logging.warning(f"'{website.name}' 未找到有效 RSS，删除无效缓存") - - return { - 'name': website.name, - 'status': 'active' if articles else 'error', - 'articles': articles, - 'feed_url': endpoint['url'] if endpoint else None, - 'feed_type': endpoint['feed_type'] if endpoint else 'none', - 'cache_update': cache_update.to_dict(), - 'source_used': endpoint['source'] if endpoint else 'none', - } - -def get_latest_articles_from_link(url, count=5, last_articles_path="./temp/newest_posts.json"): - """Return newly published articles relative to the last local snapshot.""" - session = requests.Session() - feed_type, feed_url = check_feed(url, session) - if feed_type == 'none': - logging.error(f"无法获取 {url} 的文章数据") - return None - - latest_articles = FeedParserService(session).parse(feed_url, count=count, blog_url=url) - updated_articles = LatestArticleTracker(last_articles_path).diff_and_persist(latest_articles) - logging.info( - f"从 {url} 获取到 {len(latest_articles)} 篇文章，其中 {0 if updated_articles is None else len(updated_articles)} 篇为新文章" - ) - return updated_articles - +from friend_circle_lite.crawler.single_site_legacy import * # noqa: F401,F403 diff --git a/friend_circle_lite/storage/__init__.py b/friend_circle_lite/storage/__init__.py new file mode 100644 index 00000000000..1dccef8b143 --- /dev/null +++ b/friend_circle_lite/storage/__init__.py @@ -0,0 +1,3 @@ +"""Persistent stores for feed cache, article tracking, and link checks.""" + +from friend_circle_lite.storage.sqlite_store import ArticleTrackingStore, FeedCacheStore, LinkCheckStore diff --git a/friend_circle_lite/storage/sqlite_store.py b/friend_circle_lite/storage/sqlite_store.py new file mode 100644 index 00000000000..960b89eb8f9 --- /dev/null +++ b/friend_circle_lite/storage/sqlite_store.py @@ -0,0 +1,486 @@ +"""Persistent RSS cache and article tracking storage. + +SQLite is used for both feed cache and article tracking because it is more robust +than hand-edited text formats for internal state: + +- schema is explicit and stable; +- writes are transactional; +- corruption risk from accidental manual edits is lower; +- Python ships with `sqlite3`, so no extra dependency is required. + +For smooth upgrades, this store can also migrate legacy cache data from the old +JSON cache file and the intermediate YAML cache file if they exist. +""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +from datetime import datetime +from pathlib import Path + +import yaml + +from friend_circle_lite.domain.models import Article, CacheRecord, LinkCheckRecord, LinkMethodStatus + + +class FeedCacheStore: + """Persist and load discovered RSS endpoints using SQLite.""" + + def __init__(self, cache_path: str | Path | None): + self.cache_path = Path(cache_path) if cache_path else None + + def load_records(self) -> list[CacheRecord]: + """Load cache records from SQLite, migrating legacy formats if needed.""" + if not self.cache_path: + return [] + + if self.cache_path.exists(): + return self._load_from_sqlite() + + migrated_records = self._load_legacy_records() + if migrated_records: + if self.save_records(migrated_records): + logging.info(f"已从旧格式迁移 {len(migrated_records)} 条 RSS 缓存到 SQLite") + return migrated_records + + logging.info(f"RSS 缓存文件不存在，将在首次抓取后自动创建") + return [] + + def save_records(self, records: list[CacheRecord]) -> bool: + """Persist cache records to the SQLite database.""" + if not self.cache_path: + return True + + try: + self.cache_path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(self.cache_path) as connection: + self._ensure_schema(connection) + connection.execute("DELETE FROM feed_cache") + connection.executemany( + "INSERT INTO feed_cache(name, url, source) VALUES (?, ?, ?)", + [(record.name, record.url, record.source) for record in sorted(records, key=lambda item: item.name)], + ) + connection.commit() + logging.info(f"RSS 缓存已保存（{len(records)} 条）") + return True + except Exception as exc: + logging.error(f"保存 RSS 缓存失败: {exc}") + return False + + def _load_from_sqlite(self) -> list[CacheRecord]: + """Load records from the current SQLite cache file.""" + try: + with sqlite3.connect(self.cache_path) as connection: + self._ensure_schema(connection) + rows = connection.execute( + "SELECT name, url, source FROM feed_cache ORDER BY name" + ).fetchall() + except Exception as exc: + logging.warning(f"读取 RSS 缓存失败: {exc}") + return [] + + return [ + CacheRecord(name=name, url=url, source=source or "cache") + for name, url, source in rows + if name and url + ] + + @staticmethod + def _ensure_schema(connection: sqlite3.Connection) -> None: + """Create the cache table when it does not exist yet.""" + connection.execute( + """ + CREATE TABLE IF NOT EXISTS feed_cache ( + name TEXT PRIMARY KEY, + url TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'cache' + ) + """ + ) + + def _load_legacy_records(self) -> list[CacheRecord]: + """Read old cache formats for seamless upgrades.""" + json_records = self._load_legacy_json_cache() + if json_records: + return json_records + + yaml_records = self._load_legacy_yaml_cache() + if yaml_records: + return yaml_records + + return [] + + def _load_legacy_json_cache(self) -> list[CacheRecord]: + """Read the previous JSON cache file format.""" + if not self.cache_path: + return [] + + legacy_path = self.cache_path.with_name("cache.json") + if not legacy_path.exists(): + return [] + + try: + with open(legacy_path, "r", encoding="utf-8") as file: + payload = json.load(file) + except Exception as exc: + logging.warning(f"读取旧 JSON 缓存失败: {exc}") + return [] + + if not isinstance(payload, list): + return [] + + return self._normalize_legacy_items(payload) + + def _load_legacy_yaml_cache(self) -> list[CacheRecord]: + """Read the temporary YAML cache format used during refactoring.""" + if not self.cache_path: + return [] + + legacy_path = self.cache_path.with_name("feed_cache.yaml") + if not legacy_path.exists(): + return [] + + try: + with open(legacy_path, "r", encoding="utf-8") as file: + payload = yaml.safe_load(file) or {} + except Exception as exc: + logging.warning(f"读取旧 YAML 缓存失败: {exc}") + return [] + + items = payload.get("feeds", []) if isinstance(payload, dict) else [] + return self._normalize_legacy_items(items) + + @staticmethod + def _normalize_legacy_items(items: list[object]) -> list[CacheRecord]: + """Normalize legacy cache items into typed cache records.""" + records: list[CacheRecord] = [] + for item in items: + if not isinstance(item, dict): + continue + name = str(item.get("name", "")).strip() + url = str(item.get("url", "")).strip() + source = str(item.get("source", "cache")).strip() or "cache" + if name and url: + records.append(CacheRecord(name=name, url=url, source=source)) + return records + + +class ArticleTrackingStore: + """Persist and load article tracking data using SQLite.""" + + def __init__(self, storage_path: str | Path | None, max_tracked_articles: int = 10): + self.storage_path = Path(storage_path) if storage_path else None + self.max_tracked_articles = max_tracked_articles + + def load_articles(self) -> list[Article]: + """Load tracked articles from SQLite, migrating from legacy JSON if needed.""" + if not self.storage_path: + return [] + + if self.storage_path.exists(): + return self._load_from_sqlite() + + # Try to migrate from legacy JSON format + migrated_articles = self._load_legacy_json() + if migrated_articles: + if self.save_articles(migrated_articles): + logging.info(f"已从旧 JSON 格式迁移 {len(migrated_articles)} 篇文章记录到 SQLite") + return migrated_articles + + logging.info(f"文章追踪数据不存在，这是首次运行") + return [] + + def save_articles(self, articles: list[Article]) -> bool: + """Persist articles to SQLite, keeping only the most recent max_tracked_articles.""" + if not self.storage_path: + return True + + try: + # Sort by date and keep only the most recent articles + valid_articles = [article for article in articles if article.published] + valid_articles.sort( + key=lambda item: datetime.strptime(item.published, "%Y-%m-%d %H:%M"), + reverse=True + ) + articles_to_save = valid_articles[:self.max_tracked_articles] + + self.storage_path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(self.storage_path) as connection: + self._ensure_schema(connection) + connection.execute("DELETE FROM article_tracking") + connection.executemany( + """INSERT INTO article_tracking(title, author, link, published, summary, content) + VALUES (?, ?, ?, ?, ?, ?)""", + [ + ( + article.title, + article.author, + article.link, + article.published, + article.summary, + article.content, + ) + for article in articles_to_save + ], + ) + connection.commit() + return True + except Exception as exc: + logging.error(f"保存文章追踪数据失败: {exc}") + return False + + def _load_from_sqlite(self) -> list[Article]: + """Load articles from the SQLite database.""" + try: + with sqlite3.connect(self.storage_path) as connection: + self._ensure_schema(connection) + rows = connection.execute( + """SELECT title, author, link, published, summary, content + FROM article_tracking + ORDER BY published DESC""" + ).fetchall() + except Exception as exc: + logging.warning(f"读取文章追踪数据失败: {exc}") + return [] + + return [ + Article( + title=title or "", + author=author or "", + link=link or "", + published=published or "", + summary=summary or "", + content=content or "", + ) + for title, author, link, published, summary, content in rows + ] + + @staticmethod + def _ensure_schema(connection: sqlite3.Connection) -> None: + """Create the article tracking table when it does not exist yet.""" + connection.execute( + """ + CREATE TABLE IF NOT EXISTS article_tracking ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + author TEXT NOT NULL, + link TEXT NOT NULL, + published TEXT NOT NULL, + summary TEXT, + content TEXT + ) + """ + ) + + def _load_legacy_json(self) -> list[Article]: + """Read the old JSON format for seamless upgrades.""" + if not self.storage_path: + return [] + + legacy_path = self.storage_path.with_name("newest_posts.json") + if not legacy_path.exists(): + return [] + + try: + with open(legacy_path, "r", encoding="utf-8") as file: + payload = json.load(file) + except Exception as exc: + logging.warning(f"读取旧 JSON 文章追踪文件失败: {exc}") + return [] + + articles_data = payload.get("articles", []) if isinstance(payload, dict) else [] + articles: list[Article] = [] + for item in articles_data: + if not isinstance(item, dict): + continue + articles.append( + Article( + title=item.get("title", ""), + author=item.get("author", ""), + link=item.get("link", ""), + published=item.get("published", ""), + summary=item.get("summary", ""), + content=item.get("content", ""), + ) + ) + return articles + + +class LinkCheckStore: + """Persist friend link reachability checks using SQLite.""" + + def __init__(self, cache_path: str | Path | None): + self.cache_path = Path(cache_path) if cache_path else None + + def load_records(self, urls: list[str] | None = None) -> dict[str, LinkCheckRecord]: + if not self.cache_path or not self.cache_path.exists(): + return {} + + try: + with sqlite3.connect(self.cache_path) as connection: + self._ensure_schema(connection) + rows = connection.execute( + """ + SELECT url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, + best_method, best_latency, fail_count, backlink_checked, has_author_link, + rss_crawl_reason, direct_success, direct_status_code, direct_latency, + proxy_success, proxy_status_code, proxy_latency, api_success, + api_status_code, api_latency + FROM link_check_state + """ + ).fetchall() + except Exception as exc: + logging.warning(f"读取友链检测缓存失败: {exc}") + return {} + + allowed_urls = set(urls or []) + records: dict[str, LinkCheckRecord] = {} + for row in rows: + ( + url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, + best_method, best_latency, fail_count, backlink_checked, has_author_link, + rss_crawl_reason, direct_success, direct_status_code, direct_latency, + proxy_success, proxy_status_code, proxy_latency, api_success, + api_status_code, api_latency, + ) = row + if allowed_urls and url not in allowed_urls: + continue + records[url] = LinkCheckRecord( + name=name or "", + url=url or "", + avatar=avatar or "", + linkpage=linkpage or "", + checked_at=checked_at or "", + reachable=bool(reachable), + crawl_allowed=bool(crawl_allowed), + best_method=best_method or "none", + best_latency=best_latency if best_latency is not None else -1, + fail_count=fail_count or 0, + backlink_checked=bool(backlink_checked), + has_author_link=bool(has_author_link), + rss_crawl_reason=rss_crawl_reason or "blocked_unreachable", + direct=LinkMethodStatus(bool(direct_success), direct_status_code, direct_latency if direct_latency is not None else -1), + proxy=LinkMethodStatus(bool(proxy_success), proxy_status_code, proxy_latency if proxy_latency is not None else -1), + api=LinkMethodStatus(bool(api_success), api_status_code, api_latency if api_latency is not None else -1), + ) + return records + + def save_records(self, records: list[LinkCheckRecord]) -> bool: + if not self.cache_path: + return True + + try: + self.cache_path.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(self.cache_path) as connection: + self._ensure_schema(connection) + connection.executemany( + """ + INSERT INTO link_check_state( + url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, + best_method, best_latency, fail_count, backlink_checked, has_author_link, + rss_crawl_reason, direct_success, direct_status_code, direct_latency, + proxy_success, proxy_status_code, proxy_latency, api_success, + api_status_code, api_latency + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(url) DO UPDATE SET + name = excluded.name, + avatar = excluded.avatar, + linkpage = excluded.linkpage, + checked_at = excluded.checked_at, + reachable = excluded.reachable, + crawl_allowed = excluded.crawl_allowed, + best_method = excluded.best_method, + best_latency = excluded.best_latency, + fail_count = excluded.fail_count, + backlink_checked = excluded.backlink_checked, + has_author_link = excluded.has_author_link, + rss_crawl_reason = excluded.rss_crawl_reason, + direct_success = excluded.direct_success, + direct_status_code = excluded.direct_status_code, + direct_latency = excluded.direct_latency, + proxy_success = excluded.proxy_success, + proxy_status_code = excluded.proxy_status_code, + proxy_latency = excluded.proxy_latency, + api_success = excluded.api_success, + api_status_code = excluded.api_status_code, + api_latency = excluded.api_latency + """, + [self._record_to_row(record) for record in records], + ) + connection.commit() + logging.info(f"友链检测缓存已保存（{len(records)} 条）") + return True + except Exception as exc: + logging.error(f"保存友链检测缓存失败: {exc}") + return False + + @staticmethod + def is_fresh(record: LinkCheckRecord, max_age_hours: int) -> bool: + if not record.checked_at: + return False + try: + checked_at = datetime.strptime(record.checked_at, "%Y-%m-%d %H:%M:%S") + except ValueError: + return False + age_seconds = (datetime.now() - checked_at).total_seconds() + return age_seconds < max_age_hours * 3600 + + @staticmethod + def _record_to_row(record: LinkCheckRecord) -> tuple: + return ( + record.url, + record.name, + record.avatar, + record.linkpage, + record.checked_at, + int(record.reachable), + int(record.crawl_allowed), + record.best_method, + record.best_latency, + record.fail_count, + int(record.backlink_checked), + int(record.has_author_link), + record.rss_crawl_reason, + int(record.direct.success), + record.direct.status_code, + record.direct.latency, + int(record.proxy.success), + record.proxy.status_code, + record.proxy.latency, + int(record.api.success), + record.api.status_code, + record.api.latency, + ) + + @staticmethod + def _ensure_schema(connection: sqlite3.Connection) -> None: + connection.execute( + """ + CREATE TABLE IF NOT EXISTS link_check_state ( + url TEXT PRIMARY KEY, + name TEXT NOT NULL, + avatar TEXT DEFAULT '', + linkpage TEXT DEFAULT '', + checked_at TEXT NOT NULL, + reachable INTEGER NOT NULL DEFAULT 0, + crawl_allowed INTEGER NOT NULL DEFAULT 0, + best_method TEXT NOT NULL DEFAULT 'none', + best_latency REAL DEFAULT -1, + fail_count INTEGER NOT NULL DEFAULT 0, + backlink_checked INTEGER NOT NULL DEFAULT 0, + has_author_link INTEGER NOT NULL DEFAULT 0, + rss_crawl_reason TEXT NOT NULL DEFAULT '', + direct_success INTEGER NOT NULL DEFAULT 0, + direct_status_code INTEGER, + direct_latency REAL DEFAULT -1, + proxy_success INTEGER NOT NULL DEFAULT 0, + proxy_status_code INTEGER, + proxy_latency REAL DEFAULT -1, + api_success INTEGER NOT NULL DEFAULT 0, + api_status_code INTEGER, + api_latency REAL DEFAULT -1 + ) + """ + ) diff --git a/friend_circle_lite/utils/cache.py b/friend_circle_lite/utils/cache.py index f5803e639f7..6647090081a 100644 --- a/friend_circle_lite/utils/cache.py +++ b/friend_circle_lite/utils/cache.py @@ -4,8 +4,8 @@ These wrappers keep the old function names available for any external callers. """ -from friend_circle_lite.cache_store import FeedCacheStore -from friend_circle_lite.models import CacheRecord +from friend_circle_lite.domain.models import CacheRecord +from friend_circle_lite.storage.sqlite_store import FeedCacheStore def load_cache(cache_file: str): diff --git a/friend_circle_lite/utils/config.py b/friend_circle_lite/utils/config.py index 37976d65e14..18a9274be7d 100644 --- a/friend_circle_lite/utils/config.py +++ b/friend_circle_lite/utils/config.py @@ -6,7 +6,7 @@ import yaml -from friend_circle_lite.app_config import ApplicationConfig +from friend_circle_lite.config.models import ApplicationConfig def load_raw_config(config_file: str) -> dict: """Load the raw YAML config dictionary from disk.""" diff --git a/friend_circle_lite/utils/github.py b/friend_circle_lite/utils/github.py index 2cbdc50a196..fce69727ace 100644 --- a/friend_circle_lite/utils/github.py +++ b/friend_circle_lite/utils/github.py @@ -1,39 +1,6 @@ -import logging -import requests -import re -from friend_circle_lite import HEADERS_JSON +"""Backward-compatible GitHub notification helpers. -def extract_emails_from_issues(api_url): - """ - 从GitHub issues API中提取以[e-mail]开头的title中的邮箱地址。 +New code should import from `friend_circle_lite.notifications.github`. +""" - 参数： - api_url (str): GitHub issues API的URL。 - - 返回： - dict: 包含所有提取的邮箱地址的字典。 - { - "emails": [ - "3162475700@qq.com" - ] - } - """ - try: - response = requests.get(api_url, headers=HEADERS_JSON, timeout=10) - response.raise_for_status() - issues = response.json() - except Exception as e: - logging.error(f"无法获取 GitHub issues 数据，错误信息: {e}") - return None - - email_pattern = re.compile(r'^\[邮箱订阅\](.+)$') - emails = [] - - for issue in issues: - title = issue.get("title", "") - match = email_pattern.match(title) - if match: - email = match.group(1).strip() - emails.append(email) - - return {"emails": emails} \ No newline at end of file +from friend_circle_lite.notifications.github import * # noqa: F401,F403 diff --git a/friend_circle_lite/utils/mail.py b/friend_circle_lite/utils/mail.py index bb7df1e8b92..ea8071b31f1 100644 --- a/friend_circle_lite/utils/mail.py +++ b/friend_circle_lite/utils/mail.py @@ -1,255 +1,6 @@ -import logging -import smtplib -import time -import os -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText -from email.utils import formatdate, make_msgid, parseaddr -from jinja2 import Environment, FileSystemLoader +"""Backward-compatible mail helpers. -# ============================================================ -# 内部工具 -# ============================================================ +New code should import from `friend_circle_lite.notifications.mail`. +""" -def _render_message( - target_email, - sender_email, - subject, - body, - template_path=None, - template_data=None, -): - """ - 构建 MIME 邮件对象，支持纯文本 + 可选 HTML。 - """ - msg = MIMEMultipart("alternative") - msg["From"] = sender_email - msg["To"] = target_email - msg["Subject"] = subject - msg["Date"] = formatdate(localtime=True) - domain = sender_email.split("@")[-1] if "@" in sender_email else "localhost" - msg["Message-ID"] = make_msgid(domain=domain) - - # 纯文本内容 - msg.attach(MIMEText(body or "", "plain", "utf-8")) - - # HTML 模板内容 - if template_path and template_data: - env = Environment(loader=FileSystemLoader(os.path.dirname(template_path))) - template = env.get_template(os.path.basename(template_path)) - html_content = template.render(template_data) - msg.attach(MIMEText(html_content, "html", "utf-8")) - - return msg - - -def _smtp_connect(smtp_server, port, sender_email, password, use_tls=True, timeout=30): - """ - 智能 SMTP 连接： - - use_tls=True: 优先尝试 SMTP_SSL，失败则回退到 SMTP + STARTTLS。 - - use_tls=False: 明文连接。 - """ - try: - if use_tls: - try: - server = smtplib.SMTP_SSL(smtp_server, port, timeout=timeout) - except Exception as e_ssl: - logging.warning(f"SMTP_SSL 连接失败，尝试 STARTTLS: {e_ssl}") - server = smtplib.SMTP(smtp_server, port, timeout=timeout) - server.starttls() - else: - server = smtplib.SMTP(smtp_server, port, timeout=timeout) - - server.login(sender_email, password) - return server - except Exception as e: - logging.error(f"SMTP 连接失败: {e}") - raise - - -def _validate_email(addr: str) -> bool: - """ - 基础 email 格式检查。 - """ - if not addr: - return False - name, email = parseaddr(addr) - if "@" not in email or email.count("@") != 1: - return False - local, domain = email.rsplit("@", 1) - if not local or not domain or "." not in domain: - return False - return True - - -# ============================================================ -# 单封邮件发送 -# ============================================================ - -def email_sender( - target_email, - sender_email, - smtp_server, - port, - password, - subject, - body, - template_path=None, - template_data=None, - use_tls=True, -): - """ - 发送单封邮件。 - """ - msg = _render_message( - target_email=target_email, - sender_email=sender_email, - subject=subject, - body=body, - template_path=template_path, - template_data=template_data, - ) - - try: - server = _smtp_connect(smtp_server, port, sender_email, password, use_tls=use_tls) - server.sendmail(sender_email, [target_email], msg.as_string()) - server.quit() - print(f"邮件已发送到 {target_email}") - except Exception as e: - logging.error(f"邮件发送失败，目标地址: {target_email}，错误信息: {e}") - - -# ============================================================ -# 批量邮件发送 -# ============================================================ - -def send_emails( - emails, - sender_email, - smtp_server, - port, - password, - subject, - body, - template_path=None, - template_data=None, - use_tls=True, -): - """ - 批量发送邮件： - - 分批（默认100封为一批，可通过 EMAIL_BATCH_SIZE 环境变量调整） - - 单封发送，防止泄露邮箱 - - SMTP 连接复用，失败隔离 - - 返回 summary - """ - batch_size = int(os.getenv("EMAIL_BATCH_SIZE", "100")) - sleep_between_batches = float(os.getenv("EMAIL_BATCH_SLEEP", "0")) - validate_strict = os.getenv("EMAIL_VALIDATE_STRICT", "1") not in ("0", "false", "False") - - # 去重 & 校验 - seen = set() - cleaned, invalid = [], [] - for addr in emails: - addr = addr.strip() - if not addr or addr in seen: - continue - seen.add(addr) - if validate_strict and not _validate_email(addr): - invalid.append(addr) - logging.warning(f"无效邮箱: {addr}") - continue - cleaned.append(addr) - - total = len(cleaned) - logging.info(f"准备发送 {total} 封邮件 (原始 {len(emails)}, 无效 {len(invalid)})") - - if total == 0: - return { - "total_requested": len(emails), - "total_valid": 0, - "sent_success": 0, - "sent_failed": 0, - "invalid": invalid, - "failed": [], - } - - # 预渲染 HTML 模板 - html_cache = None - if template_path and template_data: - env = Environment(loader=FileSystemLoader(os.path.dirname(template_path))) - template = env.get_template(os.path.basename(template_path)) - html_cache = template.render(template_data) - - def build_msg_for(to_addr): - msg = MIMEMultipart("alternative") - msg["From"] = sender_email - msg["To"] = to_addr - msg["Subject"] = subject - msg["Date"] = formatdate(localtime=True) - domain = sender_email.split("@")[-1] if "@" in sender_email else "localhost" - msg["Message-ID"] = make_msgid(domain=domain) - msg.attach(MIMEText(body or "", "plain", "utf-8")) - if html_cache: - msg.attach(MIMEText(html_cache, "html", "utf-8")) - return msg - - try: - server = _smtp_connect(smtp_server, port, sender_email, password, use_tls=use_tls) - except Exception: - return { - "total_requested": len(emails), - "total_valid": total, - "sent_success": 0, - "sent_failed": total, - "invalid": invalid, - "failed": cleaned, - } - - successes, failures = [], [] - - for i in range(0, total, batch_size): - batch = cleaned[i:i + batch_size] - logging.info(f"发送批次 {i // batch_size + 1}: {len(batch)} 封") - - for addr in batch: - msg = build_msg_for(addr) - try: - refused = server.sendmail(sender_email, [addr], msg.as_string()) - if refused: - failures.append(addr) - logging.error(f"发送被拒绝: {addr} - {refused}") - else: - successes.append(addr) - except (smtplib.SMTPServerDisconnected, smtplib.SMTPConnectError): - # 尝试重连一次 - try: - logging.warning("SMTP 连接断开，尝试重连...") - server = _smtp_connect(smtp_server, port, sender_email, password, use_tls=use_tls) - server.sendmail(sender_email, [addr], msg.as_string()) - successes.append(addr) - except Exception as e: - failures.append(addr) - logging.error(f"重连后发送失败: {addr} - {e}") - except Exception as e: - failures.append(addr) - logging.error(f"发送失败: {addr} - {e}") - - if sleep_between_batches > 0 and i + batch_size < total: - time.sleep(sleep_between_batches) - - try: - server.quit() - except Exception: - pass - - summary = { - "total_requested": len(emails), - "total_valid": total, - "sent_success": len(successes), - "sent_failed": len(failures), - "invalid": invalid, - "success": successes, - "failed": failures, - } - logging.info(f"批量发送完成: 成功 {summary['sent_success']} / {summary['total_valid']}") - return summary +from friend_circle_lite.notifications.mail import * # noqa: F401,F403 diff --git a/run.py b/run.py index 5447522ef53..de7ef9857f4 100644 --- a/run.py +++ b/run.py @@ -4,7 +4,7 @@ import logging -from friend_circle_lite.application import FriendCircleLiteApplication +from friend_circle_lite.cli import FriendCircleLiteApplication from friend_circle_lite.utils.config import load_config diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000000..011a32e9287 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test package for Friend-Circle-Lite.""" diff --git a/tests/test_refactor_contracts.py b/tests/test_refactor_contracts.py new file mode 100644 index 00000000000..500a7e6759f --- /dev/null +++ b/tests/test_refactor_contracts.py @@ -0,0 +1,155 @@ +import unittest +from unittest.mock import patch + +from friend_circle_lite.all_friends import deal_with_large_data, merge_link_data_from_json_url +from friend_circle_lite.app_config import ApplicationConfig +from friend_circle_lite.models import LinkCheckRecord, LinkMethodStatus, Website + + +class RefactorContractsTest(unittest.TestCase): + def test_config_keeps_existing_yaml_keys(self): + config = ApplicationConfig.from_dict({ + "spider_settings": { + "enable": True, + "json_url": "https://example.com/friends.json", + "article_count": 3, + }, + "proxy_settings": {"proxy_url": "https://proxy.example/"}, + "merge_settings": { + "enable": True, + "remote_base_url": "https://remote.example", + "merge_article_data": False, + "merge_link_check_data": True, + }, + "link_check": { + "enable": False, + "max_age_hours": 6, + "timeout": 9, + "max_workers": 2, + "status_api_url": "https://status.example?url={url}", + "enable_backlink_check": True, + "author_url": "example.com", + }, + "runtime_paths": { + "cache_file": "./tmp/state.sqlite3", + "all_json_file": "./public/all.json", + "errors_json_file": "./public/errors.json", + "link_json_file": "./public/link.json", + }, + "specific_RSS": [{"name": "Manual", "url": "https://example.com/feed.xml"}], + }) + + self.assertEqual(config.spider_settings.json_url, "https://example.com/friends.json") + self.assertEqual(config.spider_settings.article_count, 3) + self.assertEqual(config.proxy_settings.proxy_url, "https://proxy.example/") + self.assertTrue(config.merge_settings.enable) + self.assertFalse(config.merge_settings.merge_article_data) + self.assertFalse(config.link_check.enable) + self.assertEqual(config.link_check.author_url, "example.com") + self.assertEqual(config.runtime_paths.cache_file, "./tmp/state.sqlite3") + self.assertEqual(config.specific_rss[0]["name"], "Manual") + + def test_website_and_link_record_public_shapes_are_stable(self): + website = Website.from_friend_item(["Alice", "https://alice.example", "https://alice.example/links", "avatar.png"]) + self.assertEqual(website.to_error_payload(), ["Alice", "https://alice.example", "avatar.png"]) + + record = LinkCheckRecord( + name=website.name, + url=website.url, + avatar=website.avatar, + linkpage=website.linkpage, + checked_at="2026-06-06 12:00:00", + reachable=True, + crawl_allowed=True, + best_method="proxy", + best_latency=1.2, + fail_count=0, + backlink_checked=True, + has_author_link=True, + rss_crawl_reason="allowed_by_proxy", + direct=LinkMethodStatus(False, 403, 2.0), + proxy=LinkMethodStatus(True, 200, 1.2), + ) + + self.assertEqual(record.to_link_dict(), { + "name": "Alice", + "link": "https://alice.example", + "link_page": "https://alice.example/links", + "avatar": "avatar.png", + "reachable": True, + "crawlable": True, + "method": "proxy", + "latency": 1.2, + "fail_count": 0, + "checked_at": "2026-06-06 12:00:00", + "has_backlink": True, + "reason": "allowed_by_proxy", + }) + + def test_large_data_sorting_keeps_public_article_schema(self): + payload = { + "statistical_data": {"article_num": 0}, + "article_data": [ + {"title": "Old", "created": "2024-01-01 00:00", "link": "https://old", "author": "A", "avatar": "a.png"}, + {"title": "New", "created": "2024-01-02 00:00", "link": "https://new", "author": "B", "avatar": "b.png"}, + ], + } + + result = deal_with_large_data(payload) + + self.assertEqual([article["title"] for article in result["article_data"]], ["New", "Old"]) + self.assertEqual(result["statistical_data"]["article_num"], 2) + self.assertEqual(set(result["article_data"][0].keys()), {"title", "created", "link", "author", "avatar"}) + + def test_link_merge_keeps_best_reachability_shape(self): + local = { + "statistical_data": {}, + "link_data": [{ + "name": "Site", + "link": "https://site.example", + "link_page": "", + "avatar": "", + "reachable": True, + "crawlable": False, + "method": "api", + "latency": 3.0, + "fail_count": 2, + "checked_at": "2026-06-05 12:00:00", + "has_backlink": None, + "reason": "blocked_api_only", + }], + } + remote = { + "statistical_data": {}, + "link_data": [{ + "name": "Site", + "link": "https://site.example", + "link_page": "", + "avatar": "", + "reachable": True, + "crawlable": True, + "method": "proxy", + "latency": 1.0, + "fail_count": 0, + "checked_at": "2026-06-06 12:00:00", + "has_backlink": True, + "reason": "allowed_by_proxy", + }], + } + + class Response: + def json(self): + return remote + + with patch("requests.get", return_value=Response()): + merged = merge_link_data_from_json_url(local, "https://remote.example/link.json") + + self.assertEqual(merged["link_data"][0]["method"], "proxy") + self.assertEqual(merged["link_data"][0]["latency"], 1.0) + self.assertEqual(merged["link_data"][0]["fail_count"], 0) + self.assertTrue(merged["link_data"][0]["has_backlink"]) + self.assertEqual(merged["statistical_data"]["link_total_num"], 1) + + +if __name__ == "__main__": + unittest.main() From e781ccfe0b6681d56a096d100102e63350b6cbe3 Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Sun, 7 Jun 2026 02:33:33 +0800 Subject: [PATCH 19/30] =?UTF-8?q?=F0=9F=98=98=E6=9B=B4=E6=96=B0=E9=80=BB?= =?UTF-8?q?=E8=BE=91=EF=BC=8C=E5=AE=9E=E7=8E=B0=E6=9B=B4=E5=8A=A0=E4=BC=98?= =?UTF-8?q?=E9=9B=85=E7=9A=84=E5=AE=9E=E7=8E=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- conf.yaml | 82 ++- friend_circle_lite/cli.py | 20 +- friend_circle_lite/config/models.py | 26 +- friend_circle_lite/config/printer.py | 81 +-- friend_circle_lite/crawler/feed_service.py | 28 +- friend_circle_lite/crawler/http_client.py | 97 +++ friend_circle_lite/crawler/service.py | 86 +-- friend_circle_lite/domain/models.py | 43 +- friend_circle_lite/link_checker/service.py | 263 ++++++--- friend_circle_lite/outputs/legacy_api.py | 38 +- friend_circle_lite/storage/__init__.py | 1 + friend_circle_lite/storage/diagnostics.py | 248 ++++++++ friend_circle_lite/storage/sqlite_store.py | 29 +- static/index.html | 28 +- tests/test_refactor_contracts.py | 651 ++++++++++++++++++++- 15 files changed, 1454 insertions(+), 267 deletions(-) create mode 100644 friend_circle_lite/crawler/http_client.py create mode 100644 friend_circle_lite/storage/diagnostics.py diff --git a/conf.yaml b/conf.yaml index 537a07a106a..2f7c0f128f1 100644 --- a/conf.yaml +++ b/conf.yaml @@ -1,29 +1,35 @@ +# 调试开关 +# 说明：默认关闭。开启后，程序结束前会全量打印 SQLite 缓存表结构与所有数据，并保守清理核心表残留旧字段。 +# 支持环境变量 FCL_DEBUG=1 覆盖此配置。 +debug: false + # 爬虫相关配置 -# 解释：使用request实现友链文章爬取，并放置到根目录的all.json下 -# enable: 是否启用爬虫 -# json_url: 请填写对应格式json的地址，仅支持网络地址 -# article_count: 请填写每个博客需要获取的最大文章数量 +# 说明：从友链 JSON 中读取站点列表，抓取各站点 RSS 文章，并生成 all.json。 +# enable: 是否启用爬虫 +# json_url: 友链 JSON 地址，仅支持网络地址 +# article_count: 每个站点最多抓取的文章数量 spider_settings: enable: true json_url: "https://blog.liushen.fun/friend.json" article_count: 5 # 代理配置 -# 解释：用于友链检测和 RSS 抓取的代理服务，可以访问被墙站点并获取完整页面内容 -# proxy_url: 代理前缀，比如 Nginx 反向代理或 Cloudflare Worker -# 示例：https://nginx.430070.xyz/{url} 或 https://proxy.example.com?url={url} -# ⚠️ 注意：代理服务可能违反某些服务条款，请谨慎使用，建议仅用于调试 -# 支持环境变量 PROXY_URL 覆盖此配置（优先级更高） -# 留空则不使用代理 +# 说明：用于友链检测和 RSS 抓取。程序会先直连，请求失败且配置了代理时自动走代理。 +# proxy_url: 代理地址涉及一定违规风险和隐私风险，请尽量不要写入配置文件。 +# 推荐在仓库环境变量 PROXY_URL 中配置，并让此处保持为空。 +# 推荐格式为代理前缀，例如 https://nginx.430070.xyz/ +# 程序会自动拼接为 https://nginx.430070.xyz/https://example.com/feed.xml +# 兼容高级格式 https://proxy.example.com?url={url}，但普通反代场景不推荐这样写。 +# 留空则不使用代理。 proxy_settings: proxy_url: "" # 数据合并配置 -# 解释：合并多个数据源的结果，比如国内和国外执行结果，解决部分网站访问受限问题 -# enable: 是否启用数据合并功能 -# remote_base_url: 远程数据源的基础 URL，会自动拼接 /all.json、/link.json、/errors.json -# merge_article_data: 是否合并友圈文章数据（all.json） -# merge_link_check_data: 是否合并友链可达性数据（link.json），如果远程没有此文件会自动跳过 +# 说明：合并多个数据源的结果，比如国内和国外两条线路各自运行后的 all.json、link.json、errors.json。 +# enable: 是否启用数据合并 +# remote_base_url: 远程数据源基础 URL，会自动拼接 /all.json、/link.json、/errors.json +# merge_article_data: 是否合并友圈文章数据 +# merge_link_check_data: 是否合并友链可达性数据 merge_settings: enable: false remote_base_url: "https://fc.liushen.fun" @@ -31,16 +37,14 @@ merge_settings: merge_link_check_data: true # 友链可达性检测配置 -# 解释：先检查友链站点是否可达，再决定是否继续抓取该站点 RSS，检测结果会写入 link.json 供前端展示 -# enable: 是否启用友链可达性检测 +# 说明：友圈抓取依赖此检测结果，因此该检测始终启用。旧配置中的 link_check.enable 会被兼容读取但不再生效。 # max_age_hours: 同一友链检测结果缓存时间，默认 24 小时 -# timeout: 单次请求超时时间 +# timeout: 单次网页请求超时时间 # max_workers: 并发检测数量 -# status_api_url: 兜底状态码 API，API-only 结果只用于可达性展示，不参与 RSS 抓取 +# status_api_url: 兜底状态码 API；API-only 结果只用于可达性展示，不参与 RSS 抓取 # enable_backlink_check: 是否检测友链页是否包含你的站点链接 # author_url: 你的站点域名，用于反链检测，建议只填写域名 link_check: - enable: true max_age_hours: 24 timeout: 15 max_workers: 10 @@ -48,26 +52,16 @@ link_check: enable_backlink_check: true author_url: "blog.liushen.fun" -# 邮箱推送功能配置，暂未实现，等待后续开发 -# 解释：每天为指定邮箱推送所有友链文章的更新，仅能指定一个 -# enable: 是否启用邮箱推送功能 -# to_email: 收件人邮箱地址 -# subject: 邮件主题 -# body_template: 邮件正文的 HTML 模板文件 +# 邮件推送功能配置，暂未实现，等待后续开发 +# 说明：每次运行后向指定邮箱推送所有友链文章更新。 email_push: enable: false to_email: recipient@example.com subject: "今天的 RSS 订阅更新" body_template: "rss_template.html" -# 邮箱issue订阅功能配置 -# 解释：向在issue中提取的所有邮箱推送您网站中的更新，添加邮箱和删除邮箱均通过添加issue对应格式实现 -# enable: 是否启用邮箱推送功能 -# github_username: GitHub 用户名，用于构建issue api地址 -# github_repo: GitHub 仓库名，用于构建issue api地址 -# your_blog_url: 你的博客地址 -# website_info: 你的博客信息 -# title: 你的博客标题，如果启用了推送，用于生成邮件主题 +# 邮件 issue 订阅功能配置 +# 说明：从 GitHub issue 中提取订阅邮箱，并推送你自己站点的新文章。 rss_subscribe: enable: true github_username: willow-god @@ -78,23 +72,17 @@ rss_subscribe: title: "清羽飞扬" # SMTP 配置 -# 解释：使用其中的相关配置实现上面两种功能，若无推送要求可以不配置，请将以上两个配置置为false -# email: 发件人邮箱地址 -# server： SMTP 服务器地址 -# port： SMTP 端口号 -# use_tls： 是否使用 tls 加密 +# 说明：用于上方邮件相关功能。如果不使用邮件功能，可关闭 email_push 和 rss_subscribe。 smtp: email: notify@liushen.fun server: smtp.exmail.qq.com port: 465 use_tls: true -# 特殊RSS地址指定，可以置空但是不要删除！ -# 解释：用于指定特殊RSS地址，如B站专栏等不常见RSS地址后缀，可以添加多个 -# name: 友链名称 -# url: 指定的RSS地址 +# 特殊 RSS 地址指定 +# 说明：用于指定特殊 RSS 地址，比如 B 站专栏等不常见 RSS 地址后缀。可以置空，但不要删除此项。 specific_RSS: - - name: "阮一峰" - url: "http://feeds.feedburner.com/ruanyifeng" - # - name: "無名小栈" - # url: "https://blog.imsyy.top/rss.xml" \ No newline at end of file + - name: "阮一峰" + url: "http://feeds.feedburner.com/ruanyifeng" + # - name: "无名小栈" + # url: "https://blog.imsyy.top/rss.xml" diff --git a/friend_circle_lite/cli.py b/friend_circle_lite/cli.py index 3b712014f61..f572ab7c261 100644 --- a/friend_circle_lite/cli.py +++ b/friend_circle_lite/cli.py @@ -22,6 +22,7 @@ merge_errors_from_json_url, merge_link_data_from_json_url, ) +from friend_circle_lite.storage.diagnostics import SQLiteDebugDumper from friend_circle_lite.utils.json import write_json @@ -33,11 +34,20 @@ def __init__(self, config: ApplicationConfig): def run(self) -> None: """Execute the enabled application features in a stable order.""" - print_startup_config(self.config) - self.run_crawler_if_enabled() - mail_runtime = self.prepare_mail_runtime() - self.run_email_push_if_enabled(mail_runtime) - self.run_rss_subscription_if_enabled(mail_runtime) + try: + print_startup_config(self.config) + self.run_crawler_if_enabled() + mail_runtime = self.prepare_mail_runtime() + self.run_email_push_if_enabled(mail_runtime) + self.run_rss_subscription_if_enabled(mail_runtime) + finally: + self.dump_sqlite_debug_if_enabled() + + def dump_sqlite_debug_if_enabled(self) -> None: + """在 debug 开启时输出 SQLite 全量缓存数据，便于排查 Action 问题。""" + if not self.config.debug: + return + SQLiteDebugDumper(self.config.runtime_paths.cache_file).run() def run_crawler_if_enabled(self) -> None: """Run the article crawl and persist public output files when enabled.""" diff --git a/friend_circle_lite/config/models.py b/friend_circle_lite/config/models.py index 7635506cc74..08f7898e621 100644 --- a/friend_circle_lite/config/models.py +++ b/friend_circle_lite/config/models.py @@ -20,6 +20,25 @@ DEFAULT_LINK_JSON = "./link.json" +def _as_bool(value: object, default: bool = False) -> bool: + """稳健解析布尔值，兼容 YAML 布尔与字符串写法。""" + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "on"} + return bool(value) + + +def _env_flag(name: str) -> bool | None: + """读取布尔环境变量；未配置时返回 None，避免覆盖配置文件。""" + value = os.getenv(name) + if value is None: + return None + return _as_bool(value) + + @dataclass(slots=True) class MergeSettings: """Options for merging local crawl results with remote data sources.""" @@ -50,6 +69,7 @@ class SpiderSettings: class LinkCheckConfig: """Settings for friend link reachability checks.""" + # 兼容旧配置字段。当前抓取流程依赖可达性检测，因此运行时会始终视为启用。 enable: bool = True max_age_hours: int = 24 timeout: int = 15 @@ -122,6 +142,7 @@ class ApplicationConfig: specific_rss: list[dict] runtime_paths: RuntimePaths = field(default_factory=RuntimePaths) future_article_tolerance_days: int = 2 + debug: bool = False @classmethod def from_dict(cls, data: dict) -> "ApplicationConfig": @@ -135,6 +156,8 @@ def from_dict(cls, data: dict) -> "ApplicationConfig": website_info_raw = rss_subscribe_raw.get("website_info", {}) smtp_raw = data.get("smtp", {}) runtime_raw = data.get("runtime_paths", {}) + debug_from_env = _env_flag("FCL_DEBUG") + debug_enabled = debug_from_env if debug_from_env is not None else _as_bool(data.get("debug"), False) return cls( spider_settings=SpiderSettings( @@ -152,7 +175,7 @@ def from_dict(cls, data: dict) -> "ApplicationConfig": merge_link_check_data=bool(merge_raw.get("merge_link_check_data", True)), ), link_check=LinkCheckConfig( - enable=bool(link_check_raw.get("enable", True)), + enable=True, max_age_hours=int(link_check_raw.get("max_age_hours", 24)), timeout=int(link_check_raw.get("timeout", 15)), max_workers=int(link_check_raw.get("max_workers", 10)), @@ -189,6 +212,7 @@ def from_dict(cls, data: dict) -> "ApplicationConfig": errors_json_file=str(runtime_raw.get("errors_json_file", DEFAULT_ERRORS_JSON)).strip() or DEFAULT_ERRORS_JSON, link_json_file=str(runtime_raw.get("link_json_file", DEFAULT_LINK_JSON)).strip() or DEFAULT_LINK_JSON, ), + debug=debug_enabled, ) diff --git a/friend_circle_lite/config/printer.py b/friend_circle_lite/config/printer.py index e87c82a4214..0a4f0ad4222 100644 --- a/friend_circle_lite/config/printer.py +++ b/friend_circle_lite/config/printer.py @@ -1,54 +1,61 @@ -"""Configuration printer for startup diagnostics.""" +"""启动时打印关键配置。 + +本文件只负责把当前生效配置输出到日志，方便在 GitHub Action 或本地运行时确认： +- 爬虫数据源与文章数量； +- 代理、友链可达性检测、数据合并参数； +- 邮件与 RSS 订阅开关； +- debug 诊断开关。 +""" + +from __future__ import annotations import logging -def print_startup_config(config): - """Print all configuration settings at startup for debugging.""" +def print_startup_config(config) -> None: + """打印启动配置，避免把配置解析细节散落在主流程中。""" logging.info("=" * 60) - logging.info("🚀 Friend-Circle-Lite 启动配置") + logging.info("Friend-Circle-Lite 启动配置") logging.info("=" * 60) - # Spider settings - logging.info("📡 爬虫配置:") - logging.info(f" - 启用状态: {'✅ 已启用' if config.spider_settings.enable else '❌ 已禁用'}") + logging.info("爬虫配置:") + logging.info(f" - 启用状态: {'已启用' if config.spider_settings.enable else '已禁用'}") if config.spider_settings.enable: - logging.info(f" - 数据源: {config.spider_settings.json_url}") + logging.info(f" - 数据源: {config.spider_settings.json_url} ") logging.info(f" - 每站文章数: {config.spider_settings.article_count}") - # Proxy settings - logging.info("🔀 代理配置:") + logging.info("代理配置:") if config.proxy_settings.proxy_url: - logging.info(f" - 代理地址: {config.proxy_settings.proxy_url}") - logging.info(f" - 用途: 友链检测 + RSS 抓取") + logging.info(" - 代理状态: 已配置（日志不显示具体地址）") + logging.info(" - 建议: 使用仓库环境变量 PROXY_URL 覆盖，避免代理地址出现在配置文件中") + logging.info(" - 用途: 友链检测 + RSS 抓取") else: - logging.info(f" - 代理状态: ❌ 未配置") - - # Link check settings - logging.info("🔍 友链检测配置:") - logging.info(f" - 启用状态: {'✅ 已启用' if config.link_check.enable else '❌ 已禁用'}") - if config.link_check.enable: - logging.info(f" - 缓存时间: {config.link_check.max_age_hours} 小时") - logging.info(f" - 超时时间: {config.link_check.timeout} 秒") - logging.info(f" - 并发数: {config.link_check.max_workers}") - logging.info(f" - 反链检测: {'✅ 已启用' if config.link_check.enable_backlink_check else '❌ 已禁用'}") - if config.link_check.enable_backlink_check: - logging.info(f" - 站点域名: {config.link_check.author_url}") - - # Merge settings - logging.info("🔗 数据合并配置:") - logging.info(f" - 启用状态: {'✅ 已启用' if config.merge_settings.enable else '❌ 已禁用'}") + logging.info(" - 代理状态: 未配置") + + logging.info("友链检测配置:") + logging.info(" - 启用状态: 始终启用（友圈抓取依赖此检测结果）") + logging.info(f" - 缓存时间: {config.link_check.max_age_hours} 小时") + logging.info(f" - 超时时间: {config.link_check.timeout} 秒") + logging.info(f" - 并发数: {config.link_check.max_workers}") + logging.info(f" - 状态 API: {config.link_check.status_api_url} ") + logging.info(f" - 反链检测: {'已启用' if config.link_check.enable_backlink_check else '已禁用'}") + if config.link_check.enable_backlink_check: + logging.info(f" - 站点域名: {config.link_check.author_url} ") + + logging.info("数据合并配置:") + logging.info(f" - 启用状态: {'已启用' if config.merge_settings.enable else '已禁用'}") if config.merge_settings.enable: - logging.info(f" - 远程数据源: {config.merge_settings.remote_base_url}") - logging.info(f" - 合并文章数据: {'✅ 是' if config.merge_settings.merge_article_data else '❌ 否'}") - logging.info(f" - 合并友链数据: {'✅ 是' if config.merge_settings.merge_link_check_data else '❌ 否'}") + logging.info(f" - 远程数据源: {config.merge_settings.remote_base_url} ") + logging.info(f" - 合并文章数据: {'是' if config.merge_settings.merge_article_data else '否'}") + logging.info(f" - 合并友链数据: {'是' if config.merge_settings.merge_link_check_data else '否'}") + + logging.info("邮件推送配置:") + logging.info(f" - 启用状态: {'已启用' if config.email_push.enable else '已禁用'}") - # Email push settings - logging.info("📧 邮件推送配置:") - logging.info(f" - 启用状态: {'✅ 已启用' if config.email_push.enable else '❌ 已禁用'}") + logging.info("RSS 订阅配置:") + logging.info(f" - 启用状态: {'已启用' if config.rss_subscribe.enable else '已禁用'}") - # RSS subscribe settings - logging.info("📮 RSS 订阅配置:") - logging.info(f" - 启用状态: {'✅ 已启用' if config.rss_subscribe.enable else '❌ 已禁用'}") + logging.info("调试配置:") + logging.info(f" - SQLite 全量输出: {'已启用' if config.debug else '已禁用'}") logging.info("=" * 60) diff --git a/friend_circle_lite/crawler/feed_service.py b/friend_circle_lite/crawler/feed_service.py index 3f6736f8e77..8afb2d0cbb4 100644 --- a/friend_circle_lite/crawler/feed_service.py +++ b/friend_circle_lite/crawler/feed_service.py @@ -1,4 +1,4 @@ -"""Feed discovery, parsing, and incremental tracking services.""" +"""RSS 发现、解析与文章更新追踪服务。""" from __future__ import annotations @@ -12,7 +12,9 @@ import requests from friend_circle_lite import HEADERS_XML, timeout -from friend_circle_lite.domain.models import Article, FeedEndpoint, Website +from friend_circle_lite.config.models import ProxySettings +from friend_circle_lite.crawler.http_client import WebFetchClient +from friend_circle_lite.domain.models import Article, FeedEndpoint, Website, normalize_latency from friend_circle_lite.utils.time import format_published_time from friend_circle_lite.utils.url import replace_non_domain @@ -34,19 +36,19 @@ class FeedDiscoveryService: ("rss11", "/feed.php"), # 同上 ] - def __init__(self, session: requests.Session): + def __init__(self, session: requests.Session, proxy_settings: ProxySettings | None = None): self.session = session + self.fetcher = WebFetchClient(session, proxy_settings) + self.last_latency = 0.01 def discover(self, website_url: str) -> FeedEndpoint | None: """Try common feed endpoints and return the first valid match.""" for feed_type, path in self.POSSIBLE_FEEDS: feed_url = website_url.rstrip("/") + path - try: - response = self.session.get(feed_url, headers=HEADERS_XML, timeout=timeout) - except requests.RequestException: - continue + result = self.fetcher.get(feed_url, headers=HEADERS_XML, timeout=timeout, desc="RSS 探测") + response = result.response - if response.status_code != 200: + if response is None or response.status_code != 200: continue content_type = response.headers.get("Content-Type", "").lower() @@ -64,8 +66,10 @@ def discover(self, website_url: str) -> FeedEndpoint | None: class FeedParserService: """Parse a discovered feed into normalized article objects.""" - def __init__(self, session: requests.Session): + def __init__(self, session: requests.Session, proxy_settings: ProxySettings | None = None): self.session = session + self.fetcher = WebFetchClient(session, proxy_settings) + self.last_latency = 0.01 def parse(self, feed_url: str, count: int = 5, blog_url: str = "") -> list[Article]: """Parse a feed URL and return the newest `count` articles. @@ -74,7 +78,11 @@ def parse(self, feed_url: str, count: int = 5, blog_url: str = "") -> list[Artic model, while preserving the original public output fields. """ try: - response = self.session.get(feed_url, headers=HEADERS_XML, timeout=timeout) + result = self.fetcher.get(feed_url, headers=HEADERS_XML, timeout=timeout, desc="RSS 抓取") + self.last_latency = normalize_latency(result.latency) + if result.response is None: + return [] + response = result.response # 强制使用 UTF-8 编码，因为 apparent_encoding 可能检测错误 response.encoding = "utf-8" feed = feedparser.parse(response.text) diff --git a/friend_circle_lite/crawler/http_client.py b/friend_circle_lite/crawler/http_client.py new file mode 100644 index 00000000000..fe903f23e7e --- /dev/null +++ b/friend_circle_lite/crawler/http_client.py @@ -0,0 +1,97 @@ +"""统一网页请求封装。 + +本模块负责把“直连优先，失败后自动尝试代理”的请求逻辑收口到一个地方。 +调用方只关心是否拿到响应，不需要感知重试细节。 +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass + +import requests + +from friend_circle_lite.config.models import ProxySettings +from friend_circle_lite.domain.models import normalize_latency + + +@dataclass(slots=True) +class FetchResult: + """一次网页请求的结果。""" + + response: requests.Response | None + latency: float = -1 + used_proxy: bool = False + + @property + def success(self) -> bool: + """是否成功取得 HTTP 200 响应。""" + return self.response is not None and self.response.status_code == 200 + + +class WebFetchClient: + """网页请求客户端，封装直连和代理回退逻辑。""" + + def __init__(self, session: requests.Session, proxy_settings: ProxySettings | None = None): + self.session = session + self.proxy_settings = proxy_settings or ProxySettings() + + def get( + self, + url: str, + headers: dict[str, str] | None = None, + timeout: int | tuple | None = None, + desc: str = "网页请求", + ) -> FetchResult: + """先直连请求，失败时自动尝试代理请求。""" + direct = self._get_once(url, headers=headers, timeout=timeout, desc=desc, used_proxy=False) + if direct.success or not self.proxy_settings.proxy_url: + return direct + + proxy_url = self._build_proxy_url(url) + proxy = self._get_once( + proxy_url, + headers=headers, + timeout=timeout, + desc=f"{desc} 代理", + used_proxy=True, + display_url=f"{url} （通过代理）", + ) + return proxy if proxy.success else direct + + def _get_once( + self, + url: str, + headers: dict[str, str] | None, + timeout: int | tuple | None, + desc: str, + used_proxy: bool, + display_url: str | None = None, + ) -> FetchResult: + log_url = display_url or url + start_time = time.time() + try: + response = self.session.get(url, headers=headers, timeout=timeout) + latency = self._elapsed_latency(start_time) + if response.status_code == 200: + logging.info(f"[{desc}] 成功访问: {log_url} ，延迟 {latency} 秒") + else: + logging.warning(f"[{desc}] 状态码异常: {log_url} -> {response.status_code}") + return FetchResult(response=response, latency=latency, used_proxy=used_proxy) + except requests.RequestException as exc: + error_text = exc.__class__.__name__ if used_proxy else str(exc) + logging.warning(f"[{desc}] 请求失败: {log_url} ，错误: {error_text}") + return FetchResult(response=None, latency=self._elapsed_latency(start_time), used_proxy=used_proxy) + + def _build_proxy_url(self, url: str) -> str: + proxy_url = self.proxy_settings.proxy_url + if "{}" in proxy_url: + return proxy_url.format(url) + if "{url}" in proxy_url: + return proxy_url.format(url=url) + return f"{proxy_url}{url}" + + @staticmethod + def _elapsed_latency(start_time: float) -> float: + return normalize_latency(time.time() - start_time) diff --git a/friend_circle_lite/crawler/service.py b/friend_circle_lite/crawler/service.py index 1efb487171c..0b7889f316a 100644 --- a/friend_circle_lite/crawler/service.py +++ b/friend_circle_lite/crawler/service.py @@ -69,22 +69,7 @@ def crawl(self, website: Website, count: int) -> CrawlResult: parse_error = endpoint is not None and not articles if parse_error and endpoint and endpoint.source in ("cache", "unknown"): - logging.warning(f"'{website.name}' 缓存的 RSS 源无效，尝试重新探测...") - rediscovered = self.resolver.discovery_service.discover(website.url) - if rediscovered: - articles = self._parse_articles(rediscovered, website, count) - if articles: - endpoint = rediscovered - cache_update = CacheUpdate(action="set", name=website.name, url=rediscovered.url, reason="repair_cache") - logging.info(f"'{website.name}' 重新探测成功，更新缓存：{rediscovered.url}") - else: - endpoint = None - cache_update = CacheUpdate(action="delete", name=website.name, url=None, reason="remove_invalid") - logging.warning(f"'{website.name}' 重新探测失败，删除无效缓存") - else: - endpoint = None - cache_update = CacheUpdate(action="delete", name=website.name, url=None, reason="remove_invalid") - logging.warning(f"'{website.name}' 未找到有效 RSS，删除无效缓存") + logging.warning(f"'{website.name}' 缓存 RSS 本次抓取失败，将等待下次友链检测刷新 RSS 缓存") status = "active" if articles else "error" if not articles: @@ -131,7 +116,7 @@ def __init__( self.count = count self.specific_rss = specific_rss or [] self.cache_store = FeedCacheStore(cache_file) - self.link_check_config = link_check_config or LinkCheckConfig(enable=False) + self.link_check_config = link_check_config or LinkCheckConfig() self.proxy_settings = proxy_settings or ProxySettings() self.link_check_store = LinkCheckStore(cache_file) @@ -142,20 +127,28 @@ def run(self) -> tuple[dict, list[list[str]]] | None: if websites is None: return None - link_check_records = self._check_links(websites) - link_check_map = {record.url: record for record in link_check_records} - crawlable_websites = [website for website in websites if link_check_map.get(website.url, LinkCheckRecord.unchecked(website)).crawl_allowed] - skipped_count = len(websites) - len(crawlable_websites) - if skipped_count: - logging.info(f"🔎 根据友链可达性检测跳过 {skipped_count} 个不可抓取站点") - cache_records = self.cache_store.load_records() manual_records = self._build_manual_records() merged_records = self._merge_feed_records(cache_records, manual_records) manual_names = {record.name for record in manual_records} - discovery_service = FeedDiscoveryService(session) - parser_service = FeedParserService(session) + link_check_records = self._check_links(websites, merged_records, manual_names) + link_check_map = {record.url: record for record in link_check_records} + + cache_records = self.cache_store.load_records() + merged_records = self._merge_feed_records(cache_records, manual_records) + feed_names = {record.name for record in merged_records} + crawlable_websites = [ + website for website in websites + if link_check_map.get(website.url, LinkCheckRecord.unchecked(website)).crawl_allowed + and website.name in feed_names + ] + skipped_count = len(websites) - len(crawlable_websites) + if skipped_count: + logging.info(f"🔎 根据友链可达性检测跳过 {skipped_count} 个不可抓取站点") + + discovery_service = FeedDiscoveryService(session, self.proxy_settings) + parser_service = FeedParserService(session, self.proxy_settings) resolver = FeedResolver(discovery_service=discovery_service, configured_feeds=merged_records) crawler = SingleSiteCrawler(parser_service=parser_service, resolver=resolver) @@ -188,7 +181,6 @@ def run(self) -> tuple[dict, list[list[str]]] | None: article_num=len(all_articles), ) stats_payload = statistics.to_dict() - stats_payload.update(self._build_link_statistics(link_check_records)) result = { "statistical_data": stats_payload, "article_data": all_articles, @@ -200,9 +192,36 @@ def run(self) -> tuple[dict, list[list[str]]] | None: ) return result, error_results, link_payload - def _check_links(self, websites: list[Website]) -> list[LinkCheckRecord]: - service = LinkReachabilityService(config=self.link_check_config, proxy_settings=self.proxy_settings, store=self.link_check_store) - return service.check_websites(websites) + def _check_links(self, websites: list[Website], feed_records: list[CacheRecord], manual_names: set[str]) -> list[LinkCheckRecord]: + service = LinkReachabilityService( + config=self.link_check_config, + proxy_settings=self.proxy_settings, + store=self.link_check_store, + feed_records=feed_records, + ) + records = service.check_websites(websites) + if service.feed_updates: + self._apply_feed_updates_from_link_check(service.feed_updates, manual_names) + return records + + def _apply_feed_updates_from_link_check(self, updates: dict[str, CacheRecord | None], manual_names: set[str]) -> None: + """保存可达性检测阶段发现或失效的 RSS 缓存。""" + cache_map = {record.name: record for record in self.cache_store.load_records()} + changed = False + for name, record in updates.items(): + if name in manual_names: + continue + if record is None: + if name in cache_map: + cache_map.pop(name) + changed = True + logging.info(f"🗑️ 可达性检测删除失效 RSS 缓存: {name}") + else: + cache_map[name] = record + changed = True + logging.info(f"💾 可达性检测保存 RSS 缓存: {name} -> {record.url}") + if changed: + self.cache_store.save_records(list(cache_map.values())) @staticmethod def _build_link_statistics(records: list[LinkCheckRecord]) -> dict[str, int | str]: @@ -268,13 +287,14 @@ def _load_websites(self, session: requests.Session) -> list[Website] | None: logging.error(f"无法获取链接：{self.json_url} ：{exc}", exc_info=True) return None - websites: list[Website] = [] + website_map: dict[str, Website] = {} for friend in friends_data.get("friends", []): try: - websites.append(Website.from_friend_item(friend)) + website = Website.from_friend_item(friend) + website_map[website.url] = website except Exception: logging.warning(f"发现格式异常的友链数据，已跳过: {friend!r}") - return websites + return list(website_map.values()) def _build_manual_records(self) -> list[CacheRecord]: manual_records: list[CacheRecord] = [] diff --git a/friend_circle_lite/domain/models.py b/friend_circle_lite/domain/models.py index b8ab4390232..fabeaf46c82 100644 --- a/friend_circle_lite/domain/models.py +++ b/friend_circle_lite/domain/models.py @@ -9,10 +9,38 @@ from dataclasses import dataclass, field from datetime import datetime +from urllib.parse import urlsplit, urlunsplit from zoneinfo import ZoneInfo SHANGHAI_TZ = ZoneInfo("Asia/Shanghai") +MIN_RECORDED_LATENCY = 0.01 + + +def normalize_latency(value: float | int | str | None, default: float = MIN_RECORDED_LATENCY) -> float: + """规范化延迟值，对外记录时不使用 0 或负数表示未知。""" + try: + latency = float(value) + except (TypeError, ValueError): + return default + if latency <= 0: + return default + return max(round(latency, 2), default) + + +def normalize_homepage_url(url: str) -> str: + """规范化站点主页 URL，用于缓存匹配和持久化。""" + value = str(url or "").strip() + if not value: + return "" + try: + parts = urlsplit(value) + if parts.scheme and parts.netloc: + path = (parts.path.rstrip("/") + "/") if parts.path else "/" + return urlunsplit((parts.scheme.lower(), parts.netloc.lower(), path, parts.query, parts.fragment)) + except Exception: + pass + return value.rstrip("/") + "/" @dataclass(slots=True) @@ -24,13 +52,16 @@ class Website: avatar: str = "" linkpage: str = "" + def __post_init__(self) -> None: + self.url = normalize_homepage_url(self.url) + @classmethod def from_friend_item(cls, raw_friend: list | tuple | dict) -> "Website": """Create a website from common friend link structures.""" if isinstance(raw_friend, dict): return cls( name=str(raw_friend.get("name", "")).strip(), - url=str(raw_friend.get("link") or raw_friend.get("url") or "").strip(), + url=normalize_homepage_url(raw_friend.get("link") or raw_friend.get("url") or ""), avatar=str(raw_friend.get("avatar", "")).strip(), linkpage=str(raw_friend.get("linkpage", "")).strip(), ) @@ -43,7 +74,7 @@ def from_friend_item(cls, raw_friend: list | tuple | dict) -> "Website": else: linkpage = "" avatar = raw_friend[2] if len(raw_friend) > 2 else "" - return cls(name=str(name).strip(), url=str(url).strip(), avatar=str(avatar or "").strip(), linkpage=str(linkpage or "").strip()) + return cls(name=str(name).strip(), url=normalize_homepage_url(url), avatar=str(avatar or "").strip(), linkpage=str(linkpage or "").strip()) def to_error_payload(self) -> list[str]: """Return the legacy structure used by `errors.json`.""" @@ -99,6 +130,9 @@ class LinkCheckRecord: def unchecked(cls, website: Website, checked_at: str = "") -> "LinkCheckRecord": return cls(name=website.name, url=website.url, avatar=website.avatar, linkpage=website.linkpage, checked_at=checked_at) + def __post_init__(self) -> None: + self.url = normalize_homepage_url(self.url) + def to_public_dict(self) -> dict[str, object]: return { "name": self.name, @@ -128,12 +162,9 @@ def to_link_dict(self) -> dict[str, object]: "avatar": self.avatar, "reachable": self.reachable, "crawlable": self.crawl_allowed, - "method": self.best_method, - "latency": self.best_latency, + "latency": normalize_latency(self.best_latency), "fail_count": self.fail_count, - "checked_at": self.checked_at, "has_backlink": self.has_author_link if self.backlink_checked else None, - "reason": self.rss_crawl_reason, } diff --git a/friend_circle_lite/link_checker/service.py b/friend_circle_lite/link_checker/service.py index e6e21bd8c3e..ddc2168b63d 100644 --- a/friend_circle_lite/link_checker/service.py +++ b/friend_circle_lite/link_checker/service.py @@ -1,4 +1,12 @@ -"""Friend link reachability checks used before RSS crawling.""" +"""友链可达性与 RSS 可抓取性检测。 + +检测策略： +1. 优先检查手动 RSS 或缓存 RSS，能解析文章则认为站点可达且可抓取。 +2. RSS 不可用时自动探测常见 RSS 地址。 +3. 仍找不到 RSS 时检查主页，主页可访问则只标记可达，不参与朋友圈抓取。 +4. 主页不可访问时保留 API 兜底，用于判断站点是否可能可达。 +5. 反链检测只在站点可达时执行。 +""" from __future__ import annotations @@ -6,12 +14,14 @@ import time from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import datetime -from urllib.parse import quote, urlparse +from urllib.parse import quote, urlparse, urlsplit, urlunsplit import requests from friend_circle_lite.config.models import LinkCheckConfig, ProxySettings -from friend_circle_lite.domain.models import LinkCheckRecord, LinkMethodStatus, Website +from friend_circle_lite.crawler.feed_service import FeedDiscoveryService, FeedParserService +from friend_circle_lite.crawler.http_client import WebFetchClient +from friend_circle_lite.domain.models import CacheRecord, FeedEndpoint, LinkCheckRecord, LinkMethodStatus, Website, normalize_latency from friend_circle_lite.storage.sqlite_store import LinkCheckStore @@ -35,43 +45,65 @@ class LinkReachabilityService: - """Check friend homepage reachability and cache results.""" + """检查友链是否可达，以及是否可参与 RSS 抓取。""" - def __init__(self, config: LinkCheckConfig, proxy_settings: ProxySettings, store: LinkCheckStore): + def __init__( + self, + config: LinkCheckConfig, + proxy_settings: ProxySettings, + store: LinkCheckStore, + feed_records: list[CacheRecord] | None = None, + feed_parser=None, + feed_discovery=None, + fetcher: WebFetchClient | None = None, + ): self.config = config self.proxy_settings = proxy_settings self.store = store + self.feed_lookup = {record.name: record for record in (feed_records or [])} + self.feed_parser = feed_parser + self.feed_discovery = feed_discovery + self.fetcher = fetcher + self.feed_updates: dict[str, CacheRecord | None] = {} def check_websites(self, websites: list[Website]) -> list[LinkCheckRecord]: - if not self.config.enable: - now = self._now_text() - return [self._build_disabled_record(website, now) for website in websites] - + """检查一组友链，优先复用未过期缓存。""" cached_records = self.store.load_records([website.url for website in websites]) records_by_url: dict[str, LinkCheckRecord] = {} websites_to_check: list[Website] = [] + backlink_refresh_records: list[tuple[Website, LinkCheckRecord]] = [] for website in websites: cached = cached_records.get(website.url) if cached and self._can_reuse_cached_record(cached, website): - records_by_url[website.url] = self._refresh_cached_metadata(cached, website) + linkpage_changed = not self._same_linkpage(cached.linkpage, website.linkpage) + refreshed = self._refresh_cached_metadata(cached, website) + records_by_url[website.url] = refreshed + if self._should_refresh_backlink(refreshed, website, linkpage_changed): + backlink_refresh_records.append((website, refreshed)) else: websites_to_check.append(website) if websites_to_check: - logging.info(f"🔎 开始检测 {len(websites_to_check)} 个友链可达性") + logging.info(f"🔎 开始检测 {len(websites_to_check)} 个友链状态") checked_records = self._check_fresh_websites(websites_to_check, cached_records) self.store.save_records(checked_records) for record in checked_records: records_by_url[record.url] = record else: - logging.info("🔎 友链可达性检测缓存仍有效，本次复用缓存结果") + logging.info("🔎 友链状态缓存仍有效，本次复用缓存结果") + + if backlink_refresh_records: + self._refresh_backlinks_only(backlink_refresh_records) return [records_by_url.get(website.url) or LinkCheckRecord.unchecked(website) for website in websites] def _check_fresh_websites(self, websites: list[Website], cached_records: dict[str, LinkCheckRecord]) -> list[LinkCheckRecord]: records: list[LinkCheckRecord] = [] with requests.Session() as session: + self.feed_parser = self.feed_parser or FeedParserService(session, self.proxy_settings) + self.feed_discovery = self.feed_discovery or FeedDiscoveryService(session, self.proxy_settings) + self.fetcher = self.fetcher or WebFetchClient(session, self.proxy_settings) with ThreadPoolExecutor(max_workers=max(1, self.config.max_workers)) as executor: future_to_website = { executor.submit(self._check_website, session, website, cached_records.get(website.url)): website @@ -87,86 +119,115 @@ def _check_fresh_websites(self, websites: list[Website], cached_records: dict[st return records def _check_website(self, session: requests.Session, website: Website, cached: LinkCheckRecord | None) -> LinkCheckRecord: - direct = self._request_method(session, website.url, "直接访问") - proxy = LinkMethodStatus() - api = LinkMethodStatus() - - if not direct.success: - proxy_url = self._build_proxy_url(website.url) - if proxy_url: - proxy = self._request_method(session, proxy_url, "代理访问") - - if not direct.success and not proxy.success: - api = self._request_api(session, website.url) - time.sleep(0.2) + record = self._check_rss_first(website, cached) + if record is None: + homepage = self._request_homepage(website.url) + api = LinkMethodStatus() + if not homepage.success: + api = self._request_api(session, website.url) + time.sleep(0.2) + record = self._compose_non_rss_record(website, cached, homepage, api) - record = self._compose_record(website, cached, direct, proxy, api) if record.reachable and self.config.enable_backlink_check and self.config.author_url and website.linkpage: record.backlink_checked = True record.has_author_link = self._check_author_link_in_page(session, website.linkpage) + elif not record.reachable: + record.backlink_checked = bool(website.linkpage) + record.has_author_link = False return record - def _request_method(self, session: requests.Session, url: str, desc: str) -> LinkMethodStatus: + def _check_rss_first(self, website: Website, cached: LinkCheckRecord | None) -> LinkCheckRecord | None: + configured = self.feed_lookup.get(website.name) + if configured: + endpoint = FeedEndpoint(url=configured.url, feed_type="specific", source=configured.source) + if self._feed_has_articles(endpoint, website): + return self._build_feed_record(website, endpoint, self._last_feed_latency()) + logging.warning(f"友链 {website.name} 的缓存 RSS 失效: {configured.url} ，开始重新探测") + if configured.source == "cache": + self.feed_updates[website.name] = None + + discovered = self.feed_discovery.discover(website.url) if self.feed_discovery else None + if discovered and self._feed_has_articles(discovered, website): + self.feed_updates[website.name] = CacheRecord(name=website.name, url=discovered.url, source="cache") + return self._build_feed_record(website, discovered, self._last_feed_latency()) + return None + + def _feed_has_articles(self, endpoint: FeedEndpoint, website: Website) -> bool: + articles = self.feed_parser.parse(endpoint.url, count=1, blog_url=website.url) + return bool(articles) + + def _last_feed_latency(self) -> float: + return normalize_latency(getattr(self.feed_parser, "last_latency", None)) + + def _build_feed_record(self, website: Website, endpoint: FeedEndpoint, latency: float) -> LinkCheckRecord: + method = "rss_cache" if endpoint.source == "cache" else "rss" + return LinkCheckRecord( + name=website.name, + url=website.url, + avatar=website.avatar, + linkpage=website.linkpage, + checked_at=self._now_text(), + reachable=True, + crawl_allowed=True, + best_method=method, + best_latency=latency, + fail_count=0, + rss_crawl_reason=f"allowed_by_{method}", + ) + + def _request_homepage(self, url: str) -> LinkMethodStatus: if not self._is_url(url): return LinkMethodStatus() - - response, latency = self._request_url(session, url, headers=LINK_CHECK_HEADERS, desc=desc) - if response is None: - return LinkMethodStatus(success=False, status_code=None, latency=latency) - success = response.status_code == 200 - if success: - logging.info(f"[{desc}] 成功访问: {url}，延迟 {latency} 秒") - else: - logging.warning(f"[{desc}] 状态码异常: {url} -> {response.status_code}") - return LinkMethodStatus(success=success, status_code=response.status_code, latency=latency) + result = self.fetcher.get(url, headers=LINK_CHECK_HEADERS, timeout=self.config.timeout, desc="主页检测") + if result.response is None: + return LinkMethodStatus(success=False, status_code=None, latency=result.latency) + return LinkMethodStatus(success=result.success, status_code=result.response.status_code, latency=result.latency) def _request_api(self, session: requests.Session, url: str) -> LinkMethodStatus: if not self.config.status_api_url: return LinkMethodStatus() api_url = self.config.status_api_url.format(url=quote(url, safe="")) - response, latency = self._request_url(session, api_url, headers=RAW_HEADERS, desc="API 检查", timeout=30) - if response is None: - return LinkMethodStatus(success=False, status_code=None, latency=latency) + start_time = time.time() + try: + response = session.get(api_url, headers=RAW_HEADERS, timeout=30) + latency = normalize_latency(time.time() - start_time) + except requests.RequestException as exc: + logging.warning(f"[API 检查] 请求失败: {url} ，错误: {exc}") + return LinkMethodStatus(success=False, status_code=None, latency=normalize_latency(time.time() - start_time)) try: payload = response.json() status_code = int(payload.get("data", 0)) success = int(payload.get("code", 0)) == 200 and status_code == 200 if success: - logging.info(f"[API] 成功访问: {url}，状态码 200") + logging.info(f"[API 检查] 成功访问: {url} ，状态码 200") else: - logging.warning(f"[API] 状态异常: {url} -> [{payload.get('code')}, {payload.get('data')}]") + logging.warning(f"[API 检查] 状态异常: {url} -> [{payload.get('code')}, {payload.get('data')}]") return LinkMethodStatus(success=success, status_code=status_code, latency=latency) except Exception as exc: - logging.warning(f"[API] 解析响应失败: {url}，错误: {exc}") + logging.warning(f"[API 检查] 解析响应失败: {url} ，错误: {exc}") return LinkMethodStatus(success=False, status_code=response.status_code, latency=latency) - def _compose_record( + def _compose_non_rss_record( self, website: Website, cached: LinkCheckRecord | None, - direct: LinkMethodStatus, - proxy: LinkMethodStatus, + homepage: LinkMethodStatus, api: LinkMethodStatus, ) -> LinkCheckRecord: - reachable = direct.success or proxy.success or api.success - crawl_allowed = direct.success or proxy.success - if direct.success: - best_method = "direct" - best_latency = direct.latency - reason = "allowed_by_direct" - elif proxy.success: - best_method = "proxy" - best_latency = proxy.latency - reason = "allowed_by_proxy" + reachable = homepage.success or api.success + if homepage.success: + best_method = "homepage" + best_latency = homepage.latency + reason = "reachable_without_rss" elif api.success: best_method = "api" best_latency = api.latency - reason = "blocked_api_only" + reason = "api_reachable_without_rss" else: best_method = "none" - best_latency = -1 + best_latency = self._first_measured_latency(homepage, api) reason = "blocked_unreachable" fail_count = 0 if reachable else ((cached.fail_count if cached else 0) + 1) @@ -177,19 +238,27 @@ def _compose_record( linkpage=website.linkpage, checked_at=self._now_text(), reachable=reachable, - crawl_allowed=crawl_allowed, + crawl_allowed=False, best_method=best_method, best_latency=best_latency, fail_count=fail_count, rss_crawl_reason=reason, - direct=direct, - proxy=proxy, + direct=homepage, api=api, ) + @staticmethod + def _first_measured_latency(*statuses: LinkMethodStatus) -> float: + for status in statuses: + if status.latency > 0: + return normalize_latency(status.latency) + return normalize_latency(None) + def _check_author_link_in_page(self, session: requests.Session, linkpage_url: str) -> bool: - response, _ = self._request_url(session, linkpage_url, headers=RAW_HEADERS, desc="友链页面检测") - if not response: + fetcher = self.fetcher or WebFetchClient(session, self.proxy_settings) + result = fetcher.get(linkpage_url, headers=RAW_HEADERS, timeout=self.config.timeout, desc="友链页面检测") + response = result.response + if response is None: return False author_url = self.config.author_url @@ -218,24 +287,14 @@ def _check_author_link_in_page(self, session: requests.Session, linkpage_url: st return True return False - def _request_url( - self, - session: requests.Session, - url: str, - headers: dict[str, str], - desc: str, - timeout: int | None = None, - ) -> tuple[requests.Response | None, float]: - try: - start_time = time.time() - response = session.get(url, headers=headers, timeout=timeout or self.config.timeout) - return response, round(time.time() - start_time, 2) - except requests.RequestException as exc: - logging.warning(f"[{desc}] 请求失败: {url}，错误: {exc}") - return None, -1 - def _can_reuse_cached_record(self, cached: LinkCheckRecord, website: Website) -> bool: - if self.config.enable_backlink_check and cached.linkpage != website.linkpage: + try: + has_measured_latency = float(cached.best_latency) > 0 + except (TypeError, ValueError): + has_measured_latency = False + if not has_measured_latency: + return False + if cached.crawl_allowed and website.name not in self.feed_lookup: return False return self.store.is_fresh(cached, self.config.max_age_hours) @@ -246,6 +305,23 @@ def _refresh_cached_metadata(cached: LinkCheckRecord, website: Website) -> LinkC cached.linkpage = website.linkpage return cached + def _should_refresh_backlink(self, record: LinkCheckRecord, website: Website, linkpage_changed: bool) -> bool: + return bool( + linkpage_changed + and record.reachable + and self.config.enable_backlink_check + and self.config.author_url + and website.linkpage + ) + + def _refresh_backlinks_only(self, items: list[tuple[Website, LinkCheckRecord]]) -> None: + with requests.Session() as session: + self.fetcher = self.fetcher or WebFetchClient(session, self.proxy_settings) + for website, record in items: + record.backlink_checked = True + record.has_author_link = self._check_author_link_in_page(session, website.linkpage) + self.store.save_records([record for _, record in items]) + def _build_failed_record(self, website: Website, cached: LinkCheckRecord | None) -> LinkCheckRecord: record = LinkCheckRecord.unchecked(website, self._now_text()) record.fail_count = (cached.fail_count if cached else 0) + 1 @@ -262,27 +338,34 @@ def _build_disabled_record(website: Website, checked_at: str) -> LinkCheckRecord reachable=True, crawl_allowed=True, best_method="disabled", - best_latency=-1, + best_latency=0.01, rss_crawl_reason="link_check_disabled", ) - def _build_proxy_url(self, url: str) -> str: - if not self.proxy_settings.proxy_url: - return "" - if "{}" in self.proxy_settings.proxy_url: - return self.proxy_settings.proxy_url.format(url) - if "{url}" in self.proxy_settings.proxy_url: - return self.proxy_settings.proxy_url.format(url=url) - return f"{self.proxy_settings.proxy_url}{url}" - @staticmethod def _is_url(path: str) -> bool: return urlparse(path).scheme in ("http", "https") + @staticmethod + def _same_linkpage(left: str, right: str) -> bool: + return LinkReachabilityService._normalize_linkpage(left) == LinkReachabilityService._normalize_linkpage(right) + + @staticmethod + def _normalize_linkpage(url: str) -> str: + url = (url or "").strip() + if not url: + return "" + try: + parts = urlsplit(url) + path = parts.path.rstrip("/") or "/" + return urlunsplit((parts.scheme.lower(), parts.netloc.lower(), path, parts.query, parts.fragment)) + except Exception: + return url.rstrip("/") + @staticmethod def _now_text() -> str: return datetime.now().strftime("%Y-%m-%d %H:%M:%S") -# Backward-compatible class name kept for legacy imports. +# 兼容旧类名。 LinkCheckService = LinkReachabilityService diff --git a/friend_circle_lite/outputs/legacy_api.py b/friend_circle_lite/outputs/legacy_api.py index b9e9d669cea..03ec39cc512 100644 --- a/friend_circle_lite/outputs/legacy_api.py +++ b/friend_circle_lite/outputs/legacy_api.py @@ -9,6 +9,7 @@ import requests from friend_circle_lite import HEADERS_JSON, timeout +from friend_circle_lite.domain.models import normalize_latency from friend_circle_lite.crawler.service import ( FriendCircleCrawlService, limit_large_dataset as _limit_large_dataset, @@ -109,11 +110,17 @@ def merge_link_data_from_json_url(link_data, merge_json_url): local_link = link_map[url] link_map[url] = _merge_single_link(local_link, remote_link) - merged_links = list(link_map.values()) + merged_links = [_to_public_link(link) for link in link_map.values()] logging.info(f"合并友链数据完成，共有 {len(merged_links)} 条友链") # 重新计算统计数据 merged_stats = _recalculate_link_statistics(merged_links) + checked_times = [ + link_data.get("statistical_data", {}).get("link_last_checked_time", ""), + remote_data.get("statistical_data", {}).get("link_last_checked_time", ""), + merged_stats.get("link_last_checked_time", ""), + ] + merged_stats["link_last_checked_time"] = max([item for item in checked_times if item] or [""]) return { 'statistical_data': merged_stats, @@ -133,8 +140,8 @@ def _merge_single_link(local, remote): """ method_priority = {'direct': 4, 'proxy': 3, 'api': 2, 'disabled': 1, 'none': 0, '': 0} - local_priority = method_priority.get(local.get('method', ''), 0) - remote_priority = method_priority.get(remote.get('method', ''), 0) + local_priority = _link_priority(local, method_priority) + remote_priority = _link_priority(remote, method_priority) # 选择优先级更高的作为基础 if remote_priority > local_priority: @@ -177,6 +184,31 @@ def _merge_single_link(local, remote): return base +def _link_priority(link, method_priority): + """计算友链合并优先级，兼容新旧 link.json 字段。""" + if link.get("crawlable"): + return 10 + method_priority.get(link.get("method", ""), 0) + if link.get("reachable"): + return 5 + method_priority.get(link.get("method", ""), 0) + return method_priority.get(link.get("method", ""), 0) + + +def _to_public_link(link): + """转换为前端需要的精简友链状态结构。""" + latency = normalize_latency(link.get("latency", link.get("best_latency"))) + return { + "name": link.get("name", ""), + "link": link.get("link") or link.get("url", ""), + "link_page": link.get("link_page") or link.get("linkpage", ""), + "avatar": link.get("avatar", ""), + "reachable": bool(link.get("reachable")), + "crawlable": bool(link.get("crawlable") if "crawlable" in link else link.get("crawl_allowed")), + "latency": latency, + "fail_count": int(link.get("fail_count", 0) or 0), + "has_backlink": link.get("has_backlink"), + } + + def _recalculate_link_statistics(links): """重新计算合并后的友链统计数据。""" reachable = [link for link in links if link.get('reachable')] diff --git a/friend_circle_lite/storage/__init__.py b/friend_circle_lite/storage/__init__.py index 1dccef8b143..75aad775e0b 100644 --- a/friend_circle_lite/storage/__init__.py +++ b/friend_circle_lite/storage/__init__.py @@ -1,3 +1,4 @@ """Persistent stores for feed cache, article tracking, and link checks.""" from friend_circle_lite.storage.sqlite_store import ArticleTrackingStore, FeedCacheStore, LinkCheckStore +from friend_circle_lite.storage.diagnostics import SQLiteDebugDumper diff --git a/friend_circle_lite/storage/diagnostics.py b/friend_circle_lite/storage/diagnostics.py new file mode 100644 index 00000000000..3e5eadb5d3d --- /dev/null +++ b/friend_circle_lite/storage/diagnostics.py @@ -0,0 +1,248 @@ +"""SQLite 缓存诊断与安全整理。 + +本文件只负责调试期开启的 SQLite 检查： +- 输出当前数据库中的全部表结构与全部数据； +- 对项目内部核心表执行保守 schema 整理，移除残留旧字段； +- 不自动删除未知表，避免误删用户额外保存的数据。 +""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +from contextlib import closing +from pathlib import Path + + +EXPECTED_SCHEMAS: dict[str, tuple[list[str], str]] = { + "feed_cache": ( + ["name", "url", "source"], + """ + CREATE TABLE feed_cache ( + name TEXT PRIMARY KEY, + url TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'cache' + ) + """, + ), + "article_tracking": ( + ["id", "title", "author", "link", "published", "summary", "content"], + """ + CREATE TABLE article_tracking ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + author TEXT NOT NULL, + link TEXT NOT NULL, + published TEXT NOT NULL, + summary TEXT, + content TEXT + ) + """, + ), + "link_check_state": ( + [ + "url", + "name", + "avatar", + "linkpage", + "checked_at", + "reachable", + "crawl_allowed", + "best_method", + "best_latency", + "fail_count", + "backlink_checked", + "has_author_link", + "rss_crawl_reason", + "direct_success", + "direct_status_code", + "direct_latency", + "proxy_success", + "proxy_status_code", + "proxy_latency", + "api_success", + "api_status_code", + "api_latency", + ], + """ + CREATE TABLE link_check_state ( + url TEXT PRIMARY KEY, + name TEXT NOT NULL, + avatar TEXT DEFAULT '', + linkpage TEXT DEFAULT '', + checked_at TEXT NOT NULL, + reachable INTEGER NOT NULL DEFAULT 0, + crawl_allowed INTEGER NOT NULL DEFAULT 0, + best_method TEXT NOT NULL DEFAULT 'none', + best_latency REAL DEFAULT -1, + fail_count INTEGER NOT NULL DEFAULT 0, + backlink_checked INTEGER NOT NULL DEFAULT 0, + has_author_link INTEGER NOT NULL DEFAULT 0, + rss_crawl_reason TEXT NOT NULL DEFAULT '', + direct_success INTEGER NOT NULL DEFAULT 0, + direct_status_code INTEGER, + direct_latency REAL DEFAULT -1, + proxy_success INTEGER NOT NULL DEFAULT 0, + proxy_status_code INTEGER, + proxy_latency REAL DEFAULT -1, + api_success INTEGER NOT NULL DEFAULT 0, + api_status_code INTEGER, + api_latency REAL DEFAULT -1 + ) + """, + ), +} + +DEFAULT_EXPRESSIONS: dict[str, str] = { + "id": "NULL", + "name": "''", + "url": "''", + "source": "'cache'", + "title": "''", + "author": "''", + "link": "''", + "published": "''", + "summary": "NULL", + "content": "NULL", + "avatar": "''", + "linkpage": "''", + "checked_at": "''", + "reachable": "0", + "crawl_allowed": "0", + "best_method": "'none'", + "best_latency": "-1", + "fail_count": "0", + "backlink_checked": "0", + "has_author_link": "0", + "rss_crawl_reason": "''", + "direct_success": "0", + "direct_status_code": "NULL", + "direct_latency": "-1", + "proxy_success": "0", + "proxy_status_code": "NULL", + "proxy_latency": "-1", + "api_success": "0", + "api_status_code": "NULL", + "api_latency": "-1", +} + + +class SQLiteDebugDumper: + """打印 SQLite 全量调试信息，并清理核心表中的旧字段。""" + + def __init__(self, database_path: str | Path | None): + self.database_path = Path(database_path) if database_path else None + + def run(self) -> str: + """执行 schema 检查、保守清理与全量数据输出。""" + lines: list[str] = [] + self._append(lines, "=" * 60) + self._append(lines, "SQLite 调试信息") + self._append(lines, "=" * 60) + + if not self.database_path: + self._append(lines, "未配置 SQLite 缓存路径，跳过调试输出") + return self._flush(lines) + + self._append(lines, f"数据库路径: {self.database_path}") + if not self.database_path.exists(): + self._append(lines, "数据库文件不存在，跳过调试输出") + return self._flush(lines) + + with closing(sqlite3.connect(self.database_path)) as connection: + connection.row_factory = sqlite3.Row + self._report_schema_state(connection, lines) + self._clean_known_tables(connection, lines) + self._dump_all_tables(connection, lines) + + return self._flush(lines) + + def _report_schema_state(self, connection: sqlite3.Connection, lines: list[str]) -> None: + tables = self._table_names(connection) + self._append(lines, f"当前表数量: {len(tables)}") + for table in tables: + columns = self._column_names(connection, table) + self._append(lines, f"表 {table} 字段: {', '.join(columns)}") + expected = EXPECTED_SCHEMAS.get(table) + if not expected: + self._append(lines, f"表 {table} 不是 Friend-Circle-Lite 核心表，保留不清理") + continue + extra_columns = [column for column in columns if column not in expected[0]] + missing_columns = [column for column in expected[0] if column not in columns] + if extra_columns: + self._append(lines, f"表 {table} 检测到旧字段: {', '.join(extra_columns)}") + if missing_columns: + self._append(lines, f"表 {table} 缺少当前字段，将使用默认值补齐: {', '.join(missing_columns)}") + + def _clean_known_tables(self, connection: sqlite3.Connection, lines: list[str]) -> None: + for table, (expected_columns, create_sql) in EXPECTED_SCHEMAS.items(): + if table not in self._table_names(connection): + connection.execute(create_sql) + self._append(lines, f"表 {table} 不存在，已按当前 schema 创建") + continue + + current_columns = self._column_names(connection, table) + if current_columns == expected_columns: + self._append(lines, f"表 {table} schema 已匹配，无需清理") + continue + + temp_table = f"__fcl_rebuild_{table}" + connection.execute(f"DROP TABLE IF EXISTS {temp_table}") + connection.execute(create_sql.replace(f"CREATE TABLE {table}", f"CREATE TABLE {temp_table}", 1)) + + common_columns = [column for column in expected_columns if column in current_columns] + insert_columns = ", ".join(self._quote_identifier(column) for column in expected_columns) + select_expressions = ", ".join( + self._quote_identifier(column) if column in current_columns else DEFAULT_EXPRESSIONS[column] + for column in expected_columns + ) + connection.execute( + f"INSERT INTO {self._quote_identifier(temp_table)} ({insert_columns}) " + f"SELECT {select_expressions} FROM {self._quote_identifier(table)}" + ) + + connection.execute(f"DROP TABLE {self._quote_identifier(table)}") + connection.execute( + f"ALTER TABLE {self._quote_identifier(temp_table)} RENAME TO {self._quote_identifier(table)}" + ) + self._append(lines, f"表 {table} 已重建为当前 schema，保留字段: {', '.join(common_columns)}") + connection.commit() + + def _dump_all_tables(self, connection: sqlite3.Connection, lines: list[str]) -> None: + tables = self._table_names(connection) + self._append(lines, "SQLite 全量数据开始") + for table in tables: + rows = connection.execute(f"SELECT * FROM {self._quote_identifier(table)}").fetchall() + self._append(lines, f"表 {table} 行数: {len(rows)}") + for index, row in enumerate(rows, start=1): + row_data = {key: row[key] for key in row.keys()} + self._append(lines, f"表 {table} 第 {index} 行: {json.dumps(row_data, ensure_ascii=False)}") + self._append(lines, "SQLite 全量数据结束") + + @staticmethod + def _table_names(connection: sqlite3.Connection) -> list[str]: + rows = connection.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name" + ).fetchall() + return [row[0] for row in rows] + + @staticmethod + def _column_names(connection: sqlite3.Connection, table: str) -> list[str]: + rows = connection.execute(f"PRAGMA table_info({SQLiteDebugDumper._quote_identifier(table)})").fetchall() + return [row[1] for row in rows] + + @staticmethod + def _quote_identifier(identifier: str) -> str: + return '"' + identifier.replace('"', '""') + '"' + + @staticmethod + def _append(lines: list[str], message: str) -> None: + lines.append(message) + + @staticmethod + def _flush(lines: list[str]) -> str: + output = "\n".join(lines) + for line in lines: + logging.info(line) + return output diff --git a/friend_circle_lite/storage/sqlite_store.py b/friend_circle_lite/storage/sqlite_store.py index 960b89eb8f9..e60f6057c46 100644 --- a/friend_circle_lite/storage/sqlite_store.py +++ b/friend_circle_lite/storage/sqlite_store.py @@ -17,12 +17,13 @@ import json import logging import sqlite3 +from contextlib import closing from datetime import datetime from pathlib import Path import yaml -from friend_circle_lite.domain.models import Article, CacheRecord, LinkCheckRecord, LinkMethodStatus +from friend_circle_lite.domain.models import Article, CacheRecord, LinkCheckRecord, LinkMethodStatus, normalize_homepage_url class FeedCacheStore: @@ -55,7 +56,7 @@ def save_records(self, records: list[CacheRecord]) -> bool: try: self.cache_path.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(self.cache_path) as connection: + with closing(sqlite3.connect(self.cache_path)) as connection: self._ensure_schema(connection) connection.execute("DELETE FROM feed_cache") connection.executemany( @@ -72,8 +73,9 @@ def save_records(self, records: list[CacheRecord]) -> bool: def _load_from_sqlite(self) -> list[CacheRecord]: """Load records from the current SQLite cache file.""" try: - with sqlite3.connect(self.cache_path) as connection: + with closing(sqlite3.connect(self.cache_path)) as connection: self._ensure_schema(connection) + connection.commit() rows = connection.execute( "SELECT name, url, source FROM feed_cache ORDER BY name" ).fetchall() @@ -207,7 +209,7 @@ def save_articles(self, articles: list[Article]) -> bool: articles_to_save = valid_articles[:self.max_tracked_articles] self.storage_path.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(self.storage_path) as connection: + with closing(sqlite3.connect(self.storage_path)) as connection: self._ensure_schema(connection) connection.execute("DELETE FROM article_tracking") connection.executemany( @@ -234,8 +236,9 @@ def save_articles(self, articles: list[Article]) -> bool: def _load_from_sqlite(self) -> list[Article]: """Load articles from the SQLite database.""" try: - with sqlite3.connect(self.storage_path) as connection: + with closing(sqlite3.connect(self.storage_path)) as connection: self._ensure_schema(connection) + connection.commit() rows = connection.execute( """SELECT title, author, link, published, summary, content FROM article_tracking @@ -319,8 +322,9 @@ def load_records(self, urls: list[str] | None = None) -> dict[str, LinkCheckReco return {} try: - with sqlite3.connect(self.cache_path) as connection: + with closing(sqlite3.connect(self.cache_path)) as connection: self._ensure_schema(connection) + connection.commit() rows = connection.execute( """ SELECT url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, @@ -335,7 +339,7 @@ def load_records(self, urls: list[str] | None = None) -> dict[str, LinkCheckReco logging.warning(f"读取友链检测缓存失败: {exc}") return {} - allowed_urls = set(urls or []) + allowed_urls = {normalize_homepage_url(url) for url in (urls or [])} records: dict[str, LinkCheckRecord] = {} for row in rows: ( @@ -345,11 +349,12 @@ def load_records(self, urls: list[str] | None = None) -> dict[str, LinkCheckReco proxy_success, proxy_status_code, proxy_latency, api_success, api_status_code, api_latency, ) = row - if allowed_urls and url not in allowed_urls: + normalized_url = normalize_homepage_url(url or "") + if allowed_urls and normalized_url not in allowed_urls: continue - records[url] = LinkCheckRecord( + records[normalized_url] = LinkCheckRecord( name=name or "", - url=url or "", + url=normalized_url, avatar=avatar or "", linkpage=linkpage or "", checked_at=checked_at or "", @@ -373,7 +378,7 @@ def save_records(self, records: list[LinkCheckRecord]) -> bool: try: self.cache_path.parent.mkdir(parents=True, exist_ok=True) - with sqlite3.connect(self.cache_path) as connection: + with closing(sqlite3.connect(self.cache_path)) as connection: self._ensure_schema(connection) connection.executemany( """ @@ -430,7 +435,7 @@ def is_fresh(record: LinkCheckRecord, max_age_hours: int) -> bool: @staticmethod def _record_to_row(record: LinkCheckRecord) -> tuple: return ( - record.url, + normalize_homepage_url(record.url), record.name, record.avatar, record.linkpage, diff --git a/static/index.html b/static/index.html index b3c4440b3d7..0561f237735 100644 --- a/static/index.html +++ b/static/index.html @@ -345,7 +345,6 @@ flex-shrink: 0; } - .method-text, .time-text { color: #000000a5; white-space: nowrap; @@ -651,7 +650,7 @@

友圈文章

const sortedLinks = state.links .filter(matchKeywordForLink) .filter(matchFilter) - .sort((a, b) => linkPriority(a) - linkPriority(b) || (b.latency ?? -1) - (a.latency ?? -1)); + .sort((a, b) => linkPriority(a) - linkPriority(b)); let errorLinksHTML = ""; let otherLinksHTML = ""; @@ -709,11 +708,15 @@

友圈文章

} function getLinkStatus(link) { - if (!link.reachable) return { className: "status-error", title: "不可达" }; - if (link.method === "api") return { className: "status-api", title: "仅 API 可达" }; - if (link.latency > 4) return { className: "status-slow", title: "响应较慢" }; - if (link.method === "proxy") return { className: "status-api", title: "代理可达" }; - return { className: "status-normal", title: "直连可达" }; + const latencyText = formatLatency(link.latency); + if (!link.reachable) return { className: "status-error", title: `不可达${latencyText}` }; + return { className: "status-normal", title: `可达${latencyText}` }; + } + + function formatLatency(value) { + const latency = Number(value); + if (!Number.isFinite(latency) || latency < 0) return ""; + return `，参考延迟 ${latency.toFixed(2)} 秒`; } function matchFilter(link) { @@ -724,7 +727,7 @@

友圈文章

function matchKeywordForLink(link) { if (!state.keyword) return true; - return [link.name, link.link, link.link_page, link.reason] + return [link.name, link.link, link.link_page] .some((value) => String(value || "").toLowerCase().includes(state.keyword)); } @@ -736,13 +739,8 @@

友圈文章

function linkPriority(link) { if (!link.reachable) return 0; - if (link.method === "api") return 1; - if (link.latency > 4) return 2; - return 3; - } - - function formatLatency(latency) { - return latency >= 0 ? `${latency}s` : "--"; + if (!link.crawlable) return 1; + return 2; } function escapeHTML(value) { diff --git a/tests/test_refactor_contracts.py b/tests/test_refactor_contracts.py index 500a7e6759f..0b161cdbff8 100644 --- a/tests/test_refactor_contracts.py +++ b/tests/test_refactor_contracts.py @@ -1,14 +1,433 @@ import unittest +import sqlite3 +import tempfile +from contextlib import closing +from datetime import datetime +from pathlib import Path from unittest.mock import patch +import requests + +from friend_circle_lite.config.models import ProxySettings +from friend_circle_lite.config.printer import print_startup_config +from friend_circle_lite.crawler.http_client import WebFetchClient +from friend_circle_lite.crawler.service import FeedResolver, FriendCircleCrawlService, SingleSiteCrawler from friend_circle_lite.all_friends import deal_with_large_data, merge_link_data_from_json_url from friend_circle_lite.app_config import ApplicationConfig -from friend_circle_lite.models import LinkCheckRecord, LinkMethodStatus, Website +from friend_circle_lite.link_checker.service import LinkReachabilityService +from friend_circle_lite.models import Article, CacheRecord, FeedEndpoint, LinkCheckRecord, LinkMethodStatus, Website +from friend_circle_lite.outputs.legacy_api import _to_public_link +from friend_circle_lite.storage.diagnostics import SQLiteDebugDumper class RefactorContractsTest(unittest.TestCase): + def test_link_check_uses_cached_rss_without_homepage_request(self): + class Store: + def load_records(self, urls): + return {} + + def save_records(self, records): + return True + + class Parser: + def parse(self, feed_url, count=1, blog_url=""): + return [Article(title="Post", author="Site", link="https://site.example/post", published="2026-06-07 10:00")] + + class Fetcher: + calls = [] + + def get(self, *args, **kwargs): + self.calls.append(args[0]) + raise AssertionError("主页不应该在 RSS 可解析时被请求") + + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"enable": True}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + feed_records=[CacheRecord(name="Site", url="https://site.example/rss.xml", source="cache")], + feed_parser=Parser(), + fetcher=Fetcher(), + ) + + records = service.check_websites([Website(name="Site", url="https://site.example", avatar="avatar.png")]) + + self.assertTrue(records[0].reachable) + self.assertTrue(records[0].crawl_allowed) + self.assertEqual(records[0].best_method, "rss_cache") + + def test_link_check_reuses_cached_linkpage_when_only_trailing_slash_differs(self): + class Store: + def load_records(self, urls): + return { + "https://wcowin.work/": LinkCheckRecord( + name="Wcowin", + url="https://wcowin.work/", + linkpage="https://wcowin.work/link/", + checked_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + reachable=True, + crawl_allowed=False, + best_method="homepage", + best_latency=0.2, + ) + } + + def save_records(self, records): + raise AssertionError("只差末尾斜杠时不应该重新检测并写入缓存") + + def is_fresh(self, record, max_age_hours): + return True + + class Fetcher: + def get(self, *args, **kwargs): + raise AssertionError("只差末尾斜杠时不应该重新请求网站") + + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({ + "link_check": { + "enable_backlink_check": True, + "author_url": "blog.liushen.fun", + } + }).link_check, + proxy_settings=ProxySettings(), + store=Store(), + fetcher=Fetcher(), + ) + + records = service.check_websites([ + Website( + name="Wcowin", + url="https://wcowin.work/", + avatar="avatar.png", + linkpage="https://wcowin.work/link", + ) + ]) + + self.assertEqual(len(records), 1) + self.assertTrue(records[0].reachable) + self.assertFalse(records[0].crawl_allowed) + + def test_linkpage_change_refreshes_backlink_only(self): + saved_records = [] + + class Store: + def load_records(self, urls): + return { + "https://site.example/": LinkCheckRecord( + name="Site", + url="https://site.example/", + linkpage="https://site.example/old-links/", + checked_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + reachable=True, + crawl_allowed=False, + best_method="homepage", + best_latency=0.2, + backlink_checked=True, + has_author_link=False, + ) + } + + def save_records(self, records): + saved_records.extend(records) + return True + + def is_fresh(self, record, max_age_hours): + return True + + class Response: + status_code = 200 + text = '清羽飞扬' + + class Fetcher: + calls = [] + + def get(self, url, *args, **kwargs): + self.calls.append(url) + if url != "https://site.example/new-links/": + raise AssertionError("反链页变化时只应请求新的友链页") + return type("Result", (), {"response": Response(), "latency": 0.2, "success": True})() + + fetcher = Fetcher() + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({ + "link_check": { + "enable_backlink_check": True, + "author_url": "blog.liushen.fun", + } + }).link_check, + proxy_settings=ProxySettings(), + store=Store(), + fetcher=fetcher, + ) + + records = service.check_websites([ + Website( + name="Site", + url="https://site.example/", + avatar="avatar.png", + linkpage="https://site.example/new-links/", + ) + ]) + + self.assertEqual(fetcher.calls, ["https://site.example/new-links/"]) + self.assertEqual(len(saved_records), 1) + self.assertTrue(records[0].has_author_link) + self.assertEqual(records[0].linkpage, "https://site.example/new-links/") + + def test_link_check_revalidates_legacy_crawlable_cache_without_rss_method(self): + class Store: + def load_records(self, urls): + return { + "https://site.example": LinkCheckRecord( + name="Site", + url="https://site.example", + checked_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + reachable=True, + crawl_allowed=True, + best_method="direct", + ) + } + + def save_records(self, records): + return True + + def is_fresh(self, record, max_age_hours): + return True + + class Parser: + def parse(self, feed_url, count=1, blog_url=""): + return [Article(title="Post", author="Site", link="https://site.example/post", published="2026-06-07 10:00")] + + class Discovery: + calls = 0 + + def discover(self, website_url): + self.calls += 1 + return FeedEndpoint(url="https://site.example/rss.xml", feed_type="specific", source="auto") + + discovery = Discovery() + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"max_age_hours": 24}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + feed_records=[], + feed_parser=Parser(), + feed_discovery=discovery, + ) + + records = service.check_websites([Website(name="Site", url="https://site.example", avatar="avatar.png")]) + + self.assertEqual(discovery.calls, 1) + self.assertTrue(records[0].crawl_allowed) + self.assertEqual(records[0].best_method, "rss") + + def test_link_check_revalidates_fresh_cache_without_measured_latency(self): + saved_records = [] + + class Store: + def load_records(self, urls): + return { + "https://site.example/": LinkCheckRecord( + name="Site", + url="https://site.example/", + checked_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + reachable=True, + crawl_allowed=True, + best_method="rss_cache", + best_latency=-1, + ) + } + + def save_records(self, records): + saved_records.extend(records) + return True + + def is_fresh(self, record, max_age_hours): + return True + + class Parser: + last_latency = 0.23 + + def parse(self, feed_url, count=1, blog_url=""): + return [Article(title="Post", author="Site", link="https://site.example/post", published="2026-06-07 10:00")] + + class Discovery: + calls = 0 + + def discover(self, website_url): + self.calls += 1 + return FeedEndpoint(url="https://site.example/rss.xml", feed_type="specific", source="auto") + + discovery = Discovery() + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"max_age_hours": 24}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + feed_records=[], + feed_parser=Parser(), + feed_discovery=discovery, + ) + + records = service.check_websites([Website(name="Site", url="https://site.example", avatar="avatar.png")]) + + self.assertEqual(discovery.calls, 1) + self.assertEqual(len(saved_records), 1) + self.assertEqual(records[0].best_latency, 0.23) + self.assertGreater(records[0].to_link_dict()["latency"], 0) + + def test_link_check_revalidates_fresh_homepage_cache_without_measured_latency(self): + saved_records = [] + + class Store: + def load_records(self, urls): + return { + "https://site.example/": LinkCheckRecord( + name="Site", + url="https://site.example/", + checked_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + reachable=True, + crawl_allowed=False, + best_method="homepage", + best_latency=-1, + ) + } + + def save_records(self, records): + saved_records.extend(records) + return True + + def is_fresh(self, record, max_age_hours): + return True + + class Discovery: + def discover(self, website_url): + return None + + class Response: + status_code = 200 + + class Fetcher: + calls = 0 + + def get(self, *args, **kwargs): + self.calls += 1 + return type("Result", (), {"response": Response(), "latency": 0.31, "success": True})() + + fetcher = Fetcher() + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"max_age_hours": 24}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + feed_parser=type("Parser", (), {"parse": lambda self, *args, **kwargs: [], "last_latency": 0.01})(), + feed_discovery=Discovery(), + fetcher=fetcher, + ) + + records = service.check_websites([Website(name="Site", url="https://site.example", avatar="avatar.png")]) + + self.assertEqual(fetcher.calls, 1) + self.assertEqual(len(saved_records), 1) + self.assertEqual(records[0].best_latency, 0.31) + + def test_article_crawl_does_not_rediscover_invalid_cached_rss(self): + class Parser: + def parse(self, feed_url, count=1, blog_url=""): + return [] + + class Discovery: + calls = 0 + + def discover(self, website_url): + self.calls += 1 + return FeedEndpoint(url="https://site.example/new-rss.xml", feed_type="specific", source="auto") + + discovery = Discovery() + resolver = FeedResolver( + discovery_service=discovery, + configured_feeds=[CacheRecord(name="Site", url="https://site.example/rss.xml", source="cache")], + ) + crawler = SingleSiteCrawler(parser_service=Parser(), resolver=resolver) + + result = crawler.crawl(Website(name="Site", url="https://site.example", avatar="avatar.png"), count=1) + + self.assertEqual(discovery.calls, 0) + self.assertEqual(result.status, "error") + self.assertEqual(result.cache_update.action, "none") + + def test_web_fetch_client_falls_back_to_proxy_url(self): + calls = [] + + class Response: + status_code = 200 + text = "ok" + + class Session: + def get(self, url, headers=None, timeout=None): + calls.append(url) + if len(calls) == 1: + raise requests.RequestException("direct failed") + return Response() + + client = WebFetchClient(Session(), ProxySettings(proxy_url="https://proxy.example/{url}")) + + result = client.get("https://site.example/feed.xml", desc="RSS") + + self.assertTrue(result.success) + self.assertTrue(result.used_proxy) + self.assertEqual(calls, [ + "https://site.example/feed.xml", + "https://proxy.example/https://site.example/feed.xml", + ]) + + def test_web_fetch_client_records_positive_latency_for_request_exception(self): + class Session: + def get(self, url, headers=None, timeout=None): + raise requests.RequestException("direct failed") + + client = WebFetchClient(Session(), ProxySettings()) + + with patch("time.time", side_effect=[10.0, 10.0]): + result = client.get("https://site.example/feed.xml", desc="RSS") + + self.assertFalse(result.success) + self.assertGreater(result.latency, 0) + self.assertNotEqual(result.latency, 0.0) + + def test_web_fetch_client_does_not_log_proxy_service_url(self): + class Session: + def __init__(self): + self.calls = 0 + + def get(self, url, headers=None, timeout=None): + self.calls += 1 + if self.calls == 1: + raise requests.RequestException("direct failed") + raise requests.RequestException("HTTPSConnectionPool(host='proxy.example', port=443)") + + client = WebFetchClient(Session(), ProxySettings(proxy_url="https://proxy.example/")) + + with patch("logging.warning") as warning: + client.get("https://site.example/feed.xml", desc="RSS") + + messages = "\n".join(str(call.args[0]) for call in warning.call_args_list) + self.assertNotIn("https://proxy.example", messages) + self.assertNotIn("proxy.example", messages) + self.assertIn("https://site.example/feed.xml", messages) + + def test_startup_config_does_not_log_proxy_service_url(self): + config = ApplicationConfig.from_dict({ + "proxy_settings": {"proxy_url": "https://proxy.example/"}, + }) + + with patch("logging.info") as info: + print_startup_config(config) + + messages = "\n".join(str(call.args[0]) for call in info.call_args_list) + self.assertNotIn("https://proxy.example", messages) + self.assertIn("代理", messages) + def test_config_keeps_existing_yaml_keys(self): config = ApplicationConfig.from_dict({ + "debug": True, "spider_settings": { "enable": True, "json_url": "https://example.com/friends.json", @@ -40,18 +459,30 @@ def test_config_keeps_existing_yaml_keys(self): }) self.assertEqual(config.spider_settings.json_url, "https://example.com/friends.json") + self.assertTrue(config.debug) self.assertEqual(config.spider_settings.article_count, 3) self.assertEqual(config.proxy_settings.proxy_url, "https://proxy.example/") self.assertTrue(config.merge_settings.enable) self.assertFalse(config.merge_settings.merge_article_data) - self.assertFalse(config.link_check.enable) + self.assertTrue(config.link_check.enable) self.assertEqual(config.link_check.author_url, "example.com") self.assertEqual(config.runtime_paths.cache_file, "./tmp/state.sqlite3") self.assertEqual(config.specific_rss[0]["name"], "Manual") + def test_debug_env_enables_sqlite_dump(self): + with patch.dict("os.environ", {"FCL_DEBUG": "1"}): + config = ApplicationConfig.from_dict({}) + + self.assertTrue(config.debug) + + def test_debug_string_false_stays_disabled(self): + config = ApplicationConfig.from_dict({"debug": "false"}) + + self.assertFalse(config.debug) + def test_website_and_link_record_public_shapes_are_stable(self): website = Website.from_friend_item(["Alice", "https://alice.example", "https://alice.example/links", "avatar.png"]) - self.assertEqual(website.to_error_payload(), ["Alice", "https://alice.example", "avatar.png"]) + self.assertEqual(website.to_error_payload(), ["Alice", "https://alice.example/", "avatar.png"]) record = LinkCheckRecord( name=website.name, @@ -73,19 +504,58 @@ def test_website_and_link_record_public_shapes_are_stable(self): self.assertEqual(record.to_link_dict(), { "name": "Alice", - "link": "https://alice.example", + "link": "https://alice.example/", "link_page": "https://alice.example/links", "avatar": "avatar.png", "reachable": True, "crawlable": True, - "method": "proxy", "latency": 1.2, "fail_count": 0, - "checked_at": "2026-06-06 12:00:00", "has_backlink": True, - "reason": "allowed_by_proxy", }) + def test_public_link_never_uses_zero_latency_as_unknown_fallback(self): + public_link = _to_public_link({ + "name": "Legacy", + "url": "https://legacy.example/", + "reachable": False, + "crawl_allowed": False, + "best_latency": -1, + }) + + self.assertGreater(public_link["latency"], 0) + self.assertNotEqual(public_link["latency"], 0.0) + + def test_homepage_url_normalization_adds_trailing_slash_to_paths(self): + website = Website.from_friend_item(["PathSite", "https://example.com/blog", "avatar.png"]) + + self.assertEqual(website.url, "https://example.com/blog/") + + def test_load_websites_deduplicates_by_normalized_homepage_url(self): + service = FriendCircleCrawlService(json_url="https://example.com/friends.json", count=1) + + class Response: + def raise_for_status(self): + return None + + def json(self): + return { + "friends": [ + ["Wcowin", "https://wcowin.work", "https://wcowin.work/link", "old.png"], + ["Wcowin", "https://wcowin.work/", "https://wcowin.work/link/", "new.png"], + ] + } + + class Session: + def get(self, *args, **kwargs): + return Response() + + websites = service._load_websites(Session()) + + self.assertEqual(len(websites), 1) + self.assertEqual(websites[0].url, "https://wcowin.work/") + self.assertEqual(websites[0].avatar, "new.png") + def test_large_data_sorting_keeps_public_article_schema(self): payload = { "statistical_data": {"article_num": 0}, @@ -144,12 +614,177 @@ def json(self): with patch("requests.get", return_value=Response()): merged = merge_link_data_from_json_url(local, "https://remote.example/link.json") - self.assertEqual(merged["link_data"][0]["method"], "proxy") + self.assertNotIn("method", merged["link_data"][0]) + self.assertNotIn("checked_at", merged["link_data"][0]) + self.assertNotIn("reason", merged["link_data"][0]) self.assertEqual(merged["link_data"][0]["latency"], 1.0) self.assertEqual(merged["link_data"][0]["fail_count"], 0) self.assertTrue(merged["link_data"][0]["has_backlink"]) self.assertEqual(merged["statistical_data"]["link_total_num"], 1) + def test_all_json_statistics_do_not_include_link_statistics(self): + with tempfile.TemporaryDirectory() as temp_dir: + service = FriendCircleCrawlService( + json_url="https://example.com/friends.json", + count=1, + cache_file=str(Path(temp_dir) / "cache.sqlite3"), + ) + website = Website(name="Site", url="https://site.example", avatar="avatar.png") + + def load_websites(_session): + return [website] + + def check_links(_websites, _feed_records, _manual_names): + return [ + LinkCheckRecord( + name="Site", + url="https://site.example", + avatar="avatar.png", + checked_at="2026-06-07 12:00:00", + reachable=False, + crawl_allowed=False, + ) + ] + + service._load_websites = load_websites + service._check_links = check_links + + all_payload, _errors, link_payload = service.run() + + self.assertEqual(set(all_payload["statistical_data"].keys()), { + "friends_num", + "active_num", + "error_num", + "article_num", + "last_updated_time", + }) + self.assertIn("link_total_num", link_payload["statistical_data"]) + + def test_crawl_filter_uses_crawl_allowed_and_feed_cache_not_best_method(self): + with tempfile.TemporaryDirectory() as temp_dir: + service = FriendCircleCrawlService( + json_url="https://example.com/friends.json", + count=1, + specific_rss=[{"name": "WithRSS", "url": "https://with.example/rss.xml"}], + cache_file=str(Path(temp_dir) / "cache.sqlite3"), + ) + websites = [ + Website(name="WithRSS", url="https://with.example", avatar="with.png"), + Website(name="NoRSS", url="https://no.example", avatar="no.png"), + ] + + def load_websites(_session): + return websites + + def check_links(_websites, _feed_records, _manual_names): + return [ + LinkCheckRecord( + name="WithRSS", + url="https://with.example", + avatar="with.png", + checked_at="2026-06-07 12:00:00", + reachable=True, + crawl_allowed=True, + best_method="homepage", + ), + LinkCheckRecord( + name="NoRSS", + url="https://no.example", + avatar="no.png", + checked_at="2026-06-07 12:00:00", + reachable=True, + crawl_allowed=True, + best_method="rss", + ), + ] + + crawled_names = [] + + def crawl(_crawler, website, count): + crawled_names.append(website.name) + return type("Result", (), { + "website": website, + "status": "active", + "articles": [ + Article( + title=f"{website.name} Post", + author=website.name, + link=f"{website.url}/post", + published="2026-06-07 10:00", + avatar=website.avatar, + ) + ], + "feed_url": "https://with.example/rss.xml", + "feed_type": "specific", + "source_used": "manual", + "cache_update": type("Update", (), {"name": None, "action": "none", "url": None})(), + })() + + service._load_websites = load_websites + service._check_links = check_links + + with patch("friend_circle_lite.crawler.service.SingleSiteCrawler.crawl", crawl): + service.run() + + self.assertEqual(crawled_names, ["WithRSS"]) + + def test_sqlite_debug_dumper_prints_all_rows_and_cleans_extra_columns(self): + with tempfile.TemporaryDirectory() as temp_dir: + db_path = Path(temp_dir) / "cache.sqlite3" + with closing(sqlite3.connect(db_path)) as connection: + connection.execute( + """ + CREATE TABLE feed_cache ( + name TEXT PRIMARY KEY, + url TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'cache', + old_column TEXT + ) + """ + ) + connection.execute( + "INSERT INTO feed_cache(name, url, source, old_column) VALUES (?, ?, ?, ?)", + ("Site", "https://site.example/rss.xml", "cache", "legacy"), + ) + connection.commit() + + output = SQLiteDebugDumper(db_path).run() + + self.assertIn("feed_cache", output) + self.assertIn("https://site.example/rss.xml", output) + self.assertIn("old_column", output) + with closing(sqlite3.connect(db_path)) as connection: + columns = [row[1] for row in connection.execute("PRAGMA table_info(feed_cache)").fetchall()] + self.assertEqual(columns, ["name", "url", "source"]) + + def test_sqlite_debug_dumper_rebuilds_tables_with_missing_columns(self): + with tempfile.TemporaryDirectory() as temp_dir: + db_path = Path(temp_dir) / "cache.sqlite3" + with closing(sqlite3.connect(db_path)) as connection: + connection.execute( + """ + CREATE TABLE link_check_state ( + url TEXT PRIMARY KEY, + name TEXT NOT NULL, + reachable INTEGER NOT NULL DEFAULT 0 + ) + """ + ) + connection.execute( + "INSERT INTO link_check_state(url, name, reachable) VALUES (?, ?, ?)", + ("https://site.example", "Site", 1), + ) + connection.commit() + + output = SQLiteDebugDumper(db_path).run() + + self.assertIn("缺少当前字段", output) + with closing(sqlite3.connect(db_path)) as connection: + row = connection.execute( + "SELECT url, name, checked_at, crawl_allowed, best_method FROM link_check_state" + ).fetchone() + self.assertEqual(row, ("https://site.example", "Site", "", 0, "none")) + if __name__ == "__main__": unittest.main() From 5e9dd6b80c26df8739dce1d64b8a89484160122a Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Sun, 7 Jun 2026 16:18:44 +0800 Subject: [PATCH 20/30] =?UTF-8?q?=F0=9F=A4=AA=E5=AE=8C=E5=96=84=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E6=B5=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/deal_subscribe_issue.yml | 3 + .github/workflows/friend_circle_lite.yml | 3 +- friend_circle_lite/cli.py | 14 ++-- friend_circle_lite/crawler/feed_service.py | 22 ++--- friend_circle_lite/crawler/service.py | 43 +++++----- friend_circle_lite/link_checker/service.py | 17 +++- friend_circle_lite/storage/sqlite_store.py | 24 +++--- tests/test_refactor_contracts.py | 96 ++++++++++++++++++++++ 8 files changed, 168 insertions(+), 54 deletions(-) diff --git a/.github/workflows/deal_subscribe_issue.yml b/.github/workflows/deal_subscribe_issue.yml index a8b981e2f77..ba2efad25e2 100644 --- a/.github/workflows/deal_subscribe_issue.yml +++ b/.github/workflows/deal_subscribe_issue.yml @@ -4,6 +4,9 @@ on: issues: types: [opened] +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: handle_email_issues: if: startsWith(github.event.issue.title, '[邮箱订阅]') diff --git a/.github/workflows/friend_circle_lite.yml b/.github/workflows/friend_circle_lite.yml index 8d42e7e4024..53e57af504c 100644 --- a/.github/workflows/friend_circle_lite.yml +++ b/.github/workflows/friend_circle_lite.yml @@ -2,12 +2,13 @@ name: Friend Circle Lite on: schedule: - - cron: "0 */4 * * *" + - cron: "22 */4 * * *" workflow_dispatch: env: TZ: Asia/Shanghai PAGE_BRANCH: page + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true jobs: friend-circle-lite: diff --git a/friend_circle_lite/cli.py b/friend_circle_lite/cli.py index f572ab7c261..af47eb9ea5d 100644 --- a/friend_circle_lite/cli.py +++ b/friend_circle_lite/cli.py @@ -53,12 +53,12 @@ def run_crawler_if_enabled(self) -> None: """Run the article crawl and persist public output files when enabled.""" spider_settings = self.config.spider_settings if not spider_settings.enable: - logging.info("⏭️ 爬虫未启用，跳过抓取流程") + logging.info("[爬虫入口] 爬虫未启用，跳过抓取流程") return - logging.info("✅ 爬虫已启用") + logging.info("[爬虫入口] 爬虫已启用") logging.info( - f"📥 正在从 {spider_settings.json_url} 获取数据，每个博客获取 {spider_settings.article_count} 篇文章" + f"[爬虫入口] 正在从 {spider_settings.json_url} 获取友链原始数据，每站最多 {spider_settings.article_count} 篇文章" ) crawl_result = fetch_and_process_data( @@ -70,14 +70,14 @@ def run_crawler_if_enabled(self) -> None: proxy_settings=self.config.proxy_settings, ) if crawl_result is None: - logging.error("❌ 抓取流程失败，未生成任何输出文件") + logging.error("[爬虫入口] 抓取流程失败，未生成任何输出文件") return result, lost_friends, link_payload = crawl_result result, lost_friends, link_payload = self._merge_remote_results_if_enabled(result, lost_friends, link_payload) article_count = len(result.get("article_data", [])) - logging.info(f"📦 数据获取完毕，共有 {article_count} 篇文章，正在处理数据") + logging.info(f"[爬虫入口] 数据获取完毕，共有 {article_count} 篇文章，正在处理输出文件") result = deal_with_large_data( result, @@ -168,7 +168,7 @@ def _merge_remote_results_if_enabled( return result, lost_friends, link_payload remote_url = merge_settings.remote_base_url - logging.info(f"🔀 合并功能开启，从 {remote_url} 获取外部数据") + logging.info(f"[数据合并] 合并功能开启，从 {remote_url} 获取外部数据") if merge_settings.merge_article_data: result = merge_data_from_json_url(result, f"{remote_url}/all.json") @@ -193,7 +193,7 @@ def _load_subscriber_emails(github_username: str, github_repo: str) -> dict | No f"https://api.github.com/repos/{github_username}/{github_repo}/issues" f"?state=closed&label=subscribed&per_page=200" ) - logging.info(f"🔎 正在从 GitHub 获取订阅邮箱：{github_api_url}") + logging.info(f"[订阅邮箱] 正在从 GitHub 获取订阅邮箱：{github_api_url}") return extract_emails_from_issues(github_api_url) def _build_email_template_data(self, article: dict, github_username: str, github_repo: str) -> dict[str, str]: diff --git a/friend_circle_lite/crawler/feed_service.py b/friend_circle_lite/crawler/feed_service.py index 8afb2d0cbb4..c41e9a9a189 100644 --- a/friend_circle_lite/crawler/feed_service.py +++ b/friend_circle_lite/crawler/feed_service.py @@ -59,7 +59,7 @@ def discover(self, website_url: str) -> FeedEndpoint | None: if " list[Artic response.encoding = "utf-8" feed = feedparser.parse(response.text) except Exception as exc: - logging.error(f"解析 RSS 失败：{feed_url}，错误: {exc}") + logging.error(f"[RSS 抓取] 解析 RSS 失败：{feed_url}，错误: {exc}") return [] default_author = feed.feed.author if "author" in feed.feed else "" @@ -113,7 +113,7 @@ def safe_parse_date(article): try: return datetime.strptime(article.published, "%Y-%m-%d %H:%M") except ValueError: - logging.warning(f"文章 {article.title} 的发布时间格式异常: {article.published}，已跳过") + logging.warning(f"[RSS 抓取] 文章 {article.title} 的发布时间格式异常: {article.published}，已跳过") return None # 只保留能成功解析日期的文章 @@ -141,11 +141,11 @@ def convert_time_to_string(time_value): elif isinstance(time_value, time.struct_time): # 检查年份是否异常 if time_value.tm_year < 1900: - logging.warning(f"文章 {entry.get('title', 'Unknown')} 的时间年份异常: {time_value.tm_year}，已跳过") + logging.warning(f"[RSS 抓取] 文章 {entry.get('title', 'Unknown')} 的时间年份异常: {time_value.tm_year}，已跳过") return "" return time.strftime('%Y-%m-%dT%H:%M:%SZ', time_value) else: - logging.warning(f"文章 {entry.get('title', 'Unknown')} 的时间格式未知: {type(time_value)}，已跳过") + logging.warning(f"[RSS 抓取] 文章 {entry.get('title', 'Unknown')} 的时间格式未知: {type(time_value)}，已跳过") return "" if "published" in entry: @@ -158,10 +158,10 @@ def convert_time_to_string(time_value): if not time_str: return "" published = format_published_time(time_str) - logging.warning(f"文章 {entry.title} 未包含发布时间，已使用更新时间 {published}") + logging.warning(f"[RSS 抓取] 文章 {entry.title} 未包含发布时间，已使用更新时间 {published}") return published - logging.warning(f"文章 {entry.title} 未包含任何时间信息, 请检查原文, 跳过该文章") + logging.warning(f"[RSS 抓取] 文章 {entry.title} 未包含任何时间信息，请检查原文，跳过该文章") return "" @@ -184,7 +184,7 @@ def diff_and_persist(self, latest_articles: list[Article]) -> list[dict] | None: # First run: no previous data exists, skip sending to prevent sending old articles if not previous_articles: - logging.info(f"首次运行：跳过推送以防止发送旧文章") + logging.info("[文章追踪] 首次运行：跳过推送以防止发送旧文章") self.store.save_articles(latest_articles) return None @@ -210,16 +210,16 @@ def diff_and_persist(self, latest_articles: list[Article]) -> list[dict] | None: if previous_latest_date is None or article_date > previous_latest_date: truly_new_articles.append(article) except Exception as exc: - logging.warning(f"解析文章日期失败: {article.title}, 日期: {article.published}, 错误: {exc}") + logging.warning(f"[文章追踪] 解析文章日期失败: {article.title}, 日期: {article.published}, 错误: {exc}") continue self.store.save_articles(latest_articles) if truly_new_articles: - logging.info(f"发现 {len(truly_new_articles)} 篇新文章（日期比之前更新）") + logging.info(f"[文章追踪] 发现 {len(truly_new_articles)} 篇新文章（日期比之前更新）") return [article.to_tracking_dict() for article in truly_new_articles] else: - logging.info(f"发现 {len(new_articles)} 篇新文章，但日期不够新，跳过推送") + logging.info(f"[文章追踪] 发现 {len(new_articles)} 篇新文章，但日期不够新，跳过推送") return None @staticmethod diff --git a/friend_circle_lite/crawler/service.py b/friend_circle_lite/crawler/service.py index 0b7889f316a..40cfd12fee4 100644 --- a/friend_circle_lite/crawler/service.py +++ b/friend_circle_lite/crawler/service.py @@ -37,16 +37,16 @@ def resolve(self, website: Website) -> FeedEndpoint | None: configured = self.feed_lookup.get(website.name) if configured: if configured.source == 'manual': - logging.info(f"'{website.name}' 使用预设 RSS 源：{configured.url}") + logging.info(f"[RSS 解析] {website.name} 使用预设 RSS 源：{configured.url}") elif configured.source == 'cache': - logging.info(f"'{website.name}' 使用缓存 RSS 源：{configured.url}") + logging.info(f"[RSS 解析] {website.name} 使用缓存 RSS 源：{configured.url}") else: - logging.info(f"'{website.name}' 使用 RSS 源：{configured.url} (来源: {configured.source})") + logging.info(f"[RSS 解析] {website.name} 使用 RSS 源：{configured.url} ，来源: {configured.source}") return FeedEndpoint(url=configured.url, feed_type="specific", source=configured.source) discovered = self.discovery_service.discover(website.url) if discovered: - logging.info(f"'{website.name}' 自动探测到 RSS：{discovered.url}") + logging.info(f"[RSS 探测] {website.name} 自动探测到 RSS：{discovered.url}") return discovered @@ -69,14 +69,14 @@ def crawl(self, website: Website, count: int) -> CrawlResult: parse_error = endpoint is not None and not articles if parse_error and endpoint and endpoint.source in ("cache", "unknown"): - logging.warning(f"'{website.name}' 缓存 RSS 本次抓取失败，将等待下次友链检测刷新 RSS 缓存") + logging.warning(f"[RSS 抓取] {website.name} 缓存 RSS 本次抓取失败，将等待下次友链检测刷新 RSS 缓存") status = "active" if articles else "error" if not articles: if endpoint is None: - logging.warning(f"'{website.name}' 的博客 {website.url} 未找到有效 RSS ") + logging.warning(f"[RSS 抓取] {website.name} 的博客 {website.url} 未找到有效 RSS") else: - logging.warning(f"'{website.name}' 的 RSS {endpoint.url} 未解析出文章 ") + logging.warning(f"[RSS 抓取] {website.name} 的 RSS {endpoint.url} 未解析出文章") return CrawlResult( website=website, @@ -96,7 +96,7 @@ def _parse_articles(self, endpoint: FeedEndpoint | None, website: Website, count for article in articles: article.author = website.name article.avatar = website.avatar - logging.info(f"{website.name} 发布了新文章：{article.title}，时间：{article.published}，链接：{article.link}") + logging.info(f"[RSS 抓取] {website.name} 发布了新文章：{article.title}，时间：{article.published}，链接：{article.link}") return articles @@ -144,8 +144,10 @@ def run(self) -> tuple[dict, list[list[str]]] | None: and website.name in feed_names ] skipped_count = len(websites) - len(crawlable_websites) - if skipped_count: - logging.info(f"🔎 根据友链可达性检测跳过 {skipped_count} 个不可抓取站点") + logging.info( + f"[朋友圈抓取] 友链总数 {len(websites)} 个，可进入 RSS 抓取 {len(crawlable_websites)} 个，" + f"跳过 {skipped_count} 个不可抓取或无 RSS 缓存站点" + ) discovery_service = FeedDiscoveryService(session, self.proxy_settings) parser_service = FeedParserService(session, self.proxy_settings) @@ -153,6 +155,7 @@ def run(self) -> tuple[dict, list[list[str]]] | None: crawler = SingleSiteCrawler(parser_service=parser_service, resolver=resolver) crawl_results: list[CrawlResult] = [] + logging.info(f"[朋友圈抓取] 开始抓取 {len(crawlable_websites)} 个可抓取站点，每站最多 {self.count} 篇文章") with ThreadPoolExecutor(max_workers=10) as executor: future_to_website = { executor.submit(crawler.crawl, website, self.count): website @@ -163,7 +166,7 @@ def run(self) -> tuple[dict, list[list[str]]] | None: try: crawl_results.append(future.result()) except Exception as exc: - logging.error(f"处理 {website.to_error_payload()} 时发生错误: {exc}", exc_info=True) + logging.error(f"[朋友圈抓取] 处理 {website.to_error_payload()} 时发生错误: {exc}", exc_info=True) crawl_results.append(CrawlResult(website=website, status="error")) self._apply_cache_updates(cache_records, crawl_results, manual_names) @@ -187,8 +190,8 @@ def run(self) -> tuple[dict, list[list[str]]] | None: } link_payload = self._build_link_payload(link_check_records) logging.info( - f"数据处理完成，总共有 {len(websites)} 位朋友，其中 {len(active_results)} 位博客可抓取到文章，" - f"{len(crawl_error_results)} 位博客 RSS 抓取失败，{len(unreachable_results)} 位友链不可达。" + f"[数据汇总] 处理完成：友链总数 {len(websites)} 个，成功抓取文章站点 {len(active_results)} 个，" + f"RSS 抓取失败 {len(crawl_error_results)} 个，友链不可达 {len(unreachable_results)} 个。" ) return result, error_results, link_payload @@ -215,11 +218,11 @@ def _apply_feed_updates_from_link_check(self, updates: dict[str, CacheRecord | N if name in cache_map: cache_map.pop(name) changed = True - logging.info(f"🗑️ 可达性检测删除失效 RSS 缓存: {name}") + logging.info(f"[RSS 缓存] 可达性检测删除失效 RSS 缓存: {name}") else: cache_map[name] = record changed = True - logging.info(f"💾 可达性检测保存 RSS 缓存: {name} -> {record.url}") + logging.info(f"[RSS 缓存] 可达性检测保存 RSS 缓存: {name} -> {record.url}") if changed: self.cache_store.save_records(list(cache_map.values())) @@ -344,7 +347,7 @@ def sort_articles_by_time(data: dict, future_tolerance_days: int = 2) -> dict: for article in data.get("article_data", []): if not article.get("created"): article["created"] = "2024-01-01 00:00" - logging.warning(f"文章 {article['title']} 未包含时间信息，已设置为默认时间 2024-01-01 00:00") + logging.warning(f"[数据处理] 文章 {article['title']} 未包含时间信息，已设置为默认时间 2024-01-01 00:00") now = datetime.now(ZoneInfo("Asia/Shanghai")).replace(tzinfo=None) max_allowed_time = now + timedelta(days=future_tolerance_days) @@ -356,7 +359,7 @@ def sort_articles_by_time(data: dict, future_tolerance_days: int = 2) -> dict: if article_time > max_allowed_time: removed_count += 1 logging.warning( - f"文章 {article['title']} 的时间 {article['created']} 超出当前时间 {future_tolerance_days} 天以上，已跳过显示" + f"[数据处理] 文章 {article['title']} 的时间 {article['created']} 超出当前时间 {future_tolerance_days} 天以上，已跳过显示" ) continue filtered_articles.append(article) @@ -364,7 +367,7 @@ def sort_articles_by_time(data: dict, future_tolerance_days: int = 2) -> dict: filtered_articles.sort(key=lambda item: datetime.strptime(item["created"], "%Y-%m-%d %H:%M"), reverse=True) data["article_data"] = filtered_articles if removed_count: - logging.info(f"已过滤 {removed_count} 篇未来时间异常的文章") + logging.info(f"[数据处理] 已过滤 {removed_count} 篇未来时间异常的文章") return data @@ -376,7 +379,7 @@ def limit_large_dataset(result: dict, future_tolerance_days: int = 2) -> dict: max_articles = 150 if len(article_data) > max_articles: - logging.info("数据量较大，开始进行处理...") + logging.info(f"[数据处理] 数据量较大，开始裁剪，当前 {len(article_data)} 篇，基础保留 {max_articles} 篇") top_authors = {article["author"] for article in article_data[:max_articles]} filtered_articles = article_data[:max_articles] + [ article for article in article_data[max_articles:] @@ -384,7 +387,7 @@ def limit_large_dataset(result: dict, future_tolerance_days: int = 2) -> dict: ] result["article_data"] = filtered_articles result["statistical_data"]["article_num"] = len(filtered_articles) - logging.info(f"数据处理完成，保留 {len(filtered_articles)} 篇文章") + logging.info(f"[数据处理] 数据裁剪完成，保留 {len(filtered_articles)} 篇文章") return result diff --git a/friend_circle_lite/link_checker/service.py b/friend_circle_lite/link_checker/service.py index ddc2168b63d..3e098ae2abd 100644 --- a/friend_circle_lite/link_checker/service.py +++ b/friend_circle_lite/link_checker/service.py @@ -84,16 +84,27 @@ def check_websites(self, websites: list[Website]) -> list[LinkCheckRecord]: else: websites_to_check.append(website) + total_count = len(websites) + cached_count = total_count - len(websites_to_check) + logging.info( + f"[友链检测] 友链总数 {total_count} 个，缓存复用 {cached_count} 个，" + f"本次实际检测 {len(websites_to_check)} 个，缓存有效期 {self.config.max_age_hours} 小时" + ) + if websites_to_check: - logging.info(f"🔎 开始检测 {len(websites_to_check)} 个友链状态") + logging.info( + f"[友链检测] 开始实际检测 {len(websites_to_check)} 个友链状态，" + f"其余 {cached_count} 个复用缓存" + ) checked_records = self._check_fresh_websites(websites_to_check, cached_records) self.store.save_records(checked_records) for record in checked_records: records_by_url[record.url] = record else: - logging.info("🔎 友链状态缓存仍有效，本次复用缓存结果") + logging.info(f"[友链检测] 全部 {total_count} 个友链状态缓存仍有效，本次不发起友链检测请求") if backlink_refresh_records: + logging.info(f"[反链检测] 友链页地址变更，单独刷新 {len(backlink_refresh_records)} 个反链状态") self._refresh_backlinks_only(backlink_refresh_records) return [records_by_url.get(website.url) or LinkCheckRecord.unchecked(website) for website in websites] @@ -114,7 +125,7 @@ def _check_fresh_websites(self, websites: list[Website], cached_records: dict[st try: records.append(future.result()) except Exception as exc: - logging.warning(f"友链 {website.name} 检测失败: {exc}") + logging.warning(f"[友链检测] 友链 {website.name} 检测失败: {exc}") records.append(self._build_failed_record(website, cached_records.get(website.url))) return records diff --git a/friend_circle_lite/storage/sqlite_store.py b/friend_circle_lite/storage/sqlite_store.py index e60f6057c46..b5f1fd683bb 100644 --- a/friend_circle_lite/storage/sqlite_store.py +++ b/friend_circle_lite/storage/sqlite_store.py @@ -43,10 +43,10 @@ def load_records(self) -> list[CacheRecord]: migrated_records = self._load_legacy_records() if migrated_records: if self.save_records(migrated_records): - logging.info(f"已从旧格式迁移 {len(migrated_records)} 条 RSS 缓存到 SQLite") + logging.info(f"[RSS 缓存] 已从旧格式迁移 {len(migrated_records)} 条 RSS 缓存到 SQLite") return migrated_records - logging.info(f"RSS 缓存文件不存在，将在首次抓取后自动创建") + logging.info("[RSS 缓存] RSS 缓存文件不存在，将在首次抓取后自动创建") return [] def save_records(self, records: list[CacheRecord]) -> bool: @@ -64,10 +64,10 @@ def save_records(self, records: list[CacheRecord]) -> bool: [(record.name, record.url, record.source) for record in sorted(records, key=lambda item: item.name)], ) connection.commit() - logging.info(f"RSS 缓存已保存（{len(records)} 条）") + logging.info(f"[RSS 缓存] RSS 缓存已保存（{len(records)} 条）") return True except Exception as exc: - logging.error(f"保存 RSS 缓存失败: {exc}") + logging.error(f"[RSS 缓存] 保存 RSS 缓存失败: {exc}") return False def _load_from_sqlite(self) -> list[CacheRecord]: @@ -80,7 +80,7 @@ def _load_from_sqlite(self) -> list[CacheRecord]: "SELECT name, url, source FROM feed_cache ORDER BY name" ).fetchall() except Exception as exc: - logging.warning(f"读取 RSS 缓存失败: {exc}") + logging.warning(f"[RSS 缓存] 读取 RSS 缓存失败: {exc}") return [] return [ @@ -188,10 +188,10 @@ def load_articles(self) -> list[Article]: migrated_articles = self._load_legacy_json() if migrated_articles: if self.save_articles(migrated_articles): - logging.info(f"已从旧 JSON 格式迁移 {len(migrated_articles)} 篇文章记录到 SQLite") + logging.info(f"[文章追踪] 已从旧 JSON 格式迁移 {len(migrated_articles)} 篇文章记录到 SQLite") return migrated_articles - logging.info(f"文章追踪数据不存在，这是首次运行") + logging.info("[文章追踪] 文章追踪数据不存在，这是首次运行") return [] def save_articles(self, articles: list[Article]) -> bool: @@ -230,7 +230,7 @@ def save_articles(self, articles: list[Article]) -> bool: connection.commit() return True except Exception as exc: - logging.error(f"保存文章追踪数据失败: {exc}") + logging.error(f"[文章追踪] 保存文章追踪数据失败: {exc}") return False def _load_from_sqlite(self) -> list[Article]: @@ -245,7 +245,7 @@ def _load_from_sqlite(self) -> list[Article]: ORDER BY published DESC""" ).fetchall() except Exception as exc: - logging.warning(f"读取文章追踪数据失败: {exc}") + logging.warning(f"[文章追踪] 读取文章追踪数据失败: {exc}") return [] return [ @@ -336,7 +336,7 @@ def load_records(self, urls: list[str] | None = None) -> dict[str, LinkCheckReco """ ).fetchall() except Exception as exc: - logging.warning(f"读取友链检测缓存失败: {exc}") + logging.warning(f"[友链检测] 读取友链检测缓存失败: {exc}") return {} allowed_urls = {normalize_homepage_url(url) for url in (urls or [])} @@ -415,10 +415,10 @@ def save_records(self, records: list[LinkCheckRecord]) -> bool: [self._record_to_row(record) for record in records], ) connection.commit() - logging.info(f"友链检测缓存已保存（{len(records)} 条）") + logging.info(f"[友链检测] 友链检测缓存已保存（{len(records)} 条）") return True except Exception as exc: - logging.error(f"保存友链检测缓存失败: {exc}") + logging.error(f"[友链检测] 保存友链检测缓存失败: {exc}") return False @staticmethod diff --git a/tests/test_refactor_contracts.py b/tests/test_refactor_contracts.py index 0b161cdbff8..7dde9a95fbc 100644 --- a/tests/test_refactor_contracts.py +++ b/tests/test_refactor_contracts.py @@ -14,6 +14,7 @@ from friend_circle_lite.crawler.service import FeedResolver, FriendCircleCrawlService, SingleSiteCrawler from friend_circle_lite.all_friends import deal_with_large_data, merge_link_data_from_json_url from friend_circle_lite.app_config import ApplicationConfig +from friend_circle_lite.cli import FriendCircleLiteApplication from friend_circle_lite.link_checker.service import LinkReachabilityService from friend_circle_lite.models import Article, CacheRecord, FeedEndpoint, LinkCheckRecord, LinkMethodStatus, Website from friend_circle_lite.outputs.legacy_api import _to_public_link @@ -21,6 +22,23 @@ class RefactorContractsTest(unittest.TestCase): + def test_github_action_schedule_uses_22_minute_offset(self): + workflow = Path(".github/workflows/friend_circle_lite.yml").read_text(encoding="utf-8") + + self.assertIn('cron: "22 */4 * * *"', workflow) + self.assertNotIn('cron: "0 */4 * * *"', workflow) + + def test_github_actions_opt_into_node24_runtime(self): + workflow_paths = [ + Path(".github/workflows/friend_circle_lite.yml"), + Path(".github/workflows/deal_subscribe_issue.yml"), + ] + + for workflow_path in workflow_paths: + with self.subTest(workflow=str(workflow_path)): + workflow = workflow_path.read_text(encoding="utf-8") + self.assertIn("FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true", workflow) + def test_link_check_uses_cached_rss_without_homepage_request(self): class Store: def load_records(self, urls): @@ -55,6 +73,84 @@ def get(self, *args, **kwargs): self.assertTrue(records[0].crawl_allowed) self.assertEqual(records[0].best_method, "rss_cache") + def test_link_check_logs_total_cached_and_actual_check_counts(self): + class Store: + def load_records(self, urls): + return { + "https://cached.example/": LinkCheckRecord( + name="Cached", + url="https://cached.example/", + checked_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + reachable=True, + crawl_allowed=False, + best_method="homepage", + best_latency=0.2, + ) + } + + def save_records(self, records): + return True + + def is_fresh(self, record, max_age_hours): + return True + + class Discovery: + def discover(self, website_url): + return None + + class Response: + status_code = 200 + + class Fetcher: + def get(self, *args, **kwargs): + return type("Result", (), {"response": Response(), "latency": 0.1, "success": True})() + + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"max_age_hours": 24}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + feed_parser=type("Parser", (), {"parse": lambda self, *args, **kwargs: [], "last_latency": 0.01})(), + feed_discovery=Discovery(), + fetcher=Fetcher(), + ) + + with patch("logging.info") as info: + service.check_websites([ + Website(name="Cached", url="https://cached.example/", avatar="cached.png"), + Website(name="Fresh", url="https://fresh.example/", avatar="fresh.png"), + ]) + + messages = "\n".join(str(call.args[0]) for call in info.call_args_list) + self.assertIn("[友链检测]", messages) + self.assertIn("友链总数 2 个", messages) + self.assertIn("缓存复用 1 个", messages) + self.assertIn("本次实际检测 1 个", messages) + + def test_crawler_entry_logs_source_and_article_limit_with_module_label(self): + config = ApplicationConfig.from_dict({ + "spider_settings": { + "enable": True, + "json_url": "https://example.com/friends.json", + "article_count": 3, + }, + "runtime_paths": { + "all_json_file": "./tmp/all.json", + "errors_json_file": "./tmp/errors.json", + "link_json_file": "./tmp/link.json", + }, + }) + payload = ({"statistical_data": {}, "article_data": []}, [], {"statistical_data": {}, "link_data": []}) + + with patch("friend_circle_lite.cli.fetch_and_process_data", return_value=payload), \ + patch("friend_circle_lite.cli.write_json"), \ + patch("logging.info") as info: + FriendCircleLiteApplication(config).run_crawler_if_enabled() + + messages = "\n".join(str(call.args[0]) for call in info.call_args_list) + self.assertIn("[爬虫入口]", messages) + self.assertIn("https://example.com/friends.json", messages) + self.assertIn("每站最多 3 篇文章", messages) + def test_link_check_reuses_cached_linkpage_when_only_trailing_slash_differs(self): class Store: def load_records(self, urls): From d1fde2fe9160bfb1bbe021bca6fe761803af05ea Mon Sep 17 00:00:00 2001 From: LiuShen <01@liushen.fun> Date: Mon, 8 Jun 2026 00:37:26 +0800 Subject: [PATCH 21/30] =?UTF-8?q?=F0=9F=98=92=E6=9B=B4=E6=96=B0=E4=B8=BB?= =?UTF-8?q?=E9=A1=B5=EF=BC=8C=E4=BD=BF=E5=85=B6=E6=9B=B4=E5=8A=A0=E5=A5=BD?= =?UTF-8?q?=E7=9C=8B~?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- static/index.html | 1308 ++++++++++++++++++------------ tests/test_refactor_contracts.py | 35 + 2 files changed, 822 insertions(+), 521 deletions(-) diff --git a/static/index.html b/static/index.html index 0561f237735..338b445c588 100644 --- a/static/index.html +++ b/static/index.html @@ -1,11 +1,11 @@ - + - 友链朋友圈状态 + 友链清册 - - -
-

凤凰台上凤凰游，凤去台空江自流。

-

吴宫花草埋幽径，晋代衣冠成古丘。

-

三山半落青天外，二水中分白鹭洲。

-

总为浮云能蔽日，长安不见使人愁。

-

- —— 节选自 李白《登金陵凤凰台》

-
+ .article-title { + padding-right: 42px; + } -
- - - - - -
+ .article-avatar-mark { + top: 9px; + right: 9px; + width: 32px; + height: 32px; + padding: 2px; + } -
-
-

友链朋友圈状态

-

- 基于 Friend-Circle-Lite 生成的纯静态数据，展示友链可达性与友圈文章。 -

-

更新时间：加载中...

-
- all.json - link.json -
-
+ .poem-background { + bottom: calc(50% - 220px); + left: 54%; + opacity: 0.14; + } -
-
-
- - 友链总数 -
- 加载中... -
-
-
- - 可抓取友链 -
- 加载中... -
-
-
- - 错误友链 -
- 加载中... + .poem-background p { + font-size: 2rem; + } + } + + + +
+
+
+ FRIEND CIRCLE LITE +

友链清册

+

正在展开友链清册...

-
-
+
- 加载中... -
-
+ 友链朋友圈 + + + -
-
- - - -
- +
+ + +
+
+

友链可达性数据

+ +
+
+
-
- -

失效友链

-
- -
- -

其他友链

-
- -
- -

友圈文章

-
- -