diff --git a/.env.example b/.env.example new file mode 100644 index 00000000000..08299e579af --- /dev/null +++ b/.env.example @@ -0,0 +1,2 @@ +# SMTP 密码，请在github上配置，如果为服务器部署，不担心泄露，可以在这里直接写入 +SMTP_PWD=123432424 \ No newline at end of file diff --git a/.github/workflows/deal_subscribe_issue.yml b/.github/workflows/deal_subscribe_issue.yml index a8b981e2f77..ba2efad25e2 100644 --- a/.github/workflows/deal_subscribe_issue.yml +++ b/.github/workflows/deal_subscribe_issue.yml @@ -4,6 +4,9 @@ on: issues: types: [opened] +env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true + jobs: handle_email_issues: if: startsWith(github.event.issue.title, '[邮箱订阅]') diff --git a/.github/workflows/friend_circle_lite.yml b/.github/workflows/friend_circle_lite.yml index f215e5dd67a..2094b4695db 100644 --- a/.github/workflows/friend_circle_lite.yml +++ b/.github/workflows/friend_circle_lite.yml @@ -2,15 +2,20 @@ name: Friend Circle Lite on: schedule: - - cron: "0 */4 * * *" + - cron: "22 */4 * * *" workflow_dispatch: env: TZ: Asia/Shanghai + PAGE_BRANCH: page + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true jobs: friend-circle-lite: runs-on: ubuntu-latest + permissions: + contents: write + actions: write steps: - name: Pull Latest Repository @@ -51,7 +56,9 @@ jobs: - name: Check RSS feeds env: SMTP_PWD: ${{ secrets.SMTP_PWD }} + PROXY_URL: ${{ secrets.PROXY_URL }} FCL_REPO: ${{ github.repository }} + FCL_CONFIG_OVERRIDES: ${{ vars.FCL_CONFIG_OVERRIDES }} run: | echo "Checking RSS feeds..." python run.py @@ -69,15 +76,27 @@ jobs: git config --global user.name 'github-actions[bot]' git config --global user.email 'github-actions[bot]@users.noreply.github.com' - - name: Commit changes + - name: Build static publish directory run: | mkdir pages - cp -r main ./static/index.html ./static/readme.md ./static/favicon.ico ./static/bg-light.webp ./static/bg-dark.webp all.json errors.json pages/ + cp -r main ./static/edgeone.json ./static/_headers ./static/index.html ./static/readme.md ./static/favicon.ico ./static/bg-light.webp ./static/bg-dark.webp all.json link.json errors.json pages/ + + - name: Publish static assets to branches + run: | + published_tree="" + if git fetch origin "${PAGE_BRANCH}:refs/remotes/origin/${PAGE_BRANCH}"; then + published_tree=$(git rev-parse "origin/${PAGE_BRANCH}^{tree}") + fi cd pages git init git add . + generated_tree=$(git write-tree) + if [ -n "$published_tree" ] && [ "$generated_tree" = "$published_tree" ]; then + echo "No static asset changes; skipping page branch update." + exit 0 + fi git commit -m "⏱️ $(date +"%Y年%m月%d日-%H时%M分") GitHub Actions定时更新" - git push --force https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git HEAD:page + git push --force https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git HEAD:${{ env.PAGE_BRANCH }} - name: Delete Workflow Runs uses: Mattraks/delete-workflow-runs@v2 @@ -92,5 +111,10 @@ jobs: runs-on: ubuntu-latest permissions: actions: write + contents: read steps: - - uses: liskin/gh-workflow-keepalive@v1 \ No newline at end of file + - uses: liskin/gh-workflow-keepalive@v1 + with: + GITHUB_TOKEN: ${{ github.token }} + env: + GH_TOKEN: ${{ github.token }} diff --git a/.gitignore b/.gitignore index 7747aadb5c2..a6ab3cec929 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,10 @@ # 忽略数据文件 *.json +!edgeone.json -*.bat \ No newline at end of file +*.bat + +.env + +temp/ \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 00000000000..a9adc634ab0 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,577 @@ +# Changelog + +## [2.1.0] - 2026-06-08 + +### 本次更新范围 + +本次更新整理自 2026-06-06 至 2026-06-08 的提交： + +- `a50101a`：将代理配置单独提取出来 +- `673ee3b`：完善逻辑，将网站可达性检测深度融入 +- `e781ccf`：更新逻辑，实现更加优雅的实现 +- `5e9dd6b`：完善日志流 +- `d1fde2f`：更新主页，使其更加好看 + +这次更新重点是减少重复请求、稳定公开数据结构、整理配置项、增强调试能力，并重写独立展示首页。 + +--- + +### 核心逻辑调整 + +#### 1. 友链可达性检测成为友圈抓取前置流程 + +友圈抓取现在依赖友链可达性检测结果，`link_check.enable` 旧字段仍会兼容读取，但不再作为有效开关。 + +新的检测流程如下： + +1. 优先尝试 RSS 探测或复用缓存 RSS。 +2. RSS 可解析时，直接认为站点可达且可抓取。 +3. RSS 不可解析时，记录 `crawlable=false`，再尝试主页可达性。 +4. 主页仍不可达时，再使用状态码 API 作为兜底可达性判断。 +5. 反链检测在可达性检测后执行，只有站点可达时才继续检测。 + +这样后续友圈抓取只会尝试 `crawlable=true` 且存在 RSS 缓存的站点，不会在每次抓取时重复遍历探测所有 RSS 地址。 + +默认运行节奏为： + +- GitHub Actions 每 4 小时执行一次友链朋友圈任务。 +- 友链可达性检测与 RSS 探测按 `link_check.max_age_hours` 控制，默认 24 小时才对同一站点重新检测一次。 +- 没有 RSS 的站点会缓存为 `crawlable=false`，在缓存有效期内直接跳过，不会每 4 小时重复探测。 +- 每 4 小时的友圈抓取会复用检测缓存，只请求可抓取 RSS 的站点，从而在保证文章更新频率的同时尽量降低请求数量。 + +#### 2. RSS 缓存与可达性缓存合并维护 + +- 友链检测缓存默认 24 小时，由 `link_check.max_age_hours` 控制。 +- 缓存未过期时，RSS 地址、主页可达性、反链结果会直接复用。 +- 已缓存 RSS 地址的站点会优先重新验证该 RSS；验证失败后才重新走 RSS 探测流程。 +- 主页 URL 与友链页 URL 会做末尾斜杠归一化，避免同一站点因 `/` 差异重复检测。 +- 新增参考延迟 `latency`，取 RSS、主页或 API 阶段中实际完成检测的耗时，不再使用 `0.0` 或 `-1.0` 作为公开占位值。 + +#### 3. 反链检测独立于 RSS 探测 + +反链检测以友链原数据中的主页 URL / 友链页 URL 为依据，不依赖 RSS 探测结果。若站点不可达，则反链直接视为未通过；若站点可达且开启 `enable_backlink_check`，才请求友链页并判断是否包含 `author_url`。 + +--- + +### 配置变更 + +#### 1. 新增代理配置块 + +代理配置从友链检测中拆出，统一放到顶级 `proxy_settings`： + +```yaml +proxy_settings: + proxy_url: "" +``` + +程序会先直连，请求失败且配置了代理时自动走代理。推荐通过仓库环境变量 `PROXY_URL` 覆盖，不建议直接把代理地址写入配置文件。 + +推荐代理格式为代理前缀： + +```text +https://proxy.example.com/ +``` + +程序会自动拼接为： + +```text +https://proxy.example.com/https://example.com/feed.xml +``` + +如需自行搭建代理，可参考：[使用 CF Workers 搭建反代加速器](https://blog.liushen.fun/posts/dd89adc9/)。 + +仍兼容 `https://proxy.example.com?url={url}` 这类高级格式，但普通反代场景不推荐。 + +#### 2. `link_check` 配置简化 + +当前有效配置如下： + +```yaml +link_check: + max_age_hours: 24 + timeout: 15 + max_workers: 10 + status_api_url: "https://v2.xxapi.cn/api/status?url={url}" + enable_backlink_check: true + author_url: "blog.liushen.fun" +``` + +变化说明： + +- `enable`：旧字段兼容读取，但友圈抓取依赖可达性检测，因此不再作为有效开关。 +- `proxy_url`：已移动到 `proxy_settings.proxy_url`。 +- `status_api_url`：只作为兜底可达性检测。API-only 结果只用于展示，不参与 RSS 抓取。 + +#### 3. 新增调试开关 + +```yaml +debug: false +``` + +也可以通过环境变量开启： + +```text +FCL_DEBUG=1 +``` + +开启后，程序结束前会全量打印 SQLite 缓存表结构与所有数据，并保守清理核心表中的旧字段或残留结构，方便定位缓存兼容问题。 + +--- + +### 数据结构变更 + +#### 1. `all.json` 保持稳定 + +`all.json` 仍只面向友链朋友圈文章展示，公开结构不变： + +```json +{ + "statistical_data": { + "friends_num": 201, + "active_num": 160, + "error_num": 41, + "article_num": 275, + "last_updated_time": "2026-06-07 00:41:06" + }, + "article_data": [] +} +``` + +#### 2. `link.json` 简化为可达性展示数据 + +`link.json` 集中保存友链可达性数据和检测统计： + +```json +{ + "statistical_data": { + "link_total_num": 201, + "link_reachable_num": 182, + "link_unreachable_num": 19, + "crawl_allowed_num": 180, + "api_only_num": 0, + "has_author_link_num": 130, + "link_last_checked_time": "2026-06-07 00:40:31" + }, + "link_data": [ + { + "name": "清羽飞扬", + "link": "https://blog.liushen.fun/", + "link_page": "https://blog.liushen.fun/link/", + "avatar": "https://blog.liushen.fun/favicon.ico", + "reachable": true, + "crawlable": true, + "latency": 0.21, + "fail_count": 0, + "has_backlink": true + } + ] +} +``` + +本次移除公开展示中不必要的冗余字段： + +- `method` +- `reason` +- 单站 `checked_at` + +保留顶部 `link_last_checked_time` 用于表示本轮友链检测数据更新时间。缓存未过期且未重新检测时，该时间不会被每次运行强行刷新。 + +#### 3. `errors.json` + +`errors.json` 仍作为兼容输出保留，用于旧流程或需要查看异常友链的场景。新首页主要读取 `all.json` 与 `link.json`。 + +--- + +### 数据合并 + +数据合并继续使用顶级 `merge_settings`： + +```yaml +merge_settings: + enable: false + remote_base_url: "https://fc.liushen.fun" + merge_article_data: true + merge_link_check_data: true +``` + +合并时会按 `remote_base_url` 自动请求： + +- `/all.json` +- `/link.json` +- `/errors.json` + +文章数据与友链可达性数据可分别通过 `merge_article_data`、`merge_link_check_data` 控制。适合国内外两条线路分别维护缓存后再合并公开输出。 + +--- + +### 日志与 GitHub Actions + +- 日志增加模块标签，例如 `[爬虫入口]`、`[友链检测]`、`[RSS 抓取]`，并补充总友链数、缓存复用数、本次实际检测数等关键信息。 +- URL 后统一留空格再接中文说明，避免日志中的链接和中文被部分渲染器连在一起。 +- GitHub Actions 定时任务调整为 `22 */4 * * *`，避开整点拥堵。 +- 工作流增加 `FORCE_JAVASCRIPT_ACTIONS_TO_NODE24=true`，提前适配 GitHub Actions Node.js 24 运行时。 + +--- + +### 前端展示页 + +`static/index.html` 重写为单文件展示页： + +- 顶部可切换 `友链可达性` 与 `友链朋友圈`。 +- `友链可达性` 默认显示不可达友链，`全部` 按钮放在最后。 +- `友链朋友圈` 默认显示前 24 条，`全部` 按钮放在最后。 +- 友链状态以图标展示，鼠标悬浮显示详细说明。 +- 文章卡片展示作者头像，标题最多两行并自动省略。 +- 页面恢复国风浅蓝背景与《登金陵凤凰台》诗句底纹。 +- 滚动条、响应式布局、少量文章卡片高度等细节已优化。 +- `main/fclite.js` 和 `main/fclite.css` 保持不变，外部用户原有嵌入方式不受影响。 + +--- + +### 迁移提示 + +从旧配置升级时，建议检查以下内容： + +1. 将代理地址从 `link_check.proxy_url` 移到 `proxy_settings.proxy_url`，或直接使用仓库环境变量 `PROXY_URL`。 +2. 删除或忽略 `link_check.enable`，当前友链检测会始终作为友圈抓取前置流程运行。 +3. 若使用自定义发布脚本，请确保发布目录包含 `all.json`、`link.json`、`errors.json`。 +4. 若之前依赖 `link.json` 中的 `method`、`reason`、单站 `checked_at`，请改用 `reachable`、`crawlable`、`latency`、`fail_count`、`has_backlink` 与顶部 `link_last_checked_time`。 +5. 出现缓存兼容问题时，可临时设置 `FCL_DEBUG=1` 查看 SQLite 缓存内容。 + +--- + +## [2.0.0] - 2026-05-31 + +### 🎉 重大更新：友链可达性检测与数据结构重构 + +本次更新合并了 [check-flink](https://github.com/willow-god/check-flink) 项目的友链可达性检测功能，并对数据结构进行了重大重构，实现了友圈文章与友链状态的分离。 + +--- + +## 📋 目录 + +- [核心功能变更](#核心功能变更) +- [数据结构变更](#数据结构变更) +- [配置文件变更](#配置文件变更) +- [前端页面变更](#前端页面变更) +- [迁移指南](#迁移指南) +- [兼容性说明](#兼容性说明) + +--- + +## 🚀 核心功能变更 + +### 新增功能 + +#### 1. 友链可达性检测 +- **直连检测**：直接访问友链站点，验证可达性 +- **代理检测**：通过代理服务（如 Cloudflare Worker）检测被墙站点 +- **API 检测**：通过第三方 API 兜底检测（如 xxapi.cn） +- **反链检测**：检测友链页面是否包含你的站点链接 +- **智能缓存**：检测结果缓存 24 小时，避免频繁请求 +- **失败计数**：记录连续失败次数，便于识别长期失效友链 + +#### 2. 数据合并功能重构 +- **独立配置**：`merge_settings` 从 `spider_settings` 中独立出来 +- **并列控制**：友圈文章和友链可达性数据分别控制是否合并 +- **智能合并**： + - 可达性优先级：direct > proxy > api > none + - 延迟取最优：选择响应时间更短的结果 + - 反链取并集：任一环境检测到反链即为有反链 + - 失败次数取最小：选择失败次数较少的结果 + +#### 3. 前端展示页面 +- **新首页**：继承 check-flink 风格，同时展示友链状态和友圈文章 +- **友链状态卡片**：显示可达性、响应时间、失败次数、反链状态 +- **状态过滤**：支持按全部/失效/可抓取/仅API/较慢筛选 +- **搜索功能**：同时搜索友链、文章、作者 + +--- + +## 📊 数据结构变更 + +### 旧结构（v1.x） + +```json +// all.json（混合数据） +{ + "statistical_data": { + "friends_num": 199, + "active_num": 155, + "article_num": 250, + "error_num": 15 + }, + "article_data": [...], + "link_check_data": [...], // ❌ 已移除 + "friend_data": [...] // ❌ 已移除 +} + +// errors.json +[ + ["站点名", "站点地址", "头像地址"], + ... +] +``` + +### 新结构（v2.0） + +```json +// all.json（仅友圈文章） +{ + "statistical_data": { + "friends_num": 199, + "active_num": 155, + "article_num": 250, + "error_num": 15 + }, + "article_data": [...] +} + +// link.json（友链可达性）✨ 新增 +{ + "statistical_data": { + "link_total_num": 199, + "link_reachable_num": 187, + "link_unreachable_num": 12, + "crawl_allowed_num": 167, + "api_only_num": 20, + "has_author_link_num": 131, + "link_last_checked_time": "2026-05-31 19:33:00" + }, + "link_data": [ + { + "name": "清羽飞扬", + "link": "https://blog.liushen.fun/", + "link_page": "https://blog.liushen.fun/link/", + "avatar": "https://blog.liushen.fun/favicon.ico", + "reachable": true, + "crawlable": true, + "method": "direct", + "latency": 0.07, + "fail_count": 0, + "checked_at": "2026-05-31 19:33:00", + "has_backlink": true, + "reason": "allowed_by_direct" + }, + ... + ] +} + +// errors.json（真正不可达的友链） +[ + ["站点名", "站点地址", "头像地址"], + ... +] +``` + +### 字段说明 + +#### link.json 中的 link_data 字段 + +| 字段 | 类型 | 说明 | +|------|------|------| +| `name` | string | 友链名称 | +| `link` | string | 友链地址 | +| `link_page` | string | 友链页地址（用于反链检测） | +| `avatar` | string | 头像地址 | +| `reachable` | boolean | 是否可达 | +| `crawlable` | boolean | 是否可抓取（直连或代理可达） | +| `method` | string | 最佳检测方式：`direct`/`proxy`/`api`/`disabled` | +| `latency` | number | 响应延迟（秒），-1 表示不可达 | +| `fail_count` | number | 连续失败次数 | +| `checked_at` | string | 检测时间 | +| `has_backlink` | boolean\|null | 是否有反链，null 表示未检测 | +| `reason` | string | RSS 抓取决策原因 | + +--- + +## ⚙️ 配置文件变更 + +### 旧配置（v1.x） + +```yaml +spider_settings: + enable: true + json_url: "https://blog.liushen.fun/friend.json" + article_count: 5 + merge_result: # ❌ 已废弃 + enable: false + merge_json_url: "https://fc.liushen.fun" +``` + +### 新配置（v2.0） + +```yaml +spider_settings: + enable: true + json_url: "https://blog.liushen.fun/friend.json" + article_count: 5 + +# ✨ 新增：友链可达性检测配置 +link_check: + enable: true # 是否启用友链可达性检测 + max_age_hours: 24 # 缓存时间（小时） + timeout: 15 # 请求超时时间（秒） + max_workers: 10 # 并发检测数量 + proxy_url: "" # 代理前缀（如 Cloudflare Worker） + # ⚠️ 代理服务可能违反某些服务条款，请谨慎使用 + # 支持环境变量 LINK_CHECK_PROXY_URL 覆盖（优先级更高） + status_api_url: "https://v2.xxapi.cn/api/status?url={url}" # 兜底 API + enable_backlink_check: true # 是否检测反链 + author_url: "blog.liushen.fun" # 你的站点域名 + +# ✨ 新增：数据合并配置（独立出来） +merge_settings: + enable: false # 是否启用数据合并 + remote_base_url: "https://fc.liushen.fun" # 远程数据源基础 URL + merge_article_data: true # 是否合并友圈文章数据 + merge_link_check_data: true # 是否合并友链可达性数据 +``` + +### 配置变更说明 + +#### 1. `merge_result` → `merge_settings` +- **位置变更**：从 `spider_settings.merge_result` 移至顶级 `merge_settings` +- **字段重命名**： + - `merge_json_url` → `remote_base_url` + - 新增 `merge_article_data`（友圈文章合并开关） + - 新增 `merge_link_check_data`（友链可达性合并开关） + +#### 2. 友链数据格式 +- **旧格式（3字段）**：`["站点名", "站点地址", "头像地址"]` +- **新格式（4字段）**：`["站点名", "站点地址", "友链页地址", "头像地址"]` +- **兼容性**：两种格式均支持，自动识别 + +--- + +## 🎨 前端页面变更 + +### 文件变更 + +| 文件 | 状态 | 说明 | +|------|------|------| +| `static/index.html` | ✏️ 重写 | 继承 check-flink 风格，展示友链状态+友圈文章 | +| `static/status.html` | ❌ 删除 | 功能已合并到 index.html | +| `main/fclite.js` | ✅ 保持 | 不变，供外部引用 | +| `main/fclite.css` | ✅ 保持 | 不变，供外部引用 | + +### 新首页特性 + +- **诗词背景**：李白《登金陵凤凰台》 +- **统计卡片**：友链总数、可抓取、文章总数、失效友链 +- **友链状态展示**： + - 状态点动画（绿色=正常，黄色=较慢，蓝色=API，红色=失效） + - 失败次数显示 + - 反链图标（❤️ 有反链 / 💔 无反链） +- **友圈文章展示**：标题固定两行高度，作者+时间 +- **搜索过滤**：同时搜索友链、文章、作者 + +--- + +## 🔄 迁移指南 + +### 对于使用者 + +#### 1. 更新配置文件 + +如果你之前启用了 `merge_result`，需要调整配置： + +```yaml +# 旧配置 +spider_settings: + merge_result: + enable: true + merge_json_url: "https://example.com" + +# 新配置 +merge_settings: + enable: true + remote_base_url: "https://example.com" + merge_article_data: true + merge_link_check_data: true +``` + +#### 2. 更新友链数据格式（可选） + +如果想启用反链检测，建议更新为 4 字段格式： + +```json +{ + "friends": [ + ["清羽飞扬", "https://blog.liushen.fun/", "https://blog.liushen.fun/link/", "https://blog.liushen.fun/favicon.ico"] + ] +} +``` + +#### 3. 更新 GitHub Actions + +如果自定义了 workflow，需要添加 `link.json` 到发布文件： + +```yaml +- name: Build static publish directory + run: | + mkdir pages + cp -r main static all.json link.json errors.json pages/ +``` + +### 对于外部引用者 + +#### 如果你引用了 `all.json` + +✅ **无需修改**，`all.json` 结构保持兼容，仅移除了 `link_check_data` 和 `friend_data` 字段。 + +#### 如果你引用了 `main/fclite.js` 和 `main/fclite.css` + +✅ **无需修改**，这两个文件保持不变。 + +#### 如果你需要友链可达性数据 + +✨ **新增引用** `link.json`： + +```javascript +fetch('/link.json') + .then(response => response.json()) + .then(data => { + const links = data.link_data; + const stats = data.statistical_data; + // 使用友链数据 + }); +``` + +--- + +## ✅ 兼容性说明 + +### 向后兼容 + +- ✅ `all.json` 结构保持兼容（仅移除冗余字段） +- ✅ `errors.json` 结构不变 +- ✅ `main/fclite.js` 和 `main/fclite.css` 不变 +- ✅ 友链数据支持 3 字段和 4 字段格式 +- ✅ 旧版引用者无需修改代码 + +### 不兼容变更 + +- ❌ `all.json` 不再包含 `link_check_data` 和 `friend_data` +- ❌ `spider_settings.merge_result` 配置已废弃 +- ❌ `static/status.html` 已删除 + +### 升级建议 + +1. **最小升级**：仅更新代码，不修改配置 → 友链检测默认启用，但反链检测默认关闭 +2. **推荐升级**：更新配置 + 更新友链数据为 4 字段格式 → 完整体验所有新功能 +3. **完整升级**：更新配置 + 更新友链数据 + 启用数据合并 → 国内外数据智能合并 + +--- + +## 📝 相关链接 + +- [Friend-Circle-Lite 主仓库](https://github.com/willow-god/Friend-Circle-Lite) +- [check-flink 原项目](https://github.com/willow-god/check-flink) +- [详细文档](https://blog.liushen.fun/posts/4dc716ec/) + +--- + +## 🙏 致谢 + +感谢所有使用 Friend-Circle-Lite 的朋友们，以及为项目提供反馈和建议的贡献者！ diff --git a/conf.yaml b/conf.yaml index b26d072a945..317b4661778 100644 --- a/conf.yaml +++ b/conf.yaml @@ -1,66 +1,89 @@ +# 调试开关 +# 说明：默认关闭。开启后，程序结束前会全量打印 SQLite 缓存表结构与所有数据，并保守清理核心表残留旧字段。 +# 支持环境变量 FCL_DEBUG=1 覆盖此配置。 +debug: false + # 爬虫相关配置 -# 解释：使用request实现友链文章爬取，并放置到根目录的all.json下 -# enable: 是否启用爬虫 -# json_url: 请填写对应格式json的地址，仅支持网络地址 -# article_count: 请填写每个博客需要获取的最大文章数量 -# marge_result: 是否合并多个json文件，若为true则会合并指定网络地址和本地地址的json文件 -# enable: 是否启用合并功能，该功能提供与自部署的友链合并功能，可以解决服务器部分国外网站无法访问的问题 -# marge_json_path: 请填写网络地址的json文件，用于合并，不带空格！！！ +# 说明：从友链 JSON 中读取站点列表，抓取各站点 RSS 文章，并生成 all.json。 +# enable: 是否启用爬虫 +# json_url: 友链 JSON 地址，仅支持网络地址 +# article_count: 每个站点最多抓取的文章数量 spider_settings: enable: true json_url: "https://blog.liushen.fun/friend.json" article_count: 5 - merge_result: - enable: false - merge_json_url: "https://fc.liushen.fun" -# 邮箱推送功能配置，暂未实现，等待后续开发 -# 解释：每天为指定邮箱推送所有友链文章的更新，仅能指定一个 -# enable: 是否启用邮箱推送功能 -# to_email: 收件人邮箱地址 -# subject: 邮件主题 -# body_template: 邮件正文的 HTML 模板文件 +# 代理配置 +# 说明：用于友链检测和 RSS 抓取。程序会先直连，请求失败且配置了代理时自动走代理。 +# proxy_url: 代理地址涉及一定违规风险和隐私风险，请尽量不要写入配置文件。 +# 推荐在仓库环境变量 PROXY_URL 中配置，并让此处保持为空。 +# 推荐格式为代理前缀，例如 https://proxy.example.com/ +# 程序会自动拼接为 https://proxy.example.com/https://example.com/feed.xml +# 如需自行搭建代理，可参考：https://blog.liushen.fun/posts/dd89adc9/ +# 兼容高级格式 https://proxy.example.com?url={url}，但普通反代场景不推荐这样写。 +# 留空则不使用代理。 +proxy_settings: + proxy_url: "" + +# 数据合并配置 +# 说明：合并多个数据源的结果，比如国内和国外两条线路各自运行后的 all.json、link.json、errors.json。 +# enable: 是否启用数据合并 +# remote_base_url: 远程数据源基础 URL，会自动拼接 /all.json、/link.json、/errors.json +# merge_article_data: 是否合并友圈文章数据 +# merge_link_check_data: 是否合并友链可达性数据 +merge_settings: + enable: false + remote_base_url: "https://fc.liushen.fun" + merge_article_data: true + merge_link_check_data: true + +# 友链可达性检测配置 +# 说明：友圈抓取依赖此检测结果，因此该检测始终启用。旧配置中的 link_check.enable 会被兼容读取但不再生效。 +# max_age_hours: 同一友链检测结果缓存时间，默认 24 小时 +# timeout: 单次网页请求超时时间 +# max_workers: 并发检测数量 +# status_api_url: 兜底状态码 API；API-only 结果只用于可达性展示，不参与 RSS 抓取 +# enable_backlink_check: 是否检测友链页是否包含你的站点链接 +# author_url: 你的站点域名，用于反链检测，建议只填写域名 +link_check: + max_age_hours: 24 + timeout: 15 + max_workers: 10 + status_api_url: "https://v2.xxapi.cn/api/status?url={url}" + enable_backlink_check: true + author_url: "blog.liushen.fun" + +# 邮件推送功能配置，暂未实现，等待后续开发 +# 说明：每次运行后向指定邮箱推送所有友链文章更新。 email_push: enable: false to_email: recipient@example.com subject: "今天的 RSS 订阅更新" body_template: "rss_template.html" -# 邮箱issue订阅功能配置 -# 解释：向在issue中提取的所有邮箱推送您网站中的更新，添加邮箱和删除邮箱均通过添加issue对应格式实现 -# enable: 是否启用邮箱推送功能 -# github_username: GitHub 用户名，用于构建issue api地址 -# github_repo: GitHub 仓库名，用于构建issue api地址 -# your_blog_url: 你的博客地址 -# website_info: 你的博客信息 -# title: 你的博客标题，如果启用了推送，用于生成邮件主题 +# 邮件 issue 订阅功能配置 +# 说明：从 GitHub issue 中提取订阅邮箱，并推送你自己站点的新文章。 rss_subscribe: enable: true github_username: willow-god github_repo: Friend-Circle-Lite your_blog_url: https://blog.liushen.fun/ - email_template: "./rss_subscribe/email_template.html" + email_template: "./push_templates/default.html" website_info: title: "清羽飞扬" # SMTP 配置 -# 解释：使用其中的相关配置实现上面两种功能，若无推送要求可以不配置，请将以上两个配置置为false -# email: 发件人邮箱地址 -# server： SMTP 服务器地址 -# port： SMTP 端口号 -# use_tls： 是否使用 tls 加密 +# 说明：用于上方邮件相关功能。如果不使用邮件功能，可关闭 email_push 和 rss_subscribe。 smtp: email: notify@liushen.fun server: smtp.exmail.qq.com port: 465 use_tls: true -# 特殊RSS地址指定，可以置空但是不要删除！ -# 解释：用于指定特殊RSS地址，如B站专栏等不常见RSS地址后缀，可以添加多个 -# name: 友链名称 -# url: 指定的RSS地址 +# 特殊 RSS 地址指定 +# 说明：用于指定特殊 RSS 地址，比如 B 站专栏等不常见 RSS 地址后缀。可以置空，但不要删除此项。 specific_RSS: - - name: "阮一峰" - url: "http://feeds.feedburner.com/ruanyifeng" - # - name: "無名小栈" - # url: "https://blog.imsyy.top/rss.xml" \ No newline at end of file + - name: "阮一峰" + url: "http://feeds.feedburner.com/ruanyifeng" + # - name: "无名小栈" + # url: "https://blog.imsyy.top/rss.xml" diff --git a/deploy.sh b/deploy.sh index ac0aa082748..6d3428e1da1 100644 --- a/deploy.sh +++ b/deploy.sh @@ -1,62 +1,16 @@ #!/bin/bash -# 获取当前脚本所在目录 SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -# 定义日志文件路径 -# CRON_LOG_FILE="$SCRIPT_DIR/cron_grab.log" -# API_LOG_FILE="$SCRIPT_DIR/api_grab.log" +cd "$SCRIPT_DIR" || exit 1 -# # 定义要执行的命令 -# COMMAND="python3 $SCRIPT_DIR/run.py" +python3 run.py -# # 定义定时任务的执行间隔（例如每四小时一次） -# INTERVAL="4" - -# 添加定时任务到 crontab -# (crontab -l 2>/dev/null; echo "0 */$INTERVAL * * * $COMMAND >> $CRON_LOG_FILE 2>&1 && echo '运行成功'") | crontab - - -# echo "====================================" -# echo "定时爬取 成功设置，时间间隔：4h" -# echo "定时任务日志：$CRON_LOG_FILE" -# echo "====================================" - - - -#!/bin/bash - -# 获取当前脚本所在目录 -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" - -# 定义 API 服务的启动命令 -API_COMMAND="python3 $SCRIPT_DIR/server.py" +mkdir -p pages +cp -r main static all.json link.json errors.json pages/ echo "====================================" - -# 后台运行服务端，将数据映射到API -echo "****正在启动API服务****" -nohup $API_COMMAND &>/dev/null & -API_PID=$! -sleep 5 # 等待API服务启动，可能需要调整等待时间 - -echo "API 服务已启动：http://localhost:1223" -echo "API 服务进程号：$API_PID" -echo "API 服务关闭命令：kill -9 $API_PID" -echo "文档地址：https://blog.liushen.fun/posts/4dc716ec/" -echo "====================================" - -# 用户选择是否执行爬取 -read -p "选择操作：0 - 退出, 1 - 执行一次爬取: " USER_CHOICE - -if [ "$USER_CHOICE" -eq 1 ]; then - echo "****正在执行一次爬取****" - python3 $SCRIPT_DIR/run.py - echo "****爬取成功****" -else - echo "退出选项被选择，掰掰！" - +echo "静态文件已生成到 pages/ 目录" +echo "请将 pages/ 目录作为静态网站根目录部署" +echo "部署后检查 /all.json 和 /link.json 是否可访问" echo "====================================" -echo "定时抓取的部分请自行设置，如果有宝塔等面板可以按照说明直接添加，如果没有宝塔可以查看本脚本上面屏蔽的部分，自行添加到 crontab 中" -echo "====================================" - -fi diff --git a/docs/superpowers/plans/2026-06-25-unreachable-days.md b/docs/superpowers/plans/2026-06-25-unreachable-days.md new file mode 100644 index 00000000000..684574b7ab7 --- /dev/null +++ b/docs/superpowers/plans/2026-06-25-unreachable-days.md @@ -0,0 +1,60 @@ +# Friend Link Unreachable Days Display Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Show consecutive unreachable days on the friend-link reachability cards, especially for the unreachable filter. + +**Architecture:** Keep the existing JSON shape and derive the display directly from `fail_count`/`fails` in `static/index.html`. Add a small helper that formats the duration once and reuse it in the card's main insight and unreachable detail chips so the UI stays consistent. + +**Tech Stack:** Python tests, static HTML/JavaScript, existing Friend-Circle-Lite JSON output. + +--- + +### Task 1: Lock the expected dashboard copy in a regression test + +**Files:** +- Modify: `tests/test_refactor_contracts.py:44-106` + +- [x] **Step 1: Write the failing test** + +```python + self.assertIn("unreachableDays", html) + self.assertIn("formatUnreachableDays", html) + self.assertIn("不可达", html) +``` + +- [x] **Step 2: Run test to verify it fails** + +Run: `pytest tests/test_refactor_contracts.py -k static_index_is_standalone_dashboard_with_view_switch -v` +Expected: FAIL because the dashboard source does not yet expose unreachable-day formatting. + +- [x] **Step 3: Write minimal implementation** + +Add the formatting helper and wire it into the unreachable card copy in `static/index.html`. + +- [x] **Step 4: Run test to verify it passes** + +Run: `pytest tests/test_refactor_contracts.py -k static_index_is_standalone_dashboard_with_view_switch -v` +Expected: PASS. + +### Task 2: Render unreachable days in the card UI + +**Files:** +- Modify: `static/index.html` + +- [x] **Step 1: Write the failing test** + +Already covered by Task 1. + +- [x] **Step 2: Run test to verify it fails** + +Already covered by Task 1. + +- [x] **Step 3: Write minimal implementation** + +Use `link.fails` as the displayed value for unreachable entries, label it as unreachable duration, and keep the existing RSS / backlink / update facts. + +- [x] **Step 4: Run test to verify it passes** + +Run: `pytest tests/test_refactor_contracts.py -k static_index_is_standalone_dashboard_with_view_switch -v` +Expected: PASS. diff --git a/friend_circle_lite/__init__.py b/friend_circle_lite/__init__.py index e69de29bb2d..0df0993ad48 100644 --- a/friend_circle_lite/__init__.py +++ b/friend_circle_lite/__init__.py @@ -0,0 +1,26 @@ +# 标准化的请求头 +HEADERS_JSON = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/123.0.0.0 Safari/537.36 " + "(Friend-Circle-Lite/1.0; +https://github.com/willow-god/Friend-Circle-Lite)" + ), + "X-Friend-Circle": "1.0" +} + +HEADERS_XML = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/123.0.0.0 Safari/537.36 " + "(Friend-Circle-Lite/1.0; +https://github.com/willow-god/Friend-Circle-Lite)" + ), + "Accept": "application/atom+xml, application/rss+xml, application/xml;q=0.9, */*;q=0.8", + "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", + "Accept-Encoding": "gzip, deflate", + "Connection": "keep-alive", + "X-Friend-Circle": "1.0" +} + +timeout = (10, 15) \ No newline at end of file diff --git a/friend_circle_lite/all_friends.py b/friend_circle_lite/all_friends.py new file mode 100644 index 00000000000..ae84c05c2b0 --- /dev/null +++ b/friend_circle_lite/all_friends.py @@ -0,0 +1,6 @@ +"""Backward-compatible crawl output API. + +New code should import from `friend_circle_lite.outputs.legacy_api`. +""" + +from friend_circle_lite.outputs.legacy_api import * # noqa: F401,F403 diff --git a/friend_circle_lite/app_config.py b/friend_circle_lite/app_config.py new file mode 100644 index 00000000000..d8811970127 --- /dev/null +++ b/friend_circle_lite/app_config.py @@ -0,0 +1,6 @@ +"""Backward-compatible configuration exports. + +New code should import from `friend_circle_lite.config.models`. +""" + +from friend_circle_lite.config.models import * # noqa: F401,F403 diff --git a/friend_circle_lite/application.py b/friend_circle_lite/application.py new file mode 100644 index 00000000000..38a630b36a2 --- /dev/null +++ b/friend_circle_lite/application.py @@ -0,0 +1,6 @@ +"""Backward-compatible application entrypoint exports. + +New code should import from `friend_circle_lite.cli`. +""" + +from friend_circle_lite.cli import FriendCircleLiteApplication # noqa: F401 diff --git a/friend_circle_lite/cache_store.py b/friend_circle_lite/cache_store.py new file mode 100644 index 00000000000..2cfdded6c2d --- /dev/null +++ b/friend_circle_lite/cache_store.py @@ -0,0 +1,6 @@ +"""Backward-compatible storage exports. + +New code should import from `friend_circle_lite.storage.sqlite_store`. +""" + +from friend_circle_lite.storage.sqlite_store import * # noqa: F401,F403 diff --git a/friend_circle_lite/cli.py b/friend_circle_lite/cli.py new file mode 100644 index 00000000000..af47eb9ea5d --- /dev/null +++ b/friend_circle_lite/cli.py @@ -0,0 +1,221 @@ +"""Top-level application orchestration. + +This module keeps the main script very small by moving the end-to-end workflow +into focused orchestration methods. +""" + +from __future__ import annotations + +import logging +import os +import sys + +from friend_circle_lite.config.models import ApplicationConfig, MailRuntime +from friend_circle_lite.config.printer import print_startup_config +from friend_circle_lite.crawler.single_site_legacy import get_latest_articles_from_link +from friend_circle_lite.notifications.github import extract_emails_from_issues +from friend_circle_lite.notifications.mail import send_emails +from friend_circle_lite.outputs.legacy_api import ( + deal_with_large_data, + fetch_and_process_data, + merge_data_from_json_url, + merge_errors_from_json_url, + merge_link_data_from_json_url, +) +from friend_circle_lite.storage.diagnostics import SQLiteDebugDumper +from friend_circle_lite.utils.json import write_json + + +class FriendCircleLiteApplication: + """Application service coordinating crawl and notification workflows.""" + + def __init__(self, config: ApplicationConfig): + self.config = config + + def run(self) -> None: + """Execute the enabled application features in a stable order.""" + try: + print_startup_config(self.config) + self.run_crawler_if_enabled() + mail_runtime = self.prepare_mail_runtime() + self.run_email_push_if_enabled(mail_runtime) + self.run_rss_subscription_if_enabled(mail_runtime) + finally: + self.dump_sqlite_debug_if_enabled() + + def dump_sqlite_debug_if_enabled(self) -> None: + """在 debug 开启时输出 SQLite 全量缓存数据，便于排查 Action 问题。""" + if not self.config.debug: + return + SQLiteDebugDumper(self.config.runtime_paths.cache_file).run() + + def run_crawler_if_enabled(self) -> None: + """Run the article crawl and persist public output files when enabled.""" + spider_settings = self.config.spider_settings + if not spider_settings.enable: + logging.info("[爬虫入口] 爬虫未启用，跳过抓取流程") + return + + logging.info("[爬虫入口] 爬虫已启用") + logging.info( + f"[爬虫入口] 正在从 {spider_settings.json_url} 获取友链原始数据，每站最多 {spider_settings.article_count} 篇文章" + ) + + crawl_result = fetch_and_process_data( + json_url=spider_settings.json_url, + specific_RSS=self.config.specific_rss, + count=spider_settings.article_count, + cache_file=self.config.runtime_paths.cache_file, + link_check_config=self.config.link_check, + proxy_settings=self.config.proxy_settings, + ) + if crawl_result is None: + logging.error("[爬虫入口] 抓取流程失败，未生成任何输出文件") + return + + result, lost_friends, link_payload = crawl_result + result, lost_friends, link_payload = self._merge_remote_results_if_enabled(result, lost_friends, link_payload) + + article_count = len(result.get("article_data", [])) + logging.info(f"[爬虫入口] 数据获取完毕，共有 {article_count} 篇文章，正在处理输出文件") + + result = deal_with_large_data( + result, + future_tolerance_days=self.config.future_article_tolerance_days, + ) + write_json(self.config.runtime_paths.all_json_file, result) + write_json(self.config.runtime_paths.errors_json_file, lost_friends) + write_json(self.config.runtime_paths.link_json_file, link_payload) + + def prepare_mail_runtime(self) -> MailRuntime: + """Build SMTP runtime credentials from config and environment variables.""" + if not (self.config.email_push.enable or self.config.rss_subscribe.enable): + return MailRuntime(sender_email="", smtp_server="", port=0, password="", use_tls=False) + + logging.info("📨 推送功能已启用，正在准备中...") + smtp_conf = self.config.smtp + mail_runtime = MailRuntime( + sender_email=smtp_conf.email, + smtp_server=smtp_conf.server, + port=smtp_conf.port, + password=os.getenv("SMTP_PWD", ""), + use_tls=smtp_conf.use_tls, + ) + + logging.info(f"📡 SMTP 服务器：{mail_runtime.smtp_server}:{mail_runtime.port}") + if mail_runtime.is_ready: + logging.info(f"🔐 密码(部分)：{mail_runtime.password[:3]}*****") + else: + logging.error("❌ SMTP 信息不完整或环境变量 SMTP_PWD 未设置，无法发送邮件") + return mail_runtime + + def run_email_push_if_enabled(self, mail_runtime: MailRuntime) -> None: + """Keep the reserved email push entrypoint behavior unchanged.""" + if self.config.email_push.enable and mail_runtime.is_ready: + logging.info("📧 邮件推送已启用") + logging.info("⚠️ 抱歉，目前尚未实现邮件推送功能") + + def run_rss_subscription_if_enabled(self, mail_runtime: MailRuntime) -> None: + """Send subscription emails for newly discovered posts when enabled.""" + if not self.config.rss_subscribe.enable: + return + if not mail_runtime.is_ready: + logging.info("⏭️ RSS 订阅推送未执行，因为 SMTP 尚未就绪") + return + + logging.info("📰 RSS 订阅推送已启用") + github_username, github_repo = self._resolve_github_repo() + logging.info(f"👤 GitHub 用户名：{github_username}") + logging.info(f"📁 GitHub 仓库：{github_repo}") + + latest_articles = get_latest_articles_from_link( + url=self.config.rss_subscribe.your_blog_url, + count=10, + last_articles_path=self.config.runtime_paths.cache_file, + ) + if not latest_articles: + logging.info("📭 无新文章，无需推送") + return + + logging.info(f"🆕 获取到的最新文章：{latest_articles}") + email_list = self._load_subscriber_emails(github_username, github_repo) + if not email_list: + logging.info("⚠️ 无订阅邮箱，请检查格式或是否有订阅者") + sys.exit(0) + + logging.info(f"📬 获取到邮箱列表：{email_list}") + for article in latest_articles: + template_data = self._build_email_template_data(article, github_username, github_repo) + send_emails( + emails=email_list["emails"], + sender_email=mail_runtime.sender_email, + smtp_server=mail_runtime.smtp_server, + port=mail_runtime.port, + password=mail_runtime.password, + subject=f"{self.config.rss_subscribe.website_info.title} の最新文章：{article['title']}", + body=self._build_plaintext_mail_body(article), + template_path=self.config.rss_subscribe.email_template, + template_data=template_data, + use_tls=mail_runtime.use_tls, + ) + + def _merge_remote_results_if_enabled( + self, result: dict, lost_friends: list[list[str]], link_payload: dict + ) -> tuple[dict, list[list[str]], dict]: + """Merge remote outputs when the merge option is enabled.""" + merge_settings = self.config.merge_settings + if not merge_settings.enable: + return result, lost_friends, link_payload + + remote_url = merge_settings.remote_base_url + logging.info(f"[数据合并] 合并功能开启，从 {remote_url} 获取外部数据") + + if merge_settings.merge_article_data: + result = merge_data_from_json_url(result, f"{remote_url}/all.json") + lost_friends = merge_errors_from_json_url(lost_friends, f"{remote_url}/errors.json") + + if merge_settings.merge_link_check_data: + link_payload = merge_link_data_from_json_url(link_payload, f"{remote_url}/link.json") + + return result, lost_friends, link_payload + + def _resolve_github_repo(self) -> tuple[str, str]: + """Resolve repository coordinates from env override or config.""" + fcl_repo = os.getenv("FCL_REPO") + if fcl_repo: + return tuple(fcl_repo.split("/", 1)) + return self.config.rss_subscribe.github_username, self.config.rss_subscribe.github_repo + + @staticmethod + def _load_subscriber_emails(github_username: str, github_repo: str) -> dict | None: + """Load subscriber emails from GitHub closed issues.""" + github_api_url = ( + f"https://api.github.com/repos/{github_username}/{github_repo}/issues" + f"?state=closed&label=subscribed&per_page=200" + ) + logging.info(f"[订阅邮箱] 正在从 GitHub 获取订阅邮箱：{github_api_url}") + return extract_emails_from_issues(github_api_url) + + def _build_email_template_data(self, article: dict, github_username: str, github_repo: str) -> dict[str, str]: + """Assemble template variables for one outbound notification email.""" + return { + "title": article["title"], + "summary": article["summary"], + "published": article["published"], + "link": article["link"], + "website_title": self.config.rss_subscribe.website_info.title, + "github_issue_url": ( + f"https://github.com/{github_username}/{github_repo}" + "/issues?q=is%3Aissue+is%3Aclosed" + ), + } + + @staticmethod + def _build_plaintext_mail_body(article: dict) -> str: + """Build the plain-text fallback body for one notification email.""" + return ( + f"📄 文章标题：{article['title']}\n" + f"🔗 链接：{article['link']}\n" + f"📝 简介：{article['summary']}\n" + f"🕒 发布时间：{article['published']}" + ) diff --git a/friend_circle_lite/config/__init__.py b/friend_circle_lite/config/__init__.py new file mode 100644 index 00000000000..df04cf33002 --- /dev/null +++ b/friend_circle_lite/config/__init__.py @@ -0,0 +1,3 @@ +"""Configuration loading and typed config models.""" + +from friend_circle_lite.config.models import * # noqa: F401,F403 diff --git a/friend_circle_lite/config/models.py b/friend_circle_lite/config/models.py new file mode 100644 index 00000000000..08f7898e621 --- /dev/null +++ b/friend_circle_lite/config/models.py @@ -0,0 +1,232 @@ +"""Application configuration models. + +This module converts the raw YAML structure into typed configuration objects so +that the rest of the application can depend on explicit fields instead of a +loosely typed nested dictionary. + +The external YAML keys are preserved for backward compatibility. Internally, +snake_case names are used consistently. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, field + + +DEFAULT_CACHE_FILE = "./temp/cache.sqlite3" +DEFAULT_ALL_JSON = "./all.json" +DEFAULT_ERRORS_JSON = "./errors.json" +DEFAULT_LINK_JSON = "./link.json" + + +def _as_bool(value: object, default: bool = False) -> bool: + """稳健解析布尔值，兼容 YAML 布尔与字符串写法。""" + if value is None: + return default + if isinstance(value, bool): + return value + if isinstance(value, str): + return value.strip().lower() in {"1", "true", "yes", "on"} + return bool(value) + + +def _env_flag(name: str) -> bool | None: + """读取布尔环境变量；未配置时返回 None，避免覆盖配置文件。""" + value = os.getenv(name) + if value is None: + return None + return _as_bool(value) + + +@dataclass(slots=True) +class MergeSettings: + """Options for merging local crawl results with remote data sources.""" + + enable: bool = False + remote_base_url: str = "" + merge_article_data: bool = True + merge_link_check_data: bool = True + + +@dataclass(slots=True) +class ProxySettings: + """Proxy configuration for both link checking and RSS crawling.""" + + proxy_url: str = "" + + +@dataclass(slots=True) +class SpiderSettings: + """Crawler settings controlling source list and output density.""" + + enable: bool = True + json_url: str = "" + article_count: int = 5 + + +@dataclass(slots=True) +class LinkCheckConfig: + """Settings for friend link reachability checks.""" + + # 兼容旧配置字段。当前抓取流程依赖可达性检测，因此运行时会始终视为启用。 + enable: bool = True + max_age_hours: int = 24 + timeout: int = 15 + max_workers: int = 10 + status_api_url: str = "https://v2.xxapi.cn/api/status?url={url}" + enable_backlink_check: bool = False + author_url: str = "" + + +@dataclass(slots=True) +class EmailPushConfig: + """Reserved configuration for the not-yet-implemented email push feature.""" + + enable: bool = False + to_email: str = "" + subject: str = "" + body_template: str = "" + + +@dataclass(slots=True) +class WebsiteInfo: + """Display metadata for outbound notifications.""" + + title: str = "" + + +@dataclass(slots=True) +class RssSubscribeConfig: + """Configuration for GitHub issue based email subscriptions.""" + + enable: bool = False + github_username: str = "" + github_repo: str = "" + your_blog_url: str = "" + email_template: str = "" + website_info: WebsiteInfo = field(default_factory=WebsiteInfo) + + +@dataclass(slots=True) +class SmtpConfig: + """SMTP connection settings used by all mail sending features.""" + + email: str = "" + server: str = "" + port: int = 0 + use_tls: bool = True + + +@dataclass(slots=True) +class RuntimePaths: + """Filesystem locations used by the runtime.""" + + cache_file: str = DEFAULT_CACHE_FILE + all_json_file: str = DEFAULT_ALL_JSON + errors_json_file: str = DEFAULT_ERRORS_JSON + link_json_file: str = DEFAULT_LINK_JSON + + +@dataclass(slots=True) +class ApplicationConfig: + """Root application configuration assembled from the YAML file.""" + + spider_settings: SpiderSettings + proxy_settings: ProxySettings + merge_settings: MergeSettings + link_check: LinkCheckConfig + email_push: EmailPushConfig + rss_subscribe: RssSubscribeConfig + smtp: SmtpConfig + specific_rss: list[dict] + runtime_paths: RuntimePaths = field(default_factory=RuntimePaths) + future_article_tolerance_days: int = 2 + debug: bool = False + + @classmethod + def from_dict(cls, data: dict) -> "ApplicationConfig": + """Create a typed config object from the raw YAML dictionary.""" + spider_raw = data.get("spider_settings", {}) + proxy_raw = data.get("proxy_settings", {}) + merge_raw = data.get("merge_settings", {}) + link_check_raw = data.get("link_check", {}) + email_push_raw = data.get("email_push", {}) + rss_subscribe_raw = data.get("rss_subscribe", {}) + website_info_raw = rss_subscribe_raw.get("website_info", {}) + smtp_raw = data.get("smtp", {}) + runtime_raw = data.get("runtime_paths", {}) + debug_from_env = _env_flag("FCL_DEBUG") + debug_enabled = debug_from_env if debug_from_env is not None else _as_bool(data.get("debug"), False) + + return cls( + spider_settings=SpiderSettings( + enable=bool(spider_raw.get("enable", True)), + json_url=str(spider_raw.get("json_url", "")).strip(), + article_count=int(spider_raw.get("article_count", 5)), + ), + proxy_settings=ProxySettings( + proxy_url=os.getenv("PROXY_URL") or str(proxy_raw.get("proxy_url", "")).strip(), + ), + merge_settings=MergeSettings( + enable=bool(merge_raw.get("enable", False)), + remote_base_url=str(merge_raw.get("remote_base_url", "")).strip(), + merge_article_data=bool(merge_raw.get("merge_article_data", True)), + merge_link_check_data=bool(merge_raw.get("merge_link_check_data", True)), + ), + link_check=LinkCheckConfig( + enable=True, + max_age_hours=int(link_check_raw.get("max_age_hours", 24)), + timeout=int(link_check_raw.get("timeout", 15)), + max_workers=int(link_check_raw.get("max_workers", 10)), + status_api_url=str(link_check_raw.get("status_api_url", "https://v2.xxapi.cn/api/status?url={url}")).strip(), + enable_backlink_check=bool(link_check_raw.get("enable_backlink_check", False)), + author_url=str(link_check_raw.get("author_url", "")).strip(), + ), + email_push=EmailPushConfig( + enable=bool(email_push_raw.get("enable", False)), + to_email=str(email_push_raw.get("to_email", "")).strip(), + subject=str(email_push_raw.get("subject", "")).strip(), + body_template=str(email_push_raw.get("body_template", "")).strip(), + ), + rss_subscribe=RssSubscribeConfig( + enable=bool(rss_subscribe_raw.get("enable", False)), + github_username=str(rss_subscribe_raw.get("github_username", "")).strip(), + github_repo=str(rss_subscribe_raw.get("github_repo", "")).strip(), + your_blog_url=str(rss_subscribe_raw.get("your_blog_url", "")).strip(), + email_template=str(rss_subscribe_raw.get("email_template", "")).strip(), + website_info=WebsiteInfo( + title=str(website_info_raw.get("title", "")).strip(), + ), + ), + smtp=SmtpConfig( + email=str(smtp_raw.get("email", "")).strip(), + server=str(smtp_raw.get("server", "")).strip(), + port=int(smtp_raw.get("port", 0) or 0), + use_tls=bool(smtp_raw.get("use_tls", True)), + ), + specific_rss=list(data.get("specific_RSS", []) or []), + runtime_paths=RuntimePaths( + cache_file=str(runtime_raw.get("cache_file", DEFAULT_CACHE_FILE)).strip() or DEFAULT_CACHE_FILE, + all_json_file=str(runtime_raw.get("all_json_file", DEFAULT_ALL_JSON)).strip() or DEFAULT_ALL_JSON, + errors_json_file=str(runtime_raw.get("errors_json_file", DEFAULT_ERRORS_JSON)).strip() or DEFAULT_ERRORS_JSON, + link_json_file=str(runtime_raw.get("link_json_file", DEFAULT_LINK_JSON)).strip() or DEFAULT_LINK_JSON, + ), + debug=debug_enabled, + ) + + +@dataclass(slots=True) +class MailRuntime: + """Runtime SMTP credentials resolved from configuration and environment.""" + + sender_email: str + smtp_server: str + port: int + password: str + use_tls: bool + + @property + def is_ready(self) -> bool: + """Whether enough information is available to send email.""" + return bool(self.sender_email and self.smtp_server and self.port and self.password) diff --git a/friend_circle_lite/config/printer.py b/friend_circle_lite/config/printer.py new file mode 100644 index 00000000000..0a4f0ad4222 --- /dev/null +++ b/friend_circle_lite/config/printer.py @@ -0,0 +1,61 @@ +"""启动时打印关键配置。 + +本文件只负责把当前生效配置输出到日志，方便在 GitHub Action 或本地运行时确认： +- 爬虫数据源与文章数量； +- 代理、友链可达性检测、数据合并参数； +- 邮件与 RSS 订阅开关； +- debug 诊断开关。 +""" + +from __future__ import annotations + +import logging + + +def print_startup_config(config) -> None: + """打印启动配置，避免把配置解析细节散落在主流程中。""" + logging.info("=" * 60) + logging.info("Friend-Circle-Lite 启动配置") + logging.info("=" * 60) + + logging.info("爬虫配置:") + logging.info(f" - 启用状态: {'已启用' if config.spider_settings.enable else '已禁用'}") + if config.spider_settings.enable: + logging.info(f" - 数据源: {config.spider_settings.json_url} ") + logging.info(f" - 每站文章数: {config.spider_settings.article_count}") + + logging.info("代理配置:") + if config.proxy_settings.proxy_url: + logging.info(" - 代理状态: 已配置（日志不显示具体地址）") + logging.info(" - 建议: 使用仓库环境变量 PROXY_URL 覆盖，避免代理地址出现在配置文件中") + logging.info(" - 用途: 友链检测 + RSS 抓取") + else: + logging.info(" - 代理状态: 未配置") + + logging.info("友链检测配置:") + logging.info(" - 启用状态: 始终启用（友圈抓取依赖此检测结果）") + logging.info(f" - 缓存时间: {config.link_check.max_age_hours} 小时") + logging.info(f" - 超时时间: {config.link_check.timeout} 秒") + logging.info(f" - 并发数: {config.link_check.max_workers}") + logging.info(f" - 状态 API: {config.link_check.status_api_url} ") + logging.info(f" - 反链检测: {'已启用' if config.link_check.enable_backlink_check else '已禁用'}") + if config.link_check.enable_backlink_check: + logging.info(f" - 站点域名: {config.link_check.author_url} ") + + logging.info("数据合并配置:") + logging.info(f" - 启用状态: {'已启用' if config.merge_settings.enable else '已禁用'}") + if config.merge_settings.enable: + logging.info(f" - 远程数据源: {config.merge_settings.remote_base_url} ") + logging.info(f" - 合并文章数据: {'是' if config.merge_settings.merge_article_data else '否'}") + logging.info(f" - 合并友链数据: {'是' if config.merge_settings.merge_link_check_data else '否'}") + + logging.info("邮件推送配置:") + logging.info(f" - 启用状态: {'已启用' if config.email_push.enable else '已禁用'}") + + logging.info("RSS 订阅配置:") + logging.info(f" - 启用状态: {'已启用' if config.rss_subscribe.enable else '已禁用'}") + + logging.info("调试配置:") + logging.info(f" - SQLite 全量输出: {'已启用' if config.debug else '已禁用'}") + + logging.info("=" * 60) diff --git a/friend_circle_lite/config_printer.py b/friend_circle_lite/config_printer.py new file mode 100644 index 00000000000..c5f367d5ff8 --- /dev/null +++ b/friend_circle_lite/config_printer.py @@ -0,0 +1,6 @@ +"""Backward-compatible startup config printer export. + +New code should import from `friend_circle_lite.config.printer`. +""" + +from friend_circle_lite.config.printer import print_startup_config # noqa: F401 diff --git a/friend_circle_lite/crawler/__init__.py b/friend_circle_lite/crawler/__init__.py new file mode 100644 index 00000000000..be66f0adf1d --- /dev/null +++ b/friend_circle_lite/crawler/__init__.py @@ -0,0 +1,4 @@ +"""Crawler services for feed discovery, parsing, and aggregation.""" + +from friend_circle_lite.crawler.feed_service import FeedDiscoveryService, FeedParserService, LatestArticleTracker +from friend_circle_lite.crawler.service import FeedResolver, FriendCircleCrawlService, SingleSiteCrawler diff --git a/friend_circle_lite/crawler/feed_service.py b/friend_circle_lite/crawler/feed_service.py new file mode 100644 index 00000000000..c41e9a9a189 --- /dev/null +++ b/friend_circle_lite/crawler/feed_service.py @@ -0,0 +1,264 @@ +"""RSS 发现、解析与文章更新追踪服务。""" + +from __future__ import annotations + +import json +import logging +from datetime import datetime +from pathlib import Path +from urllib.parse import urlparse + +import feedparser +import requests + +from friend_circle_lite import HEADERS_XML, timeout +from friend_circle_lite.config.models import ProxySettings +from friend_circle_lite.crawler.http_client import WebFetchClient +from friend_circle_lite.domain.models import Article, FeedEndpoint, Website, normalize_latency +from friend_circle_lite.utils.time import format_published_time +from friend_circle_lite.utils.url import replace_non_domain + + +class FeedDiscoveryService: + """Discover an RSS or Atom endpoint for a website.""" + + POSSIBLE_FEEDS = [ + ("rss1", "/feed"), # WordPress / 最常见 + ("rss2", "/feed/"), # WordPress 兼容写法 + ("rss3", "/rss.xml"), # 很多传统站点 + ("rss4", "/atom.xml"), # 静态博客常见（Hugo / Jekyll） + ("rss5", "/feed.xml"), # 通用型 + ("rss6", "/index.xml"), # Hugo / 一些静态站 + ("rss7", "/feed.atom"), # Atom 明确路径 + ("rss8", "/rss2.xml"), # 老系统遗留 + ("rss9", "/rss/feed.xml"),# 少见但存在 + ("rss10", "/rss.php"), # 老 PHP 程序 + ("rss11", "/feed.php"), # 同上 + ] + + def __init__(self, session: requests.Session, proxy_settings: ProxySettings | None = None): + self.session = session + self.fetcher = WebFetchClient(session, proxy_settings) + self.last_latency = 0.01 + + def discover(self, website_url: str) -> FeedEndpoint | None: + """Try common feed endpoints and return the first valid match.""" + for feed_type, path in self.POSSIBLE_FEEDS: + feed_url = website_url.rstrip("/") + path + result = self.fetcher.get(feed_url, headers=HEADERS_XML, timeout=timeout, desc="RSS 探测") + response = result.response + + if response is None or response.status_code != 200: + continue + + content_type = response.headers.get("Content-Type", "").lower() + if "xml" in content_type or "rss" in content_type or "atom" in content_type: + return FeedEndpoint(url=feed_url, feed_type=feed_type, source="auto") + + text_head = response.text[:1000].lower() + if " list[Article]: + """Parse a feed URL and return the newest `count` articles. + + The returned articles are normalized to the project's internal domain + model, while preserving the original public output fields. + """ + try: + result = self.fetcher.get(feed_url, headers=HEADERS_XML, timeout=timeout, desc="RSS 抓取") + self.last_latency = normalize_latency(result.latency) + if result.response is None: + return [] + response = result.response + # 强制使用 UTF-8 编码，因为 apparent_encoding 可能检测错误 + response.encoding = "utf-8" + feed = feedparser.parse(response.text) + except Exception as exc: + logging.error(f"[RSS 抓取] 解析 RSS 失败：{feed_url}，错误: {exc}") + return [] + + default_author = feed.feed.author if "author" in feed.feed else "" + articles: list[Article] = [] + + for entry in feed.entries: + published = self._extract_published_time(entry) + article_link = replace_non_domain(entry.link, blog_url) if "link" in entry else "" + article = Article( + title=entry.title if "title" in entry else "", + author=default_author, + link=article_link, + published=published, + summary=entry.summary if "summary" in entry else "", + content=entry.content[0].value if "content" in entry and entry.content else entry.description if "description" in entry else "", + ) + articles.append(article) + + valid_articles = [article for article in articles if article.published] + + # 过滤掉无法解析的日期格式 + def safe_parse_date(article): + try: + return datetime.strptime(article.published, "%Y-%m-%d %H:%M") + except ValueError: + logging.warning(f"[RSS 抓取] 文章 {article.title} 的发布时间格式异常: {article.published}，已跳过") + return None + + # 只保留能成功解析日期的文章 + valid_articles_with_dates = [] + for article in valid_articles: + parsed_date = safe_parse_date(article) + if parsed_date: + valid_articles_with_dates.append((article, parsed_date)) + + # 按日期排序 + valid_articles_with_dates.sort(key=lambda item: item[1], reverse=True) + sorted_articles = [item[0] for item in valid_articles_with_dates] + + return sorted_articles[:count] if count < len(sorted_articles) else sorted_articles + + @staticmethod + def _extract_published_time(entry) -> str: + """Extract a normalized publish time from a feed entry.""" + import time + + def convert_time_to_string(time_value): + """Convert various time formats to string.""" + if isinstance(time_value, str): + return time_value + elif isinstance(time_value, time.struct_time): + # 检查年份是否异常 + if time_value.tm_year < 1900: + logging.warning(f"[RSS 抓取] 文章 {entry.get('title', 'Unknown')} 的时间年份异常: {time_value.tm_year}，已跳过") + return "" + return time.strftime('%Y-%m-%dT%H:%M:%SZ', time_value) + else: + logging.warning(f"[RSS 抓取] 文章 {entry.get('title', 'Unknown')} 的时间格式未知: {type(time_value)}，已跳过") + return "" + + if "published" in entry: + time_str = convert_time_to_string(entry.published) + if not time_str: + return "" + return format_published_time(time_str) + if "updated" in entry: + time_str = convert_time_to_string(entry.updated) + if not time_str: + return "" + published = format_published_time(time_str) + logging.warning(f"[RSS 抓取] 文章 {entry.title} 未包含发布时间，已使用更新时间 {published}") + return published + + logging.warning(f"[RSS 抓取] 文章 {entry.title} 未包含任何时间信息，请检查原文，跳过该文章") + return "" + + +class LatestArticleTracker: + """Track whether a website published new posts since the last crawl.""" + + def __init__(self, storage_path: str | Path, max_tracked_articles: int = 10): + from friend_circle_lite.storage.sqlite_store import ArticleTrackingStore + self.store = ArticleTrackingStore(storage_path, max_tracked_articles) + + def diff_and_persist(self, latest_articles: list[Article]) -> list[dict] | None: + """Return newly seen articles and update the local storage. + + Returns None if: + - This is the first run (no previous data exists) + - No new articles are found + - New articles exist but are not newer than the most recent tracked article + """ + previous_articles = self.store.load_articles() + + # First run: no previous data exists, skip sending to prevent sending old articles + if not previous_articles: + logging.info("[文章追踪] 首次运行：跳过推送以防止发送旧文章") + self.store.save_articles(latest_articles) + return None + + previous_latest_date = self._get_latest_date(previous_articles) + + # Find articles that are truly new (check only: link, title, published) + new_articles = [] + for article in latest_articles: + if self._is_truly_new_article(article, previous_articles): + new_articles.append(article) + + if not new_articles: + self.store.save_articles(latest_articles) + return None + + # Filter new articles: only keep those newer than the previous latest date + truly_new_articles = [] + for article in new_articles: + if not article.published: + continue + try: + article_date = datetime.strptime(article.published, "%Y-%m-%d %H:%M") + if previous_latest_date is None or article_date > previous_latest_date: + truly_new_articles.append(article) + except Exception as exc: + logging.warning(f"[文章追踪] 解析文章日期失败: {article.title}, 日期: {article.published}, 错误: {exc}") + continue + + self.store.save_articles(latest_articles) + + if truly_new_articles: + logging.info(f"[文章追踪] 发现 {len(truly_new_articles)} 篇新文章（日期比之前更新）") + return [article.to_tracking_dict() for article in truly_new_articles] + else: + logging.info(f"[文章追踪] 发现 {len(new_articles)} 篇新文章，但日期不够新，跳过推送") + return None + + @staticmethod + def _is_truly_new_article(article: Article, previous_articles: list[Article]) -> bool: + """Check if an article is truly new by comparing link, title, and published date. + + An article is considered new only if its link, title, and published date + do not match any previous article (empty values are skipped). + """ + for prev in previous_articles: + # Check link, title, and published: if any non-empty field matches, it's not new + if article.link and article.link == prev.link: + return False + if article.title and article.title == prev.title: + return False + if article.published and article.published == prev.published: + return False + + return True + + @staticmethod + def _get_latest_date(articles: list[Article]) -> datetime | None: + """Find the latest publish date from a list of articles.""" + latest_date = None + for article in articles: + if not article.published: + continue + try: + article_date = datetime.strptime(article.published, "%Y-%m-%d %H:%M") + if latest_date is None or article_date > latest_date: + latest_date = article_date + except Exception: + continue + return latest_date + + +def extract_blog_origin(url: str) -> str: + """Return a normalized origin for display or author profile links.""" + parsed = urlparse(url) + if not parsed.scheme or not parsed.netloc: + return url + return f"{parsed.scheme}://{parsed.netloc}" diff --git a/friend_circle_lite/crawler/http_client.py b/friend_circle_lite/crawler/http_client.py new file mode 100644 index 00000000000..fe903f23e7e --- /dev/null +++ b/friend_circle_lite/crawler/http_client.py @@ -0,0 +1,97 @@ +"""统一网页请求封装。 + +本模块负责把“直连优先，失败后自动尝试代理”的请求逻辑收口到一个地方。 +调用方只关心是否拿到响应，不需要感知重试细节。 +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass + +import requests + +from friend_circle_lite.config.models import ProxySettings +from friend_circle_lite.domain.models import normalize_latency + + +@dataclass(slots=True) +class FetchResult: + """一次网页请求的结果。""" + + response: requests.Response | None + latency: float = -1 + used_proxy: bool = False + + @property + def success(self) -> bool: + """是否成功取得 HTTP 200 响应。""" + return self.response is not None and self.response.status_code == 200 + + +class WebFetchClient: + """网页请求客户端，封装直连和代理回退逻辑。""" + + def __init__(self, session: requests.Session, proxy_settings: ProxySettings | None = None): + self.session = session + self.proxy_settings = proxy_settings or ProxySettings() + + def get( + self, + url: str, + headers: dict[str, str] | None = None, + timeout: int | tuple | None = None, + desc: str = "网页请求", + ) -> FetchResult: + """先直连请求，失败时自动尝试代理请求。""" + direct = self._get_once(url, headers=headers, timeout=timeout, desc=desc, used_proxy=False) + if direct.success or not self.proxy_settings.proxy_url: + return direct + + proxy_url = self._build_proxy_url(url) + proxy = self._get_once( + proxy_url, + headers=headers, + timeout=timeout, + desc=f"{desc} 代理", + used_proxy=True, + display_url=f"{url} （通过代理）", + ) + return proxy if proxy.success else direct + + def _get_once( + self, + url: str, + headers: dict[str, str] | None, + timeout: int | tuple | None, + desc: str, + used_proxy: bool, + display_url: str | None = None, + ) -> FetchResult: + log_url = display_url or url + start_time = time.time() + try: + response = self.session.get(url, headers=headers, timeout=timeout) + latency = self._elapsed_latency(start_time) + if response.status_code == 200: + logging.info(f"[{desc}] 成功访问: {log_url} ，延迟 {latency} 秒") + else: + logging.warning(f"[{desc}] 状态码异常: {log_url} -> {response.status_code}") + return FetchResult(response=response, latency=latency, used_proxy=used_proxy) + except requests.RequestException as exc: + error_text = exc.__class__.__name__ if used_proxy else str(exc) + logging.warning(f"[{desc}] 请求失败: {log_url} ，错误: {error_text}") + return FetchResult(response=None, latency=self._elapsed_latency(start_time), used_proxy=used_proxy) + + def _build_proxy_url(self, url: str) -> str: + proxy_url = self.proxy_settings.proxy_url + if "{}" in proxy_url: + return proxy_url.format(url) + if "{url}" in proxy_url: + return proxy_url.format(url=url) + return f"{proxy_url}{url}" + + @staticmethod + def _elapsed_latency(start_time: float) -> float: + return normalize_latency(time.time() - start_time) diff --git a/friend_circle_lite/crawler/service.py b/friend_circle_lite/crawler/service.py new file mode 100644 index 00000000000..aa65167832f --- /dev/null +++ b/friend_circle_lite/crawler/service.py @@ -0,0 +1,399 @@ +"""High-level crawler orchestration. + +This module contains the system-level services that coordinate website loading, +RSS discovery, parsing, cache updates, result aggregation, and legacy output +formatting. +""" + +from __future__ import annotations + +import logging +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo + +import requests + +from friend_circle_lite import HEADERS_JSON, timeout +from friend_circle_lite.config.models import LinkCheckConfig, ProxySettings +from friend_circle_lite.crawler.feed_service import FeedDiscoveryService, FeedParserService +from friend_circle_lite.domain.models import Article, CacheRecord, CacheUpdate, CrawlResult, CrawlStatistics, FeedEndpoint, LinkCheckRecord, Website +from friend_circle_lite.link_checker.service import LinkReachabilityService +from friend_circle_lite.storage.sqlite_store import FeedCacheStore, LinkCheckStore + + +class FeedResolver: + """Resolve which feed endpoint should be used for a website. + + Resolution order is kept compatible with the previous implementation: + manual configuration first, then cache, and finally automatic discovery. + """ + + def __init__(self, discovery_service: FeedDiscoveryService, configured_feeds: list[CacheRecord]): + self.discovery_service = discovery_service + self.feed_lookup = {item.name: item for item in configured_feeds} + + def resolve(self, website: Website) -> FeedEndpoint | None: + configured = self.feed_lookup.get(website.name) + if configured: + if configured.source == 'manual': + logging.info(f"[RSS 解析] {website.name} 使用预设 RSS 源：{configured.url}") + elif configured.source == 'cache': + logging.info(f"[RSS 解析] {website.name} 使用缓存 RSS 源：{configured.url}") + else: + logging.info(f"[RSS 解析] {website.name} 使用 RSS 源：{configured.url} ，来源: {configured.source}") + return FeedEndpoint(url=configured.url, feed_type="specific", source=configured.source) + + discovered = self.discovery_service.discover(website.url) + if discovered: + logging.info(f"[RSS 探测] {website.name} 自动探测到 RSS：{discovered.url}") + return discovered + + +class SingleSiteCrawler: + """Crawl one website and produce a normalized result.""" + + def __init__(self, parser_service: FeedParserService, resolver: FeedResolver): + self.parser_service = parser_service + self.resolver = resolver + + def crawl(self, website: Website, count: int) -> CrawlResult: + """Crawl one website while preserving legacy cache repair behavior.""" + endpoint = self.resolver.resolve(website) + cache_update = CacheUpdate(action="none", name=website.name) + + if endpoint and endpoint.source == "auto": + cache_update = CacheUpdate(action="set", name=website.name, url=endpoint.url, reason="auto_discovered") + + articles = self._parse_articles(endpoint, website, count) + parse_error = endpoint is not None and not articles + + if parse_error and endpoint and endpoint.source in ("cache", "unknown"): + logging.warning(f"[RSS 抓取] {website.name} 缓存 RSS 本次抓取失败，将等待下次友链检测刷新 RSS 缓存") + + status = "active" if articles else "error" + if not articles: + if endpoint is None: + logging.warning(f"[RSS 抓取] {website.name} 的博客 {website.url} 未找到有效 RSS") + else: + logging.warning(f"[RSS 抓取] {website.name} 的 RSS {endpoint.url} 未解析出文章") + + return CrawlResult( + website=website, + status=status, + articles=articles, + feed_url=endpoint.url if endpoint else None, + feed_type=endpoint.feed_type if endpoint else "none", + source_used=endpoint.source if endpoint else "none", + cache_update=cache_update, + ) + + def _parse_articles(self, endpoint: FeedEndpoint | None, website: Website, count: int) -> list[Article]: + if endpoint is None: + return [] + + articles = self.parser_service.parse(endpoint.url, count=count, blog_url=website.url) + for article in articles: + article.author = website.name + article.avatar = website.avatar + logging.info(f"[RSS 抓取] {website.name} 发布了新文章：{article.title}，时间：{article.published}，链接：{article.link}") + return articles + + +class FriendCircleCrawlService: + """System-level orchestrator for crawling all configured websites.""" + + def __init__( + self, + json_url: str, + count: int, + specific_rss: list[dict] | None = None, + cache_file: str | None = None, + link_check_config: LinkCheckConfig | None = None, + proxy_settings: ProxySettings | None = None, + ): + self.json_url = json_url + self.count = count + self.specific_rss = specific_rss or [] + self.cache_store = FeedCacheStore(cache_file) + self.link_check_config = link_check_config or LinkCheckConfig() + self.proxy_settings = proxy_settings or ProxySettings() + self.link_check_store = LinkCheckStore(cache_file) + + def run(self) -> tuple[dict, list[list[str]]] | None: + """Fetch website list, crawl all websites, and build public outputs.""" + session = requests.Session() + websites = self._load_websites(session) + if websites is None: + return None + + cache_records = self.cache_store.load_records() + manual_records = self._build_manual_records() + merged_records = self._merge_feed_records(cache_records, manual_records) + manual_names = {record.name for record in manual_records} + + link_check_records = self._check_links(websites, merged_records, manual_names) + link_check_map = {record.url: record for record in link_check_records} + + cache_records = self.cache_store.load_records() + merged_records = self._merge_feed_records(cache_records, manual_records) + feed_names = {record.name for record in merged_records} + crawlable_websites = [ + website for website in websites + if link_check_map.get(website.url, LinkCheckRecord.unchecked(website)).crawl_allowed + and website.name in feed_names + ] + skipped_count = len(websites) - len(crawlable_websites) + logging.info( + f"[朋友圈抓取] 友链总数 {len(websites)} 个，可进入 RSS 抓取 {len(crawlable_websites)} 个，" + f"跳过 {skipped_count} 个不可抓取或无 RSS 缓存站点" + ) + + discovery_service = FeedDiscoveryService(session, self.proxy_settings) + parser_service = FeedParserService(session, self.proxy_settings) + resolver = FeedResolver(discovery_service=discovery_service, configured_feeds=merged_records) + crawler = SingleSiteCrawler(parser_service=parser_service, resolver=resolver) + + crawl_results: list[CrawlResult] = [] + logging.info(f"[朋友圈抓取] 开始抓取 {len(crawlable_websites)} 个可抓取站点，每站最多 {self.count} 篇文章") + with ThreadPoolExecutor(max_workers=10) as executor: + future_to_website = { + executor.submit(crawler.crawl, website, self.count): website + for website in crawlable_websites + } + for future in as_completed(future_to_website): + website = future_to_website[future] + try: + crawl_results.append(future.result()) + except Exception as exc: + logging.error(f"[朋友圈抓取] 处理 {website.to_error_payload()} 时发生错误: {exc}", exc_info=True) + crawl_results.append(CrawlResult(website=website, status="error")) + + self._apply_cache_updates(cache_records, crawl_results, manual_names) + + active_results = [result for result in crawl_results if result.status == "active"] + unreachable_results = [record for record in link_check_records if not record.reachable] + crawl_error_results = [result.website.to_error_payload() for result in crawl_results if result.status != "active"] + error_results = [[record.name, record.url, record.avatar] for record in unreachable_results] + all_articles = [article.to_public_dict() for result in active_results for article in result.articles] + + statistics = CrawlStatistics.create( + friends_num=len(websites), + active_num=len(active_results), + error_num=len(websites) - len(active_results), + article_num=len(all_articles), + ) + stats_payload = statistics.to_dict() + result = { + "statistical_data": stats_payload, + "article_data": all_articles, + } + link_payload = self._build_link_payload(link_check_records) + logging.info( + f"[数据汇总] 处理完成：友链总数 {len(websites)} 个，成功抓取文章站点 {len(active_results)} 个，" + f"RSS 抓取失败 {len(crawl_error_results)} 个，友链不可达 {len(unreachable_results)} 个。" + ) + return result, error_results, link_payload + + def _check_links(self, websites: list[Website], feed_records: list[CacheRecord], manual_names: set[str]) -> list[LinkCheckRecord]: + service = LinkReachabilityService( + config=self.link_check_config, + proxy_settings=self.proxy_settings, + store=self.link_check_store, + feed_records=feed_records, + ) + records = service.check_websites(websites) + if service.feed_updates: + self._apply_feed_updates_from_link_check(service.feed_updates, manual_names) + return records + + def _apply_feed_updates_from_link_check(self, updates: dict[str, CacheRecord | None], manual_names: set[str]) -> None: + """保存可达性检测阶段发现或失效的 RSS 缓存。""" + cache_map = {record.name: record for record in self.cache_store.load_records()} + changed = False + for name, record in updates.items(): + if name in manual_names: + continue + if record is None: + if name in cache_map: + cache_map.pop(name) + changed = True + logging.info(f"[RSS 缓存] 可达性检测删除失效 RSS 缓存: {name}") + else: + cache_map[name] = record + changed = True + logging.info(f"[RSS 缓存] 可达性检测保存 RSS 缓存: {name} -> {record.url}") + if changed: + self.cache_store.save_records(list(cache_map.values())) + + @staticmethod + def _build_link_statistics(records: list[LinkCheckRecord]) -> dict[str, int | str]: + reachable = [record for record in records if record.reachable] + crawl_allowed = [record for record in records if record.crawl_allowed] + api_only = [record for record in records if record.best_method == "api"] + has_author_link = [record for record in records if record.has_author_link] + checked_times = [record.checked_at for record in records if record.checked_at] + return { + "link_total_num": len(records), + "link_reachable_num": len(reachable), + "link_unreachable_num": len(records) - len(reachable), + "crawl_allowed_num": len(crawl_allowed), + "api_only_num": len(api_only), + "has_author_link_num": len(has_author_link), + "link_last_checked_time": max(checked_times) if checked_times else "", + } + + @staticmethod + def _build_link_payload(records: list[LinkCheckRecord]) -> dict[str, object]: + return { + "statistical_data": FriendCircleCrawlService._build_link_statistics(records), + "link_data": [record.to_link_dict() for record in records], + } + + @staticmethod + def _build_friend_data( + websites: list[Website], + crawl_results: list[CrawlResult], + link_check_map: dict[str, LinkCheckRecord], + ) -> list[dict[str, object]]: + crawl_result_map = {result.website.url: result for result in crawl_results} + friend_data: list[dict[str, object]] = [] + for website in websites: + link_record = link_check_map.get(website.url) or LinkCheckRecord.unchecked(website) + crawl_result = crawl_result_map.get(website.url) + friend_data.append({ + "name": website.name, + "url": website.url, + "avatar": website.avatar, + "linkpage": website.linkpage, + "reachable": link_record.reachable, + "crawl_allowed": link_record.crawl_allowed, + "best_method": link_record.best_method, + "best_latency": link_record.best_latency, + "unreachable_since": link_record.unreachable_since, + "unreachable_days": link_record.unreachable_days, + "backlink_checked": link_record.backlink_checked, + "has_author_link": link_record.has_author_link, + "rss_crawl_reason": link_record.rss_crawl_reason, + "feed_status": crawl_result.status if crawl_result else "skipped", + "feed_url": crawl_result.feed_url if crawl_result else None, + "feed_type": crawl_result.feed_type if crawl_result else "none", + "article_count": len(crawl_result.articles) if crawl_result else 0, + }) + return friend_data + + def _load_websites(self, session: requests.Session) -> list[Website] | None: + try: + response = session.get(self.json_url, headers=HEADERS_JSON, timeout=timeout) + response.raise_for_status() + friends_data = response.json() + except Exception as exc: + logging.error(f"无法获取链接：{self.json_url} ：{exc}", exc_info=True) + return None + + website_map: dict[str, Website] = {} + for friend in friends_data.get("friends", []): + try: + website = Website.from_friend_item(friend) + website_map[website.url] = website + except Exception: + logging.warning(f"发现格式异常的友链数据，已跳过: {friend!r}") + return list(website_map.values()) + + def _build_manual_records(self) -> list[CacheRecord]: + manual_records: list[CacheRecord] = [] + for item in self.specific_rss: + if isinstance(item, dict) and item.get("name") and item.get("url"): + manual_records.append(CacheRecord(name=item["name"], url=item["url"], source="manual")) + return manual_records + + @staticmethod + def _merge_feed_records(cache_records: list[CacheRecord], manual_records: list[CacheRecord]) -> list[CacheRecord]: + merged = {record.name: record for record in cache_records} + for record in manual_records: + merged[record.name] = record + return list(merged.values()) + + def _apply_cache_updates(self, cache_records: list[CacheRecord], crawl_results: list[CrawlResult], manual_names: set[str]) -> None: + cache_map = {record.name: record for record in cache_records} + unique_updates: dict[str, CacheUpdate] = {} + + for result in crawl_results: + update = result.cache_update + if not update.name or update.action == "none" or update.name in manual_names: + continue + if update.action == "set" and update.url: + unique_updates[update.name] = update + elif update.action == "delete": + unique_updates[update.name] = update + + for name, update in unique_updates.items(): + if update.action == "set" and update.url: + cache_map[name] = CacheRecord(name=name, url=update.url, source="cache") + if update.reason == "auto_discovered": + logging.info(f"💾 缓存新增：{name} -> {update.url} (自动探测)") + elif update.reason == "repair_cache": + logging.info(f"💾 缓存修复：{name} -> {update.url} (重新探测)") + else: + logging.info(f"💾 缓存更新：{name} -> {update.url} ({update.reason})") + elif update.action == "delete" and name in cache_map: + cache_map.pop(name) + logging.info(f"🗑️ 缓存删除：{name} (RSS 源失效)") + + self.cache_store.save_records(list(cache_map.values())) + + +def sort_articles_by_time(data: dict, future_tolerance_days: int = 2) -> dict: + """Sort article payloads by time and remove far-future timestamps.""" + for article in data.get("article_data", []): + if not article.get("created"): + article["created"] = "2024-01-01 00:00" + logging.warning(f"[数据处理] 文章 {article['title']} 未包含时间信息，已设置为默认时间 2024-01-01 00:00") + + now = datetime.now(ZoneInfo("Asia/Shanghai")).replace(tzinfo=None) + max_allowed_time = now + timedelta(days=future_tolerance_days) + filtered_articles = [] + removed_count = 0 + + for article in data.get("article_data", []): + article_time = datetime.strptime(article["created"], "%Y-%m-%d %H:%M") + if article_time > max_allowed_time: + removed_count += 1 + logging.warning( + f"[数据处理] 文章 {article['title']} 的时间 {article['created']} 超出当前时间 {future_tolerance_days} 天以上，已跳过显示" + ) + continue + filtered_articles.append(article) + + filtered_articles.sort(key=lambda item: datetime.strptime(item["created"], "%Y-%m-%d %H:%M"), reverse=True) + data["article_data"] = filtered_articles + if removed_count: + logging.info(f"[数据处理] 已过滤 {removed_count} 篇未来时间异常的文章") + return data + + +def limit_large_dataset(result: dict, future_tolerance_days: int = 2) -> dict: + """Keep the existing data trimming strategy for very large datasets.""" + result = sort_articles_by_time(result, future_tolerance_days=future_tolerance_days) + article_data = result.get("article_data", []) + result["statistical_data"]["article_num"] = len(article_data) + + max_articles = 150 + if len(article_data) > max_articles: + logging.info(f"[数据处理] 数据量较大，开始裁剪，当前 {len(article_data)} 篇，基础保留 {max_articles} 篇") + top_authors = {article["author"] for article in article_data[:max_articles]} + filtered_articles = article_data[:max_articles] + [ + article for article in article_data[max_articles:] + if article["author"] in top_authors + ] + result["article_data"] = filtered_articles + result["statistical_data"]["article_num"] = len(filtered_articles) + logging.info(f"[数据处理] 数据裁剪完成，保留 {len(filtered_articles)} 篇文章") + + return result + + +# Backward-compatible class names kept for legacy imports. +WebsiteFeedResolver = FeedResolver +WebsiteCrawler = SingleSiteCrawler +FriendCircleCrawler = FriendCircleCrawlService diff --git a/friend_circle_lite/crawler/single_site_legacy.py b/friend_circle_lite/crawler/single_site_legacy.py new file mode 100644 index 00000000000..91e7765be05 --- /dev/null +++ b/friend_circle_lite/crawler/single_site_legacy.py @@ -0,0 +1,141 @@ +"""Legacy-compatible single website helpers. + +The project now uses dedicated domain models and services, but these helpers are +kept as a compatibility layer because existing entrypoints still import them. +""" + +from __future__ import annotations + +import logging + +import requests + +from friend_circle_lite.crawler.feed_service import FeedDiscoveryService, FeedParserService, LatestArticleTracker +from friend_circle_lite.domain.models import CacheUpdate, Website + +def check_feed(blog_url, session): + """Return the discovered feed type and URL in the historical tuple format.""" + endpoint = FeedDiscoveryService(session).discover(blog_url) + if endpoint is None: + return ["none", blog_url] + return [endpoint.feed_type, endpoint.url] + +def parse_feed(url, session, count=5, blog_url=''): + """Parse a feed and return the historical dictionary structure.""" + articles = FeedParserService(session).parse(url, count=count, blog_url=blog_url) + return { + 'website_name': '', + 'author': articles[0].author if articles else '', + 'link': '', + 'articles': [article.to_tracking_dict() for article in articles], + } + +def process_friend(friend, session: requests.Session, count: int, specific_and_cache=None): + """Crawl one friend entry and return the historical result shape.""" + if specific_and_cache is None: + specific_and_cache = [] + + try: + website = Website.from_friend_item(friend) + except Exception: + logging.error(f"friend 数据格式不正确: {friend!r}") + return { + 'name': None, + 'status': 'error', + 'articles': [], + 'feed_url': None, + 'feed_type': 'none', + 'cache_update': CacheUpdate(action='none', name=None, url=None, reason='bad_friend_data').to_dict(), + 'source_used': 'none', + } + + rss_lookup = {entry['name']: entry for entry in specific_and_cache if entry.get('name') and entry.get('url')} + entry = rss_lookup.get(website.name) + + endpoint = None + cache_update = CacheUpdate(action='none', name=website.name) + if entry: + source = entry.get('source', 'unknown') + endpoint = {'feed_type': 'specific', 'url': entry['url'], 'source': source} + if source == 'manual': + logging.info(f"'{website.name}' 使用预设 RSS 源：{entry['url']}") + elif source == 'cache': + logging.info(f"'{website.name}' 使用缓存 RSS 源：{entry['url']}") + else: + logging.info(f"'{website.name}' 使用 RSS 源：{entry['url']} (来源: {source})") + else: + feed_type, feed_url = check_feed(website.url, session) + if feed_type != 'none' and feed_url: + endpoint = {'feed_type': feed_type, 'url': feed_url, 'source': 'auto'} + cache_update = CacheUpdate(action='set', name=website.name, url=feed_url, reason='auto_discovered') + logging.info(f"'{website.name}' 自动探测到 RSS：{feed_url}") + + articles = [] + parse_error = endpoint is not None + if endpoint: + parsed = FeedParserService(session).parse(endpoint['url'], count=count, blog_url=website.url) + articles = [ + { + 'title': article.title, + 'created': article.published, + 'link': article.link, + 'author': website.name, + 'avatar': website.avatar, + } + for article in parsed + ] + parse_error = not articles + + if parse_error and endpoint and endpoint['source'] in ('cache', 'unknown'): + logging.warning(f"'{website.name}' 缓存的 RSS 源无效，尝试重新探测...") + new_feed_type, new_feed_url = check_feed(website.url, session) + if new_feed_type != 'none' and new_feed_url: + reparsed = FeedParserService(session).parse(new_feed_url, count=count, blog_url=website.url) + articles = [ + { + 'title': article.title, + 'created': article.published, + 'link': article.link, + 'author': website.name, + 'avatar': website.avatar, + } + for article in reparsed + ] + if articles: + endpoint = {'feed_type': new_feed_type, 'url': new_feed_url, 'source': 'auto'} + cache_update = CacheUpdate(action='set', name=website.name, url=new_feed_url, reason='repair_cache') + logging.info(f"'{website.name}' 重新探测成功，更新缓存：{new_feed_url}") + else: + endpoint = None + cache_update = CacheUpdate(action='delete', name=website.name, url=None, reason='remove_invalid') + logging.warning(f"'{website.name}' 重新探测失败，删除无效缓存") + else: + endpoint = None + cache_update = CacheUpdate(action='delete', name=website.name, url=None, reason='remove_invalid') + logging.warning(f"'{website.name}' 未找到有效 RSS，删除无效缓存") + + return { + 'name': website.name, + 'status': 'active' if articles else 'error', + 'articles': articles, + 'feed_url': endpoint['url'] if endpoint else None, + 'feed_type': endpoint['feed_type'] if endpoint else 'none', + 'cache_update': cache_update.to_dict(), + 'source_used': endpoint['source'] if endpoint else 'none', + } + +def get_latest_articles_from_link(url, count=5, last_articles_path="./temp/newest_posts.json"): + """Return newly published articles relative to the last local snapshot.""" + session = requests.Session() + feed_type, feed_url = check_feed(url, session) + if feed_type == 'none': + logging.error(f"无法获取 {url} 的文章数据") + return None + + latest_articles = FeedParserService(session).parse(feed_url, count=count, blog_url=url) + updated_articles = LatestArticleTracker(last_articles_path).diff_and_persist(latest_articles) + logging.info( + f"从 {url} 获取到 {len(latest_articles)} 篇文章，其中 {0 if updated_articles is None else len(updated_articles)} 篇为新文章" + ) + return updated_articles + diff --git a/friend_circle_lite/crawler_service.py b/friend_circle_lite/crawler_service.py new file mode 100644 index 00000000000..b81044a1c77 --- /dev/null +++ b/friend_circle_lite/crawler_service.py @@ -0,0 +1,6 @@ +"""Backward-compatible crawler service exports. + +New code should import from `friend_circle_lite.crawler.service`. +""" + +from friend_circle_lite.crawler.service import * # noqa: F401,F403 diff --git a/friend_circle_lite/domain/__init__.py b/friend_circle_lite/domain/__init__.py new file mode 100644 index 00000000000..a9dec7356c1 --- /dev/null +++ b/friend_circle_lite/domain/__init__.py @@ -0,0 +1,3 @@ +"""Domain models used by crawlers, storage, and output builders.""" + +from friend_circle_lite.domain.models import * # noqa: F401,F403 diff --git a/friend_circle_lite/domain/models.py b/friend_circle_lite/domain/models.py new file mode 100644 index 00000000000..3470e6a5ff0 --- /dev/null +++ b/friend_circle_lite/domain/models.py @@ -0,0 +1,339 @@ +"""Domain models for Friend-Circle-Lite. + +These models centralize the core concepts used across the crawler so that the +transport layer, parsing logic, cache logic, and output formatting can evolve +independently. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from math import ceil +from urllib.parse import urlsplit, urlunsplit +from zoneinfo import ZoneInfo + + +SHANGHAI_TZ = ZoneInfo("Asia/Shanghai") +MIN_RECORDED_LATENCY = 0.01 +DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S" + + +def normalize_latency(value: float | int | str | None, default: float = MIN_RECORDED_LATENCY) -> float: + """规范化延迟值，对外记录时不使用 0 或负数表示未知。""" + try: + latency = float(value) + except (TypeError, ValueError): + return default + if latency <= 0: + return default + return max(round(latency, 2), default) + + +def normalize_homepage_url(url: str) -> str: + """规范化站点主页 URL，用于缓存匹配和持久化。""" + value = str(url or "").strip() + if not value: + return "" + try: + parts = urlsplit(value) + if parts.scheme and parts.netloc: + path = (parts.path.rstrip("/") + "/") if parts.path else "/" + return urlunsplit((parts.scheme.lower(), parts.netloc.lower(), path, parts.query, parts.fragment)) + except Exception: + pass + return value.rstrip("/") + "/" + + +def calculate_elapsed_days(started_at: str) -> int | None: + """Return rounded-up elapsed days for a persisted timestamp.""" + if not started_at: + return None + try: + since = datetime.strptime(started_at, DATETIME_FORMAT) + except ValueError: + return None + elapsed_seconds = max(0, (datetime.now() - since).total_seconds()) + return max(1, ceil(elapsed_seconds / 86400)) + + +@dataclass(slots=True) +class Website: + """Represents a friend website entry from the upstream friend list.""" + + name: str + url: str + avatar: str = "" + linkpage: str = "" + + def __post_init__(self) -> None: + self.url = normalize_homepage_url(self.url) + + @classmethod + def from_friend_item(cls, raw_friend: list | tuple | dict) -> "Website": + """Create a website from common friend link structures.""" + if isinstance(raw_friend, dict): + return cls( + name=str(raw_friend.get("name", "")).strip(), + url=normalize_homepage_url(raw_friend.get("link") or raw_friend.get("url") or ""), + avatar=str(raw_friend.get("avatar", "")).strip(), + linkpage=str(raw_friend.get("linkpage", "")).strip(), + ) + + name = raw_friend[0] + url = raw_friend[1] + if len(raw_friend) > 3: + linkpage = raw_friend[2] + avatar = raw_friend[3] + else: + linkpage = "" + avatar = raw_friend[2] if len(raw_friend) > 2 else "" + return cls(name=str(name).strip(), url=normalize_homepage_url(url), avatar=str(avatar or "").strip(), linkpage=str(linkpage or "").strip()) + + def to_error_payload(self) -> list[str]: + """Return the legacy structure used by `errors.json`.""" + return [self.name, self.url, self.avatar] + + def to_public_dict(self) -> dict[str, str]: + return { + "name": self.name, + "url": self.url, + "avatar": self.avatar, + "linkpage": self.linkpage, + } + + +@dataclass(slots=True) +class LinkMethodStatus: + """Status for one link-check method.""" + + success: bool = False + status_code: int | None = None + latency: float = -1 + + def to_dict(self) -> dict[str, bool | int | float | None]: + return { + "success": self.success, + "status_code": self.status_code, + "latency": self.latency, + } + + +@dataclass(slots=True) +class LinkCheckRecord: + """Reachability status for one friend website.""" + + name: str + url: str + avatar: str = "" + linkpage: str = "" + checked_at: str = "" + reachable: bool = False + crawl_allowed: bool = False + best_method: str = "none" + best_latency: float = -1 + backlink_checked: bool = False + has_author_link: bool = False + rss_crawl_reason: str = "blocked_unreachable" + last_post_published: str = "" + last_post_days_ago: int | None = None + unreachable_since: str = "" + rss_unavailable_since: str = "" + direct: LinkMethodStatus = field(default_factory=LinkMethodStatus) + proxy: LinkMethodStatus = field(default_factory=LinkMethodStatus) + api: LinkMethodStatus = field(default_factory=LinkMethodStatus) + + @classmethod + def unchecked(cls, website: Website, checked_at: str = "") -> "LinkCheckRecord": + return cls(name=website.name, url=website.url, avatar=website.avatar, linkpage=website.linkpage, checked_at=checked_at) + + def __post_init__(self) -> None: + self.url = normalize_homepage_url(self.url) + + def to_public_dict(self) -> dict[str, object]: + return { + "name": self.name, + "url": self.url, + "avatar": self.avatar, + "linkpage": self.linkpage, + "checked_at": self.checked_at, + "reachable": self.reachable, + "crawl_allowed": self.crawl_allowed, + "best_method": self.best_method, + "best_latency": self.best_latency, + "backlink_checked": self.backlink_checked, + "has_author_link": self.has_author_link, + "rss_crawl_reason": self.rss_crawl_reason, + "last_post_published": self.last_post_published, + "last_post_days_ago": self.last_post_days_ago, + "unreachable_since": self.unreachable_since, + "unreachable_days": self.unreachable_days, + "rss_unavailable_since": self.rss_unavailable_since, + "rss_unavailable_days": self.rss_unavailable_days, + "methods": { + "direct": self.direct.to_dict(), + "proxy": self.proxy.to_dict(), + "api": self.api.to_dict(), + }, + } + + @property + def unreachable_days(self) -> int | None: + """Return rounded-up calendar duration for the current unreachable period.""" + if self.reachable: + return None + return calculate_elapsed_days(self.unreachable_since) + + @property + def rss_unavailable_days(self) -> int | None: + """Return rounded-up duration for continuous RSS unavailability.""" + if not self.reachable or self.crawl_allowed: + return None + return calculate_elapsed_days(self.rss_unavailable_since) + + def to_link_dict(self) -> dict[str, object]: + latency = normalize_latency(self.best_latency) if self.reachable else -1 + return { + "name": self.name, + "link": self.url, + "link_page": self.linkpage, + "avatar": self.avatar, + "reachable": self.reachable, + "crawlable": self.crawl_allowed, + "latency": latency, + "unreachable_days": self.unreachable_days, + "unreachable_since": self.unreachable_since if not self.reachable else "", + "has_backlink": self.has_author_link if self.backlink_checked else None, + "updated": self.last_post_published, + "stale_days": self.last_post_days_ago, + } + + +@dataclass(slots=True) +class Article: + """Represents one crawled article belonging to a website.""" + + title: str + author: str + link: str + published: str + summary: str = "" + content: str = "" + avatar: str = "" + + def to_public_dict(self) -> dict[str, str]: + """Return the legacy public article schema used by `all.json`.""" + return { + "title": self.title, + "created": self.published, + "link": self.link, + "author": self.author, + "avatar": self.avatar, + } + + def to_tracking_dict(self) -> dict[str, str]: + """Return the article schema used by the latest article tracker.""" + return { + "title": self.title, + "author": self.author, + "link": self.link, + "published": self.published, + "summary": self.summary, + "content": self.content, + } + + +@dataclass(slots=True) +class FeedEndpoint: + """Represents a concrete feed endpoint and how it was found.""" + + url: str + feed_type: str + source: str + + +@dataclass(slots=True) +class CacheRecord: + """Represents one cached RSS endpoint mapping for a website.""" + + name: str + url: str + source: str = "cache" + + def to_dict(self) -> dict[str, str]: + return { + "name": self.name, + "url": self.url, + } + + +@dataclass(slots=True) +class CacheUpdate: + """Describes how a crawl should update the persisted RSS cache.""" + + action: str = "none" + name: str | None = None + url: str | None = None + reason: str = "" + + def to_dict(self) -> dict[str, str | None]: + return { + "action": self.action, + "name": self.name, + "url": self.url, + "reason": self.reason, + } + + +@dataclass(slots=True) +class CrawlResult: + """Represents the crawl result for a single website.""" + + website: Website + status: str + articles: list[Article] = field(default_factory=list) + feed_url: str | None = None + feed_type: str = "none" + source_used: str = "none" + cache_update: CacheUpdate = field(default_factory=CacheUpdate) + + def to_legacy_dict(self) -> dict[str, object]: + return { + "name": self.website.name, + "status": self.status, + "articles": [article.to_public_dict() for article in self.articles], + "feed_url": self.feed_url, + "feed_type": self.feed_type, + "cache_update": self.cache_update.to_dict(), + "source_used": self.source_used, + } + + +@dataclass(slots=True) +class CrawlStatistics: + """Aggregated crawl statistics for the generated `all.json` output.""" + + friends_num: int = 0 + active_num: int = 0 + error_num: int = 0 + article_num: int = 0 + last_updated_time: str = "" + + @classmethod + def create(cls, friends_num: int, active_num: int, error_num: int, article_num: int) -> "CrawlStatistics": + return cls( + friends_num=friends_num, + active_num=active_num, + error_num=error_num, + article_num=article_num, + last_updated_time=datetime.now(SHANGHAI_TZ).strftime("%Y-%m-%d %H:%M:%S"), + ) + + def to_dict(self) -> dict[str, int | str]: + return { + "friends_num": self.friends_num, + "active_num": self.active_num, + "error_num": self.error_num, + "article_num": self.article_num, + "last_updated_time": self.last_updated_time, + } diff --git a/friend_circle_lite/feed_service.py b/friend_circle_lite/feed_service.py new file mode 100644 index 00000000000..870a7e8c1e3 --- /dev/null +++ b/friend_circle_lite/feed_service.py @@ -0,0 +1,6 @@ +"""Backward-compatible feed service exports. + +New code should import from `friend_circle_lite.crawler.feed_service`. +""" + +from friend_circle_lite.crawler.feed_service import * # noqa: F401,F403 diff --git a/friend_circle_lite/get_conf.py b/friend_circle_lite/get_conf.py deleted file mode 100644 index 46b487c1f9b..00000000000 --- a/friend_circle_lite/get_conf.py +++ /dev/null @@ -1,14 +0,0 @@ -import yaml - -def load_config(config_file): - """ - 加载配置文件。 - - 参数： - config_file (str): 配置文件的路径。 - - 返回： - dict: 加载的配置数据。 - """ - with open(config_file, 'r', encoding='utf-8') as file: - return yaml.safe_load(file) \ No newline at end of file diff --git a/friend_circle_lite/get_info.py b/friend_circle_lite/get_info.py deleted file mode 100644 index f2856364f09..00000000000 --- a/friend_circle_lite/get_info.py +++ /dev/null @@ -1,651 +0,0 @@ -import logging -from datetime import datetime, timedelta, timezone -import re -import os -import json -from urllib.parse import urljoin, urlparse -from dateutil import parser -from zoneinfo import ZoneInfo -import requests -import feedparser -from concurrent.futures import ThreadPoolExecutor, as_completed - -# 标准化的请求头 -HEADERS_JSON = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/123.0.0.0 Safari/537.36 " - "(Friend-Circle-Lite/1.0; +https://github.com/willow-god/Friend-Circle-Lite)" - ), - "X-Friend-Circle": "1.0" -} - -HEADERS_XML = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/123.0.0.0 Safari/537.36 " - "(Friend-Circle-Lite/1.0; +https://github.com/willow-god/Friend-Circle-Lite)" - ), - "Accept": "application/atom+xml, application/rss+xml, application/xml;q=0.9, */*;q=0.8", - "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8", - "Accept-Encoding": "gzip, deflate", - "Connection": "keep-alive", - "X-Friend-Circle": "1.0" -} - -timeout = (10, 15) # 连接超时和读取超时，防止requests接受时间过长 - -def format_published_time(time_str): - """ - 格式化发布时间为统一格式 YYYY-MM-DD HH:MM - - 参数: - time_str (str): 输入的时间字符串，可能是多种格式。 - - 返回: - str: 格式化后的时间字符串，若解析失败返回空字符串。 - """ - # 尝试自动解析输入时间字符串 - try: - parsed_time = parser.parse(time_str, fuzzy=True) - except (ValueError, parser.ParserError): - # 定义支持的时间格式 - time_formats = [ - '%a, %d %b %Y %H:%M:%S %z', # Mon, 11 Mar 2024 14:08:32 +0000 - '%a, %d %b %Y %H:%M:%S GMT', # Wed, 19 Jun 2024 09:43:53 GMT - '%Y-%m-%dT%H:%M:%S%z', # 2024-03-11T14:08:32+00:00 - '%Y-%m-%dT%H:%M:%SZ', # 2024-03-11T14:08:32Z - '%Y-%m-%d %H:%M:%S', # 2024-03-11 14:08:32 - '%Y-%m-%d' # 2024-03-11 - ] - for fmt in time_formats: - try: - parsed_time = datetime.strptime(time_str, fmt) - break - except ValueError: - continue - else: - logging.warning(f"无法解析时间字符串：{time_str}") - return '' - - # 处理时区转换 - if parsed_time.tzinfo is None: - parsed_time = parsed_time.replace(tzinfo=timezone.utc) - shanghai_time = parsed_time.astimezone(timezone(timedelta(hours=8))) - return shanghai_time.strftime('%Y-%m-%d %H:%M') - -def check_feed(blog_url, session): - """ - 检查博客的 RSS 或 Atom 订阅链接。 - - 优化点： - - 检查 HTTP 状态码。 - - 检查 Content-Type 是否包含 xml / rss / atom。 - - 检查响应内容前几百字节内是否有 RSS/Atom 的特征标签。 - """ - possible_feeds = [ - ('atom', '/atom.xml'), - ('rss', '/rss.xml'), # 2024-07-26 添加 /rss.xml内容的支持 - ('rss2', '/rss2.xml'), - ('rss3', '/rss.php'), # 2024-12-07 添加 /rss.php内容的支持 - ('feed', '/feed'), - ('feed2', '/feed.xml'), # 2024-07-26 添加 /feed.xml内容的支持 - ('feed3', '/feed/'), - ('feed4', '/feed.php'), # 2025-07-22 添加 /feed.php内容的支持 - ('index', '/index.xml') # 2024-07-25 添加 /index.xml内容的支持 - ] - - for feed_type, path in possible_feeds: - feed_url = blog_url.rstrip('/') + path - try: - response = session.get(feed_url, headers=HEADERS_XML, timeout=timeout) - if response.status_code == 200: - # 检查 Content-Type - content_type = response.headers.get('Content-Type', '').lower() - if 'xml' in content_type or 'rss' in content_type or 'atom' in content_type: - return [feed_type, feed_url] - - # 如果 Content-Type 是 text/html 或未明确，但内容本身是 RSS - text_head = response.text[:1000].lower() # 读取前1000字符 - if (' str: - """ - 暂未实现 - 检测并替换字符串中的非正常域名部分（如 IP 地址或 localhost），替换为 blog_url。 - 替换后强制使用 https，且考虑 blog_url 尾部是否有斜杠。 - - :param link: 原始地址字符串 - :param blog_url: 替换为的博客地址 - :return: 替换后的地址字符串 - """ - - # 提取link中的路径部分，无需协议和域名 - # path = re.sub(r'^https?://[^/]+', '', link) - # print(path) - - try: - parsed = urlparse(link) - if 'localhost' in parsed.netloc or re.match(r'^\d{1,3}(\.\d{1,3}){3}$', parsed.netloc): # IP地址或localhost - # 提取 path + query - path = parsed.path or '/' - if parsed.query: - path += '?' + parsed.query - return urljoin(blog_url.rstrip('/') + '/', path.lstrip('/')) - else: - return link # 合法域名则返回原链接 - except Exception as e: - logging.warning(f"替换链接时出错：{link}, error: {e}") - return link - -def process_friend(friend, session, count, specific_and_cache=None): - """ - 处理单个朋友的博客信息。 - - 参数： - friend (list/tuple): [name, blog_url, avatar] - session (requests.Session): 请求会话 - count (int): 每个博客最大文章数 - specific_and_cache (list[dict]): [{name, url, source?}]，合并后的特殊 + 缓存列表 - - 返回： - { - 'name': name, - 'status': 'active' | 'error', - 'articles': [...], - 'feed_url': str | None, - 'feed_type': str, - 'cache_update': { - 'action': 'set' | 'delete' | 'none', - 'name': name, - 'url': feed_url_or_None, - 'reason': 'auto_discovered' | 'repair_cache' | 'remove_invalid', - }, - 'source_used': 'manual' | 'cache' | 'auto' | 'none' - } - """ - if specific_and_cache is None: - specific_and_cache = [] - - # 解包 friend - try: - name, blog_url, avatar = friend - except Exception: - logging.error(f"friend 数据格式不正确: {friend!r}") - return { - 'name': None, - 'status': 'error', - 'articles': [], - 'feed_url': None, - 'feed_type': 'none', - 'cache_update': {'action': 'none', 'name': None, 'url': None, 'reason': 'bad_friend_data'}, - 'source_used': 'none', - } - - rss_lookup = {e['name']: e for e in specific_and_cache if 'name' in e and 'url' in e} - cache_update = {'action': 'none', 'name': name, 'url': None, 'reason': ''} - feed_url, feed_type, source_used = None, 'none', 'none' - - # ---- 1. 优先使用 specific 或 cache ---- - entry = rss_lookup.get(name) - if entry: - feed_url = entry['url'] - feed_type = 'specific' - source_used = entry.get('source', 'unknown') - logging.info(f"“{name}” 使用预设 RSS 源：{feed_url} （source={source_used}）。") - else: - # ---- 2. 自动探测 ---- - feed_type, feed_url = check_feed(blog_url, session) - source_used = 'auto' - logging.info(f"“{name}” 自动探测 RSS：type：{feed_type}, url：{feed_url} 。") - - if feed_type != 'none' and feed_url: - cache_update = {'action': 'set', 'name': name, 'url': feed_url, 'reason': 'auto_discovered'} - - # ---- 3. 尝试解析 RSS ---- - articles, parse_error = [], False - if feed_type != 'none' and feed_url: - try: - feed_info = parse_feed(feed_url, session, count, blog_url) - if isinstance(feed_info, dict) and 'articles' in feed_info: - articles = [ - { - 'title': a['title'], - 'created': a['published'], - 'link': a['link'], - 'author': name, - 'avatar': avatar, - } - for a in feed_info['articles'] - ] - - for a in articles: - logging.info(f"{name} 发布了新文章：{a['title']}，时间：{a['created']}，链接：{a['link']}") - else: - parse_error = True - except Exception as e: - logging.warning(f"解析 RSS 失败（{name} -> {feed_url}）：{e}") - parse_error = True - - # ---- 4. 如果缓存 RSS 无效则重新探测 ---- - if parse_error and source_used in ('cache', 'unknown'): - logging.info(f"缓存 RSS 无效，重新探测：{name} ({blog_url})。") - new_type, new_url = check_feed(blog_url, session) - if new_type != 'none' and new_url: - try: - feed_info = parse_feed(new_url, session, count, blog_url) - if isinstance(feed_info, dict) and 'articles' in feed_info: - articles = [ - { - 'title': a['title'], - 'created': a['published'], - 'link': a['link'], - 'author': name, - 'avatar': avatar, - } - for a in feed_info['articles'] - ] - - for a in articles: - logging.info(f"{name} 发布了新文章：{a['title']}，时间：{a['created']}，链接：{a['link']}") - - feed_type, feed_url, source_used = new_type, new_url, 'auto' - cache_update = {'action': 'set', 'name': name, 'url': new_url, 'reason': 'repair_cache'} - parse_error = False - except Exception as e: - logging.warning(f"重新探测解析仍失败：{name} ({new_url})：{e}") - cache_update = {'action': 'delete', 'name': name, 'url': None, 'reason': 'remove_invalid'} - feed_type, feed_url = 'none', None - else: - cache_update = {'action': 'delete', 'name': name, 'url': None, 'reason': 'remove_invalid'} - feed_type, feed_url = 'none', None - - # ---- 5. 最终状态 ---- - status = 'active' if articles else 'error' - if not articles: - if feed_type == 'none': - logging.warning(f"{name} 的博客 {blog_url} 未找到有效 RSS。") - else: - logging.warning(f"{name} 的 RSS {feed_url} 未解析出文章。") - - return { - 'name': name, - 'status': status, - 'articles': articles, - 'feed_url': feed_url, - 'feed_type': feed_type, - 'cache_update': cache_update, - 'source_used': source_used, - } - -def _load_cache(cache_file): - if not cache_file: - return [] - if not os.path.exists(cache_file): - logging.info(f"缓存文件 {cache_file} 不存在，将自动创建。") - return [] - try: - with open(cache_file, 'r', encoding='utf-8') as f: - data = json.load(f) - if not isinstance(data, list): - logging.warning(f"缓存文件 {cache_file} 格式异常（应为列表）。将忽略。") - return [] - # 标准化 - norm = [] - for item in data: - if not isinstance(item, dict): - continue - name = item.get('name') - url = item.get('url') - if name and url: - norm.append({'name': name, 'url': url, 'source': 'cache'}) - return norm - except Exception as e: - logging.warning(f"读取缓存文件 {cache_file} 失败: {e}") - return [] - -def _atomic_write_json(path, data) -> None: - """原子写，减少写坏文件风险。""" - tmp = f"{path}.tmp" - with open(tmp, 'w', encoding='utf-8') as f: - json.dump(data, f, ensure_ascii=False, indent=2) - os.replace(tmp, path) - -def _save_cache(cache_file, cache_items): - if not cache_file: - return - try: - # 丢弃 source 字段以保持文件简洁 - out = [{'name': i['name'], 'url': i['url']} for i in cache_items] - _atomic_write_json(cache_file, out) - logging.info(f"缓存已保存到 {cache_file}（{len(out)} 条）。") - except Exception as e: - logging.error(f"保存缓存文件 {cache_file} 失败: {e}") - -def fetch_and_process_data(json_url, specific_RSS=None, count=5, cache_file=None): - """ - 读取 JSON 数据并处理订阅信息，返回统计数据和文章信息。 - - 参数： - json_url (str): 包含朋友信息的 JSON 文件的 URL。 - count (int): 获取每个博客的最大文章数。 - specific_RSS (list): 包含特定 RSS 源的字典列表 [{name, url}]（来自 YAML）。 - cache_file (str): 缓存文件路径。 - - 返回： - (result_dict, error_friends_info_list) - """ - if specific_RSS is None: - specific_RSS = [] - - # 1. 加载缓存 - cache_list = _load_cache(cache_file) - - # 2. 标记 YAML 条目 - manual_list = [] - for item in specific_RSS: - if isinstance(item, dict) and 'name' in item and 'url' in item: - manual_list.append({'name': item['name'], 'url': item['url'], 'source': 'manual'}) - - # 3. 合并（缓存先，YAML 后覆盖） - combined_map = {e['name']: e for e in cache_list} - for e in manual_list: # 手动优先 - combined_map[e['name']] = e - specific_and_cache = list(combined_map.values()) - - # 4. 建立方便判断的集合：手动源名称集合 - manual_name_set = {e['name'] for e in manual_list} - - # 5. 获取朋友列表 - session = requests.Session() - try: - response = session.get(json_url, headers=HEADERS_JSON, timeout=timeout) - friends_data = response.json() - except Exception as e: - logging.error(f"无法获取链接：{json_url} ：{e}", exc_info=True) - return None - - friends = friends_data.get('friends', []) - total_friends = len(friends) - active_friends = 0 - error_friends = 0 - total_articles = 0 - article_data = [] - error_friends_info = [] - cache_updates = [] # 用于收集缓存更新（线程安全：用局部列表 + 合并） - - # 6. 并发处理 - with ThreadPoolExecutor(max_workers=10) as executor: - future_to_friend = { - executor.submit(process_friend, friend, session, count, specific_and_cache): friend - for friend in friends - } - - for future in as_completed(future_to_friend): - friend = future_to_friend[future] - try: - result = future.result() - - # 拿回缓存更新意图 - upd = result.get('cache_update', {}) - if upd and upd.get('action') != 'none': - cache_updates.append(upd) - - if result['status'] == 'active': - active_friends += 1 - article_data.extend(result['articles']) - total_articles += len(result['articles']) - else: - error_friends += 1 - error_friends_info.append(friend) - - except Exception as e: - logging.error(f"处理 {friend} 时发生错误: {e}", exc_info=True) - error_friends += 1 - error_friends_info.append(friend) - - # 7. 处理缓存更新 - cache_map = {e['name']: e for e in cache_list} - - # 去重 & 过滤无效条目 - unique_updates = {} - for upd in cache_updates: - name = upd.get('name') - action = upd.get('action') - url = upd.get('url') - if not name: - continue - - # 过滤手动 YAML 的条目（不允许覆盖） - if name in manual_name_set: - continue - - # 只缓存有效 RSS 地址 - if action == 'set': - if url and url != 'none' and url != '': - unique_updates[name] = {'action': 'set', 'url': url, 'reason': upd.get('reason', '')} - elif action == 'delete': - unique_updates[name] = {'action': 'delete', 'url': None, 'reason': upd.get('reason', '')} - - # 应用缓存更新 - for name, upd in unique_updates.items(): - if upd['action'] == 'set': - cache_map[name] = {'name': name, 'url': upd['url'], 'source': 'cache'} - logging.info(f"缓存更新：SET {name} -> {upd['url']} ({upd['reason']})") - elif upd['action'] == 'delete': - if name in cache_map: - cache_map.pop(name) - logging.info(f"缓存更新：DELETE {name} ({upd['reason']})") - - # 8. 保存缓存 - _save_cache(cache_file, list(cache_map.values())) - - # 9. 汇总统计 - result = { - 'statistical_data': { - 'friends_num': total_friends, - 'active_num': active_friends, - 'error_num': error_friends, - 'article_num': total_articles, - 'last_updated_time': datetime.now(ZoneInfo("Asia/Shanghai")).strftime('%Y-%m-%d %H:%M:%S'), - }, - 'article_data': article_data, - } - - logging.info( - f"数据处理完成，总共有 {total_friends} 位朋友，其中 {active_friends} 位博客可访问，" - f"{error_friends} 位博客无法访问。缓存更新 {len(unique_updates)} 条。" - ) - - return result, error_friends_info - -def sort_articles_by_time(data): - """ - 对文章数据按时间排序 - - 参数： - data (dict): 包含文章信息的字典 - - 返回： - dict: 按时间排序后的文章信息字典 - """ - # 先确保每个元素存在时间 - for article in data['article_data']: - if article['created'] == '' or article['created'] == None: - article['created'] = '2024-01-01 00:00' - # 输出警告信息 - logging.warning(f"文章 {article['title']} 未包含时间信息，已设置为默认时间 2024-01-01 00:00") - - if 'article_data' in data: - sorted_articles = sorted( - data['article_data'], - key=lambda x: datetime.strptime(x['created'], '%Y-%m-%d %H:%M'), - reverse=True - ) - data['article_data'] = sorted_articles - return data - -def marge_data_from_json_url(data, marge_json_url): - """ - 从另一个 JSON 文件中获取数据并合并到原数据中。 - - 参数： - data (dict): 包含文章信息的字典 - marge_json_url (str): 包含另一个文章信息的 JSON 文件的 URL。 - - 返回： - dict: 合并后的文章信息字典，已去重处理 - """ - try: - response = requests.get(marge_json_url, headers=HEADERS_JSON, timeout=timeout) - marge_data = response.json() - except Exception as e: - logging.error(f"无法获取链接：{marge_json_url}，出现的问题为：{e}", exc_info=True) - return data - - if 'article_data' in marge_data: - logging.info(f"开始合并数据，原数据共有 {len(data['article_data'])} 篇文章，第三方数据共有 {len(marge_data['article_data'])} 篇文章") - data['article_data'].extend(marge_data['article_data']) - data['article_data'] = list({v['link']:v for v in data['article_data']}.values()) - logging.info(f"合并数据完成，现在共有 {len(data['article_data'])} 篇文章") - return data - -import requests - -def marge_errors_from_json_url(errors, marge_json_url): - """ - 从另一个网络 JSON 文件中获取错误信息并遍历，删除在errors中， - 不存在于marge_errors中的友链信息。 - - 参数： - errors (list): 包含错误信息的列表 - marge_json_url (str): 包含另一个错误信息的 JSON 文件的 URL。 - - 返回： - list: 合并后的错误信息列表 - """ - try: - response = requests.get(marge_json_url, timeout=10) # 设置请求超时时间 - marge_errors = response.json() - except Exception as e: - logging.error(f"无法获取链接：{marge_json_url}，出现的问题为：{e}", exc_info=True) - return errors - - # 提取 marge_errors 中的 URL - marge_urls = {item[1] for item in marge_errors} - - # 使用过滤器保留 errors 中在 marge_errors 中出现的 URL - filtered_errors = [error for error in errors if error[1] in marge_urls] - - logging.info(f"合并错误信息完成，合并后共有 {len(filtered_errors)} 位朋友") - return filtered_errors - -def deal_with_large_data(result): - """ - 处理文章数据，保留前150篇及其作者在后续文章中的出现。 - - 参数： - result (dict): 包含统计数据和文章数据的字典。 - - 返回： - dict: 处理后的数据，只包含需要的文章。 - """ - result = sort_articles_by_time(result) - article_data = result.get("article_data", []) - - # 检查文章数量是否大于 150 - max_articles = 150 - if len(article_data) > max_articles: - logging.info("数据量较大，开始进行处理...") - # 获取前 max_articles 篇文章的作者集合 - top_authors = {article["author"] for article in article_data[:max_articles]} - - # 从第 {max_articles + 1} 篇开始过滤，只保留前 max_articles 篇出现过的作者的文章 - filtered_articles = article_data[:max_articles] + [ - article for article in article_data[max_articles:] - if article["author"] in top_authors - ] - - # 更新结果中的 article_data - result["article_data"] = filtered_articles - # 更新结果中的统计数据 - result["statistical_data"]["article_num"] = len(filtered_articles) - logging.info(f"数据处理完成，保留 {len(filtered_articles)} 篇文章") - - return result \ No newline at end of file diff --git a/friend_circle_lite/link_check_service.py b/friend_circle_lite/link_check_service.py new file mode 100644 index 00000000000..02d19695318 --- /dev/null +++ b/friend_circle_lite/link_check_service.py @@ -0,0 +1,6 @@ +"""Backward-compatible link checker exports. + +New code should import from `friend_circle_lite.link_checker.service`. +""" + +from friend_circle_lite.link_checker.service import * # noqa: F401,F403 diff --git a/friend_circle_lite/link_checker/__init__.py b/friend_circle_lite/link_checker/__init__.py new file mode 100644 index 00000000000..b607c3b2a24 --- /dev/null +++ b/friend_circle_lite/link_checker/__init__.py @@ -0,0 +1,3 @@ +"""Friend link reachability checks.""" + +from friend_circle_lite.link_checker.service import LinkReachabilityService diff --git a/friend_circle_lite/link_checker/service.py b/friend_circle_lite/link_checker/service.py new file mode 100644 index 00000000000..1a47ba0127e --- /dev/null +++ b/friend_circle_lite/link_checker/service.py @@ -0,0 +1,448 @@ +"""友链可达性与 RSS 可抓取性检测。 + +检测策略： +1. 优先检查手动 RSS 或缓存 RSS，能解析文章则认为站点可达且可抓取。 +2. RSS 不可用时自动探测常见 RSS 地址。 +3. 仍找不到 RSS 时检查主页，主页可访问则只标记可达，不参与朋友圈抓取。 +4. 主页不可访问时保留 API 兜底，用于判断站点是否可能可达。 +5. 反链检测只在站点可达时执行。 +""" + +from __future__ import annotations + +import logging +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from datetime import datetime +from urllib.parse import quote, urlparse, urlsplit, urlunsplit + +import requests + +from friend_circle_lite.config.models import LinkCheckConfig, ProxySettings +from friend_circle_lite.crawler.feed_service import FeedDiscoveryService, FeedParserService +from friend_circle_lite.crawler.http_client import WebFetchClient +from friend_circle_lite.domain.models import ( + Article, + CacheRecord, + FeedEndpoint, + LinkCheckRecord, + LinkMethodStatus, + Website, + calculate_elapsed_days, + normalize_latency, +) +from friend_circle_lite.storage.sqlite_store import LinkCheckStore + + +LINK_CHECK_HEADERS = { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/123.0.0.0 Safari/537.36 " + "(Friend-Circle-Lite/2.0; +https://github.com/willow-god/Friend-Circle-Lite)" + ), + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "zh-CN,zh;q=0.9", + "Connection": "keep-alive", + "X-Friend-Circle-Link-Check": "1.0", +} + +RAW_HEADERS = { + "User-Agent": LINK_CHECK_HEADERS["User-Agent"], + "X-Friend-Circle-Link-Check": "1.0", +} + + +class RetryBackoffPolicy: + """Compute dynamic recheck windows for long-term failures.""" + + @staticmethod + def effective_max_age_hours(since: str, default_hours: int) -> int: + days = calculate_elapsed_days(since) + if days is None or days < 10: + return default_hours + if days >= 60: + return 15 * 24 + if days >= 30: + return 10 * 24 + return 5 * 24 + + +class LinkReachabilityService: + """检查友链是否可达，以及是否可参与 RSS 抓取。""" + + def __init__( + self, + config: LinkCheckConfig, + proxy_settings: ProxySettings, + store: LinkCheckStore, + feed_records: list[CacheRecord] | None = None, + feed_parser=None, + feed_discovery=None, + fetcher: WebFetchClient | None = None, + ): + self.config = config + self.proxy_settings = proxy_settings + self.store = store + self.feed_lookup = {record.name: record for record in (feed_records or [])} + self.feed_parser = feed_parser + self.feed_discovery = feed_discovery + self.fetcher = fetcher + self.feed_updates: dict[str, CacheRecord | None] = {} + + def check_websites(self, websites: list[Website]) -> list[LinkCheckRecord]: + """检查一组友链，优先复用未过期缓存。""" + cached_records = self.store.load_records([website.url for website in websites]) + records_by_url: dict[str, LinkCheckRecord] = {} + websites_to_check: list[Website] = [] + backlink_refresh_records: list[tuple[Website, LinkCheckRecord]] = [] + + for website in websites: + cached = cached_records.get(website.url) + if cached and self._can_reuse_cached_record(cached, website): + linkpage_changed = not self._same_linkpage(cached.linkpage, website.linkpage) + refreshed = self._refresh_cached_metadata(cached, website) + records_by_url[website.url] = refreshed + if self._should_refresh_backlink(refreshed, website, linkpage_changed): + backlink_refresh_records.append((website, refreshed)) + else: + websites_to_check.append(website) + + total_count = len(websites) + cached_count = total_count - len(websites_to_check) + logging.info( + f"[友链检测] 友链总数 {total_count} 个，缓存复用 {cached_count} 个，" + f"本次实际检测 {len(websites_to_check)} 个，缓存有效期 {self.config.max_age_hours} 小时" + ) + + if websites_to_check: + logging.info( + f"[友链检测] 开始实际检测 {len(websites_to_check)} 个友链状态，" + f"其余 {cached_count} 个复用缓存" + ) + checked_records = self._check_fresh_websites(websites_to_check, cached_records) + self.store.save_records(checked_records) + for record in checked_records: + records_by_url[record.url] = record + else: + logging.info(f"[友链检测] 全部 {total_count} 个友链状态缓存仍有效，本次不发起友链检测请求") + + if backlink_refresh_records: + logging.info(f"[反链检测] 友链页地址变更，单独刷新 {len(backlink_refresh_records)} 个反链状态") + self._refresh_backlinks_only(backlink_refresh_records) + + return [records_by_url.get(website.url) or LinkCheckRecord.unchecked(website) for website in websites] + + def _check_fresh_websites(self, websites: list[Website], cached_records: dict[str, LinkCheckRecord]) -> list[LinkCheckRecord]: + records: list[LinkCheckRecord] = [] + with requests.Session() as session: + self.feed_parser = self.feed_parser or FeedParserService(session, self.proxy_settings) + self.feed_discovery = self.feed_discovery or FeedDiscoveryService(session, self.proxy_settings) + self.fetcher = self.fetcher or WebFetchClient(session, self.proxy_settings) + with ThreadPoolExecutor(max_workers=max(1, self.config.max_workers)) as executor: + future_to_website = { + executor.submit(self._check_website, session, website, cached_records.get(website.url)): website + for website in websites + } + for future in as_completed(future_to_website): + website = future_to_website[future] + try: + records.append(future.result()) + except Exception as exc: + logging.warning(f"[友链检测] 友链 {website.name} 检测失败: {exc}") + records.append(self._build_failed_record(website, cached_records.get(website.url))) + return records + + def _check_website(self, session: requests.Session, website: Website, cached: LinkCheckRecord | None) -> LinkCheckRecord: + record = self._check_rss_first(website, cached) + if record is None: + homepage = self._request_homepage(website.url) + api = LinkMethodStatus() + if not homepage.success: + api = self._request_api(session, website.url) + time.sleep(0.2) + record = self._compose_non_rss_record(website, cached, homepage, api) + + if record.reachable and self.config.enable_backlink_check and self.config.author_url and website.linkpage: + record.backlink_checked = True + record.has_author_link = self._check_author_link_in_page(session, website.linkpage) + elif not record.reachable: + record.backlink_checked = bool(website.linkpage) + record.has_author_link = False + return record + + def _check_rss_first(self, website: Website, cached: LinkCheckRecord | None) -> LinkCheckRecord | None: + configured = self.feed_lookup.get(website.name) + if configured: + endpoint = FeedEndpoint(url=configured.url, feed_type="specific", source=configured.source) + latest_article = self._latest_feed_article(endpoint, website) + if latest_article: + return self._build_feed_record(website, endpoint, self._last_feed_latency(), latest_article) + logging.warning(f"友链 {website.name} 的缓存 RSS 失效: {configured.url} ，开始重新探测") + if configured.source == "cache": + self.feed_updates[website.name] = None + + discovered = self.feed_discovery.discover(website.url) if self.feed_discovery else None + latest_article = self._latest_feed_article(discovered, website) if discovered else None + if discovered and latest_article: + self.feed_updates[website.name] = CacheRecord(name=website.name, url=discovered.url, source="cache") + return self._build_feed_record(website, discovered, self._last_feed_latency(), latest_article) + return None + + def _latest_feed_article(self, endpoint: FeedEndpoint, website: Website) -> Article | None: + articles = self.feed_parser.parse(endpoint.url, count=1, blog_url=website.url) + return articles[0] if articles else None + + def _last_feed_latency(self) -> float: + return normalize_latency(getattr(self.feed_parser, "last_latency", None)) + + def _build_feed_record(self, website: Website, endpoint: FeedEndpoint, latency: float, latest_article: Article) -> LinkCheckRecord: + method = "rss_cache" if endpoint.source == "cache" else "rss" + last_post_published, last_post_days_ago = self._article_staleness(latest_article) + return LinkCheckRecord( + name=website.name, + url=website.url, + avatar=website.avatar, + linkpage=website.linkpage, + checked_at=self._now_text(), + reachable=True, + crawl_allowed=True, + best_method=method, + best_latency=latency, + rss_crawl_reason=f"allowed_by_{method}", + last_post_published=last_post_published, + last_post_days_ago=last_post_days_ago, + unreachable_since="", + rss_unavailable_since="", + ) + + @staticmethod + def _article_staleness(article: Article) -> tuple[str, int | None]: + published = article.published or "" + if not published: + return "", None + try: + published_at = datetime.strptime(published, "%Y-%m-%d %H:%M") + except ValueError: + return published, None + days_ago = max(0, int((datetime.now() - published_at).total_seconds() // 86400)) + return published, days_ago + + def _request_homepage(self, url: str) -> LinkMethodStatus: + if not self._is_url(url): + return LinkMethodStatus() + result = self.fetcher.get(url, headers=LINK_CHECK_HEADERS, timeout=self.config.timeout, desc="主页检测") + if result.response is None: + return LinkMethodStatus(success=False, status_code=None, latency=result.latency) + return LinkMethodStatus(success=result.success, status_code=result.response.status_code, latency=result.latency) + + def _request_api(self, session: requests.Session, url: str) -> LinkMethodStatus: + if not self.config.status_api_url: + return LinkMethodStatus() + + api_url = self.config.status_api_url.format(url=quote(url, safe="")) + start_time = time.time() + try: + response = session.get(api_url, headers=RAW_HEADERS, timeout=30) + latency = normalize_latency(time.time() - start_time) + except requests.RequestException as exc: + logging.warning(f"[API 检查] 请求失败: {url} ，错误: {exc}") + return LinkMethodStatus(success=False, status_code=None, latency=normalize_latency(time.time() - start_time)) + + try: + payload = response.json() + status_code = int(payload.get("data", 0)) + success = int(payload.get("code", 0)) == 200 and status_code == 200 + if success: + logging.info(f"[API 检查] 成功访问: {url} ，状态码 200") + else: + logging.warning(f"[API 检查] 状态异常: {url} -> [{payload.get('code')}, {payload.get('data')}]") + return LinkMethodStatus(success=success, status_code=status_code, latency=latency) + except Exception as exc: + logging.warning(f"[API 检查] 解析响应失败: {url} ，错误: {exc}") + return LinkMethodStatus(success=False, status_code=response.status_code, latency=latency) + + def _compose_non_rss_record( + self, + website: Website, + cached: LinkCheckRecord | None, + homepage: LinkMethodStatus, + api: LinkMethodStatus, + ) -> LinkCheckRecord: + reachable = homepage.success or api.success + if homepage.success: + best_method = "homepage" + best_latency = homepage.latency + reason = "reachable_without_rss" + elif api.success: + best_method = "api" + best_latency = api.latency + reason = "api_reachable_without_rss" + else: + best_method = "none" + best_latency = -1 + reason = "blocked_unreachable" + + checked_at = self._now_text() + return LinkCheckRecord( + name=website.name, + url=website.url, + avatar=website.avatar, + linkpage=website.linkpage, + checked_at=checked_at, + reachable=reachable, + crawl_allowed=False, + best_method=best_method, + best_latency=best_latency, + rss_crawl_reason=reason, + unreachable_since="" if reachable else self._unreachable_since(cached, checked_at), + rss_unavailable_since=self._rss_unavailable_since(cached, checked_at) if reachable else "", + direct=homepage, + api=api, + ) + + def _check_author_link_in_page(self, session: requests.Session, linkpage_url: str) -> bool: + fetcher = self.fetcher or WebFetchClient(session, self.proxy_settings) + result = fetcher.get(linkpage_url, headers=RAW_HEADERS, timeout=self.config.timeout, desc="友链页面检测") + response = result.response + if response is None: + return False + + author_url = self.config.author_url + if not author_url.startswith(("http://", "https://")): + author_url = "https://" + author_url + + variants = { + author_url, + author_url.replace("https://", "http://"), + author_url.replace("https://", "//"), + author_url.replace("https://", ""), + self.config.author_url, + "//" + self.config.author_url, + "https://" + self.config.author_url, + "http://" + self.config.author_url, + } + content = response.text + for variant in variants: + if ( + f'href="{variant}"' in content + or f"href='{variant}'" in content + or f'href="{variant}/"' in content + or f"href='{variant}/'" in content + or variant in content + ): + return True + return False + + def _can_reuse_cached_record(self, cached: LinkCheckRecord, website: Website) -> bool: + max_age_hours = self._effective_recheck_hours(cached) + try: + has_measured_latency = float(cached.best_latency) > 0 + except (TypeError, ValueError): + has_measured_latency = False + if not has_measured_latency: + return not cached.reachable and self.store.is_fresh(cached, max_age_hours) + if cached.crawl_allowed and website.name not in self.feed_lookup: + return False + return self.store.is_fresh(cached, max_age_hours) + + @staticmethod + def _refresh_cached_metadata(cached: LinkCheckRecord, website: Website) -> LinkCheckRecord: + cached.name = website.name + cached.avatar = website.avatar + cached.linkpage = website.linkpage + return cached + + def _should_refresh_backlink(self, record: LinkCheckRecord, website: Website, linkpage_changed: bool) -> bool: + return bool( + linkpage_changed + and record.reachable + and self.config.enable_backlink_check + and self.config.author_url + and website.linkpage + ) + + def _refresh_backlinks_only(self, items: list[tuple[Website, LinkCheckRecord]]) -> None: + with requests.Session() as session: + self.fetcher = self.fetcher or WebFetchClient(session, self.proxy_settings) + for website, record in items: + record.backlink_checked = True + record.has_author_link = self._check_author_link_in_page(session, website.linkpage) + self.store.save_records([record for _, record in items]) + + def _build_failed_record(self, website: Website, cached: LinkCheckRecord | None) -> LinkCheckRecord: + checked_at = self._now_text() + record = LinkCheckRecord.unchecked(website, checked_at) + record.unreachable_since = self._unreachable_since(cached, checked_at) + record.rss_unavailable_since = "" + return record + + @staticmethod + def _unreachable_since(cached: LinkCheckRecord | None, checked_at: str) -> str: + if cached and not cached.reachable and cached.unreachable_since: + return cached.unreachable_since + if cached and not cached.reachable and cached.checked_at: + return cached.checked_at + return checked_at + + @staticmethod + def _rss_unavailable_since(cached: LinkCheckRecord | None, checked_at: str) -> str: + if cached and cached.reachable and not cached.crawl_allowed and cached.rss_unavailable_since: + return cached.rss_unavailable_since + if cached and cached.reachable and not cached.crawl_allowed and cached.checked_at: + return cached.checked_at + return checked_at + + def _effective_recheck_hours(self, cached: LinkCheckRecord) -> int: + if not cached.reachable: + return RetryBackoffPolicy.effective_max_age_hours(cached.unreachable_since, self.config.max_age_hours) + if self._is_rss_unavailable_record(cached): + return RetryBackoffPolicy.effective_max_age_hours(cached.rss_unavailable_since, self.config.max_age_hours) + return self.config.max_age_hours + + @staticmethod + def _is_rss_unavailable_record(record: LinkCheckRecord) -> bool: + return record.reachable and not record.crawl_allowed + + @staticmethod + def _build_disabled_record(website: Website, checked_at: str) -> LinkCheckRecord: + return LinkCheckRecord( + name=website.name, + url=website.url, + avatar=website.avatar, + linkpage=website.linkpage, + checked_at=checked_at, + reachable=True, + crawl_allowed=True, + best_method="disabled", + best_latency=0.01, + rss_crawl_reason="link_check_disabled", + ) + + @staticmethod + def _is_url(path: str) -> bool: + return urlparse(path).scheme in ("http", "https") + + @staticmethod + def _same_linkpage(left: str, right: str) -> bool: + return LinkReachabilityService._normalize_linkpage(left) == LinkReachabilityService._normalize_linkpage(right) + + @staticmethod + def _normalize_linkpage(url: str) -> str: + url = (url or "").strip() + if not url: + return "" + try: + parts = urlsplit(url) + path = parts.path.rstrip("/") or "/" + return urlunsplit((parts.scheme.lower(), parts.netloc.lower(), path, parts.query, parts.fragment)) + except Exception: + return url.rstrip("/") + + @staticmethod + def _now_text() -> str: + return datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + +# 兼容旧类名。 +LinkCheckService = LinkReachabilityService diff --git a/friend_circle_lite/models.py b/friend_circle_lite/models.py new file mode 100644 index 00000000000..80e41043c64 --- /dev/null +++ b/friend_circle_lite/models.py @@ -0,0 +1,6 @@ +"""Backward-compatible domain model exports. + +New code should import from `friend_circle_lite.domain.models`. +""" + +from friend_circle_lite.domain.models import * # noqa: F401,F403 diff --git a/friend_circle_lite/notifications/__init__.py b/friend_circle_lite/notifications/__init__.py new file mode 100644 index 00000000000..738310168dc --- /dev/null +++ b/friend_circle_lite/notifications/__init__.py @@ -0,0 +1,4 @@ +"""Notification integrations for GitHub issue subscriptions and email.""" + +from friend_circle_lite.notifications.github import extract_emails_from_issues +from friend_circle_lite.notifications.mail import send_emails diff --git a/friend_circle_lite/notifications/github.py b/friend_circle_lite/notifications/github.py new file mode 100644 index 00000000000..2cbdc50a196 --- /dev/null +++ b/friend_circle_lite/notifications/github.py @@ -0,0 +1,39 @@ +import logging +import requests +import re +from friend_circle_lite import HEADERS_JSON + +def extract_emails_from_issues(api_url): + """ + 从GitHub issues API中提取以[e-mail]开头的title中的邮箱地址。 + + 参数： + api_url (str): GitHub issues API的URL。 + + 返回： + dict: 包含所有提取的邮箱地址的字典。 + { + "emails": [ + "3162475700@qq.com" + ] + } + """ + try: + response = requests.get(api_url, headers=HEADERS_JSON, timeout=10) + response.raise_for_status() + issues = response.json() + except Exception as e: + logging.error(f"无法获取 GitHub issues 数据，错误信息: {e}") + return None + + email_pattern = re.compile(r'^\[邮箱订阅\](.+)$') + emails = [] + + for issue in issues: + title = issue.get("title", "") + match = email_pattern.match(title) + if match: + email = match.group(1).strip() + emails.append(email) + + return {"emails": emails} \ No newline at end of file diff --git a/push_rss_update/send_email.py b/friend_circle_lite/notifications/mail.py similarity index 99% rename from push_rss_update/send_email.py rename to friend_circle_lite/notifications/mail.py index 485a082b840..bb7df1e8b92 100644 --- a/push_rss_update/send_email.py +++ b/friend_circle_lite/notifications/mail.py @@ -1,6 +1,5 @@ import logging import smtplib -import socket import time import os from email.mime.multipart import MIMEMultipart diff --git a/friend_circle_lite/outputs/__init__.py b/friend_circle_lite/outputs/__init__.py new file mode 100644 index 00000000000..0db718bfaa4 --- /dev/null +++ b/friend_circle_lite/outputs/__init__.py @@ -0,0 +1,3 @@ +"""Output assembly and legacy-compatible JSON helpers.""" + +from friend_circle_lite.outputs.legacy_api import * # noqa: F401,F403 diff --git a/friend_circle_lite/outputs/legacy_api.py b/friend_circle_lite/outputs/legacy_api.py new file mode 100644 index 00000000000..e670335f5c6 --- /dev/null +++ b/friend_circle_lite/outputs/legacy_api.py @@ -0,0 +1,331 @@ +"""Legacy-compatible crawl entrypoints. + +The internal implementation is now delegated to `crawler_service`, but these +functions keep the existing public API stable for `run.py` and external users. +""" + +import logging +from datetime import datetime +from math import ceil + +import requests + +from friend_circle_lite import HEADERS_JSON, timeout +from friend_circle_lite.domain.models import normalize_latency +from friend_circle_lite.crawler.service import ( + FriendCircleCrawlService, + limit_large_dataset as _limit_large_dataset, + sort_articles_by_time as _sort_articles_by_time, +) + +def fetch_and_process_data( + json_url: str, + specific_RSS: list = None, + count: int = 5, + cache_file: str = None, + link_check_config=None, + proxy_settings=None, +): + """Legacy wrapper around the new crawler orchestration service.""" + return FriendCircleCrawlService( + json_url=json_url, + count=count, + specific_rss=specific_RSS, + cache_file=cache_file, + link_check_config=link_check_config, + proxy_settings=proxy_settings, + ).run() + +def sort_articles_by_time(data, future_tolerance_days=2): + """Legacy wrapper around the refactored sort helper.""" + return _sort_articles_by_time(data, future_tolerance_days=future_tolerance_days) + +def marge_data_from_json_url(data, marge_json_url): + """ + 从另一个 JSON 文件中获取数据并合并到原数据中。 + + 参数： + data (dict): 包含文章信息的字典 + marge_json_url (str): 包含另一个文章信息的 JSON 文件的 URL。 + + 返回： + dict: 合并后的文章信息字典，已去重处理 + """ + try: + response = requests.get(marge_json_url, headers=HEADERS_JSON, timeout=timeout) + marge_data = response.json() + except Exception as e: + logging.error(f"无法获取链接：{marge_json_url} ，出现的问题为：{e}", exc_info=True) + return data + + if 'article_data' in marge_data: + logging.info(f"开始合并文章数据，原数据共有 {len(data['article_data'])} 篇文章，第三方数据共有 {len(marge_data['article_data'])} 篇文章") + data['article_data'].extend(marge_data['article_data']) + data['article_data'] = list({v['link']:v for v in data['article_data']}.values()) + logging.info(f"合并文章数据完成，现在共有 {len(data['article_data'])} 篇文章") + return data + + +def merge_link_data_from_json_url(link_data, merge_json_url): + """ + 从另一个 link.json 文件中获取友链可达性数据并智能合并。 + + 合并策略： + - 可达性优先级：direct > proxy > api > none + - 延迟取最优（最小值） + - 反链取并集（任一为 true 则为 true） + - 失败次数取最小值 + + 参数： + link_data (dict): 本地友链数据，包含 statistical_data 和 link_data + merge_json_url (str): 远程 link.json 的 URL + + 返回： + dict: 合并后的友链数据 + """ + try: + response = requests.get(merge_json_url, headers=HEADERS_JSON, timeout=timeout) + remote_data = response.json() + except Exception as e: + logging.warning(f"无法获取友链数据：{merge_json_url} ，跳过友链数据合并。错误：{e}") + return link_data + + if not _extract_links(remote_data): + logging.warning("远程数据不包含可用友链字段，跳过友链数据合并") + return link_data + + local_links = [_normalize_merge_link(link) for link in _extract_links(link_data)] + remote_links = [_normalize_merge_link(link) for link in _extract_links(remote_data)] + + logging.info(f"开始合并友链数据，本地 {len(local_links)} 条，远程 {len(remote_links)} 条") + + # 按 URL 建立索引 + link_map = {link["url"]: link for link in local_links} + + for remote_link in remote_links: + url = remote_link["url"] + if url not in link_map: + # 新友链，直接添加 + link_map[url] = remote_link + else: + # 已存在，智能合并 + local_link = link_map[url] + link_map[url] = _merge_single_link(local_link, remote_link) + + # 重新计算统计数据 + merged_records = list(link_map.values()) + merged_stats = _recalculate_link_statistics(merged_records) + merged_links = [_to_public_link(link) for link in merged_records] + logging.info(f"合并友链数据完成，共有 {len(merged_links)} 条友链") + local_stats = _extract_stats(link_data) + remote_stats = _extract_stats(remote_data) + checked_times = [ + local_stats.get("checked", "") or local_stats.get("link_last_checked_time", ""), + remote_stats.get("checked", "") or remote_stats.get("link_last_checked_time", ""), + merged_stats.get("link_last_checked_time", ""), + ] + merged_stats["link_last_checked_time"] = max([item for item in checked_times if item] or [""]) + + return { + 'statistical_data': merged_stats, + 'link_data': merged_links, + } + + +def _merge_single_link(local, remote): + """ + 合并单条友链数据，优先选择更好的检测结果。 + + 优先级： + 1. 可达性：direct > proxy > api > none + 2. 延迟：取最小值 + 3. 反链：任一为 true 则为 true + 4. 失败次数：取最小值 + """ + method_priority = {'direct': 4, 'proxy': 3, 'api': 2, 'disabled': 1, 'none': 0, '': 0} + + local_priority = _link_priority(local, method_priority) + remote_priority = _link_priority(remote, method_priority) + + # 选择优先级更高的作为基础 + if remote_priority > local_priority: + base = remote.copy() + alt = local + elif remote_priority < local_priority: + base = local.copy() + alt = remote + else: + # 优先级相同，选择延迟更低的 + local_latency = local.get('latency', 999) + remote_latency = remote.get('latency', 999) + if remote_latency >= 0 and (local_latency < 0 or remote_latency < local_latency): + base = remote.copy() + alt = local + else: + base = local.copy() + alt = remote + + # 反链取并集 + local_backlink = local.get('backlink') + remote_backlink = remote.get('backlink') + if local_backlink is True or remote_backlink is True: + base['backlink'] = True + elif local_backlink is False and remote_backlink is False: + base['backlink'] = False + # 否则保持 base 的值 + + unreachable_since_values = [ + value for value in (local.get("unreachable_since"), remote.get("unreachable_since")) + if value + ] + base["unreachable_since"] = min(unreachable_since_values) if unreachable_since_values else "" + + # 最新文章时间取更新的一侧，天数取更小的一侧 + if remote.get("updated", "") > local.get("updated", ""): + base["updated"] = remote.get("updated", "") + elif local.get("updated"): + base["updated"] = local.get("updated", "") + stale_values = [value for value in (local.get("stale_days"), remote.get("stale_days")) if value is not None] + base["stale_days"] = min(stale_values) if stale_values else None + + # 检测时间取最新 + local_checked = local.get('_checked', '') + remote_checked = remote.get('_checked', '') + if remote_checked > local_checked: + base['_checked'] = remote_checked + + return base + + +def _link_priority(link, method_priority): + """计算友链合并优先级，兼容新旧 link.json 字段。""" + if link.get("rss"): + return 10 + method_priority.get(link.get("_method", ""), 0) + if link.get("ok"): + return 5 + method_priority.get(link.get("_method", ""), 0) + return method_priority.get(link.get("_method", ""), 0) + + +def _to_public_link(link): + """转换为前端需要的精简友链状态结构。""" + normalized = _normalize_merge_link(link) + latency = normalize_latency(normalized.get("latency")) if normalized.get("ok") else -1 + return { + "name": normalized.get("name", ""), + "link": normalized.get("url", ""), + "link_page": normalized.get("page", ""), + "avatar": normalized.get("avatar", ""), + "reachable": bool(normalized.get("ok")), + "crawlable": bool(normalized.get("rss")), + "latency": latency, + "unreachable_days": _calculate_unreachable_days(normalized.get("unreachable_since")) if not normalized.get("ok") else None, + "unreachable_since": normalized.get("unreachable_since", "") if not normalized.get("ok") else "", + "has_backlink": normalized.get("backlink"), + "updated": normalized.get("updated", ""), + "stale_days": normalized.get("stale_days"), + } + + +def _extract_links(payload): + """读取新旧 link.json 列表字段。""" + return payload.get("links") or payload.get("link_data") or [] + + +def _extract_stats(payload): + """读取新旧 link.json 统计字段。""" + return payload.get("stats") or payload.get("statistical_data") or {} + + +def _normalize_merge_link(link): + """把新旧单条友链结构统一为内部紧凑结构。""" + ok = link.get("ok") if "ok" in link else link.get("reachable") + rss = link.get("rss") if "rss" in link else link.get("crawlable", link.get("crawl_allowed")) + return { + "name": link.get("name", ""), + "url": link.get("url") or link.get("link") or "", + "page": link.get("page") or link.get("link_page") or link.get("linkpage", ""), + "avatar": link.get("avatar", ""), + "ok": bool(ok), + "rss": bool(rss), + "latency": link.get("latency", link.get("best_latency", -1)), + "unreachable_since": link.get("unreachable_since", ""), + "backlink": link.get("backlink") if "backlink" in link else link.get("has_backlink"), + "updated": link.get("updated") or link.get("last_post_published", ""), + "stale_days": link.get("stale_days", link.get("last_post_days_ago")), + "_method": link.get("method") or link.get("best_method", ""), + "_checked": link.get("checked_at", ""), + } + + +def _calculate_unreachable_days(unreachable_since: str) -> int | None: + """Calculate rounded-up unreachable days from a persisted start time.""" + if not unreachable_since: + return None + try: + since = datetime.strptime(unreachable_since, "%Y-%m-%d %H:%M:%S") + except ValueError: + return None + elapsed_seconds = max(0, (datetime.now() - since).total_seconds()) + return max(1, ceil(elapsed_seconds / 86400)) + + +def _recalculate_link_statistics(links): + """重新计算合并后的友链统计数据。""" + reachable = [link for link in links if link.get('ok')] + crawl_allowed = [link for link in links if link.get('rss')] + api_only = [link for link in links if link.get('_method') == 'api'] + has_backlink = [link for link in links if link.get('backlink') is True] + checked_times = [link.get('_checked', '') for link in links if link.get('_checked')] + + return { + 'link_total_num': len(links), + 'link_reachable_num': len(reachable), + 'link_unreachable_num': len(links) - len(reachable), + 'crawl_allowed_num': len(crawl_allowed), + 'api_only_num': len(api_only), + 'has_author_link_num': len(has_backlink), + 'link_last_checked_time': max(checked_times) if checked_times else '', + } + + +def marge_errors_from_json_url(errors, marge_json_url): + """ + 从另一个网络 JSON 文件中获取错误信息并遍历，删除在errors中， + 不存在于marge_errors中的友链信息。 + + 参数： + errors (list): 包含错误信息的列表 + marge_json_url (str): 包含另一个错误信息的 JSON 文件的 URL。 + + 返回： + list: 合并后的错误信息列表 + """ + try: + response = requests.get(marge_json_url, timeout=10) # 设置请求超时时间 + marge_errors = response.json() + except Exception as e: + logging.error(f"无法获取链接：{marge_json_url} ，出现的问题为：{e}", exc_info=True) + return errors + + # 提取 marge_errors 中的 URL + marge_urls = {item[1] for item in marge_errors} + + # 使用过滤器保留 errors 中在 marge_errors 中出现的 URL + filtered_errors = [error for error in errors if error[1] in marge_urls] + + logging.info(f"合并错误信息完成，合并后共有 {len(filtered_errors)} 位朋友") + return filtered_errors + +def deal_with_large_data(result, future_tolerance_days=2): + """Legacy wrapper around the refactored dataset trimming helper.""" + return _limit_large_dataset(result, future_tolerance_days=future_tolerance_days) + + +def merge_data_from_json_url(data, merge_json_url): + """Correctly spelled wrapper for the legacy article merge helper.""" + return marge_data_from_json_url(data, merge_json_url) + + +def merge_errors_from_json_url(errors, merge_json_url): + """Correctly spelled wrapper for the legacy error merge helper.""" + return marge_errors_from_json_url(errors, merge_json_url) diff --git a/friend_circle_lite/single_friend.py b/friend_circle_lite/single_friend.py new file mode 100644 index 00000000000..677fc18b65a --- /dev/null +++ b/friend_circle_lite/single_friend.py @@ -0,0 +1,6 @@ +"""Backward-compatible single-site crawl helpers. + +New code should import from `friend_circle_lite.crawler.single_site_legacy`. +""" + +from friend_circle_lite.crawler.single_site_legacy import * # noqa: F401,F403 diff --git a/friend_circle_lite/storage/__init__.py b/friend_circle_lite/storage/__init__.py new file mode 100644 index 00000000000..75aad775e0b --- /dev/null +++ b/friend_circle_lite/storage/__init__.py @@ -0,0 +1,4 @@ +"""Persistent stores for feed cache, article tracking, and link checks.""" + +from friend_circle_lite.storage.sqlite_store import ArticleTrackingStore, FeedCacheStore, LinkCheckStore +from friend_circle_lite.storage.diagnostics import SQLiteDebugDumper diff --git a/friend_circle_lite/storage/diagnostics.py b/friend_circle_lite/storage/diagnostics.py new file mode 100644 index 00000000000..ad5abc730c7 --- /dev/null +++ b/friend_circle_lite/storage/diagnostics.py @@ -0,0 +1,260 @@ +"""SQLite 缓存诊断与安全整理。 + +本文件只负责调试期开启的 SQLite 检查： +- 输出当前数据库中的全部表结构与全部数据； +- 对项目内部核心表执行保守 schema 整理，移除残留旧字段； +- 不自动删除未知表，避免误删用户额外保存的数据。 +""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +from contextlib import closing +from pathlib import Path + + +EXPECTED_SCHEMAS: dict[str, tuple[list[str], str]] = { + "feed_cache": ( + ["name", "url", "source"], + """ + CREATE TABLE feed_cache ( + name TEXT PRIMARY KEY, + url TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'cache' + ) + """, + ), + "article_tracking": ( + ["id", "title", "author", "link", "published", "summary", "content"], + """ + CREATE TABLE article_tracking ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + author TEXT NOT NULL, + link TEXT NOT NULL, + published TEXT NOT NULL, + summary TEXT, + content TEXT + ) + """, + ), + "link_check_state": ( + [ + "url", + "name", + "avatar", + "linkpage", + "checked_at", + "reachable", + "crawl_allowed", + "best_method", + "best_latency", + "fail_count", + "backlink_checked", + "has_author_link", + "rss_crawl_reason", + "last_post_published", + "last_post_days_ago", + "unreachable_since", + "rss_unavailable_since", + "direct_success", + "direct_status_code", + "direct_latency", + "proxy_success", + "proxy_status_code", + "proxy_latency", + "api_success", + "api_status_code", + "api_latency", + ], + """ + CREATE TABLE link_check_state ( + url TEXT PRIMARY KEY, + name TEXT NOT NULL, + avatar TEXT DEFAULT '', + linkpage TEXT DEFAULT '', + checked_at TEXT NOT NULL, + reachable INTEGER NOT NULL DEFAULT 0, + crawl_allowed INTEGER NOT NULL DEFAULT 0, + best_method TEXT NOT NULL DEFAULT 'none', + best_latency REAL DEFAULT -1, + fail_count INTEGER NOT NULL DEFAULT 0, + backlink_checked INTEGER NOT NULL DEFAULT 0, + has_author_link INTEGER NOT NULL DEFAULT 0, + rss_crawl_reason TEXT NOT NULL DEFAULT '', + last_post_published TEXT DEFAULT '', + last_post_days_ago INTEGER, + unreachable_since TEXT DEFAULT '', + rss_unavailable_since TEXT DEFAULT '', + direct_success INTEGER NOT NULL DEFAULT 0, + direct_status_code INTEGER, + direct_latency REAL DEFAULT -1, + proxy_success INTEGER NOT NULL DEFAULT 0, + proxy_status_code INTEGER, + proxy_latency REAL DEFAULT -1, + api_success INTEGER NOT NULL DEFAULT 0, + api_status_code INTEGER, + api_latency REAL DEFAULT -1 + ) + """, + ), +} + +DEFAULT_EXPRESSIONS: dict[str, str] = { + "id": "NULL", + "name": "''", + "url": "''", + "source": "'cache'", + "title": "''", + "author": "''", + "link": "''", + "published": "''", + "summary": "NULL", + "content": "NULL", + "avatar": "''", + "linkpage": "''", + "checked_at": "''", + "reachable": "0", + "crawl_allowed": "0", + "best_method": "'none'", + "best_latency": "-1", + "fail_count": "0", + "backlink_checked": "0", + "has_author_link": "0", + "rss_crawl_reason": "''", + "last_post_published": "''", + "last_post_days_ago": "NULL", + "unreachable_since": "''", + "rss_unavailable_since": "''", + "direct_success": "0", + "direct_status_code": "NULL", + "direct_latency": "-1", + "proxy_success": "0", + "proxy_status_code": "NULL", + "proxy_latency": "-1", + "api_success": "0", + "api_status_code": "NULL", + "api_latency": "-1", +} + + +class SQLiteDebugDumper: + """打印 SQLite 全量调试信息，并清理核心表中的旧字段。""" + + def __init__(self, database_path: str | Path | None): + self.database_path = Path(database_path) if database_path else None + + def run(self) -> str: + """执行 schema 检查、保守清理与全量数据输出。""" + lines: list[str] = [] + self._append(lines, "=" * 60) + self._append(lines, "SQLite 调试信息") + self._append(lines, "=" * 60) + + if not self.database_path: + self._append(lines, "未配置 SQLite 缓存路径，跳过调试输出") + return self._flush(lines) + + self._append(lines, f"数据库路径: {self.database_path}") + if not self.database_path.exists(): + self._append(lines, "数据库文件不存在，跳过调试输出") + return self._flush(lines) + + with closing(sqlite3.connect(self.database_path)) as connection: + connection.row_factory = sqlite3.Row + self._report_schema_state(connection, lines) + self._clean_known_tables(connection, lines) + self._dump_all_tables(connection, lines) + + return self._flush(lines) + + def _report_schema_state(self, connection: sqlite3.Connection, lines: list[str]) -> None: + tables = self._table_names(connection) + self._append(lines, f"当前表数量: {len(tables)}") + for table in tables: + columns = self._column_names(connection, table) + self._append(lines, f"表 {table} 字段: {', '.join(columns)}") + expected = EXPECTED_SCHEMAS.get(table) + if not expected: + self._append(lines, f"表 {table} 不是 Friend-Circle-Lite 核心表，保留不清理") + continue + extra_columns = [column for column in columns if column not in expected[0]] + missing_columns = [column for column in expected[0] if column not in columns] + if extra_columns: + self._append(lines, f"表 {table} 检测到旧字段: {', '.join(extra_columns)}") + if missing_columns: + self._append(lines, f"表 {table} 缺少当前字段，将使用默认值补齐: {', '.join(missing_columns)}") + + def _clean_known_tables(self, connection: sqlite3.Connection, lines: list[str]) -> None: + for table, (expected_columns, create_sql) in EXPECTED_SCHEMAS.items(): + if table not in self._table_names(connection): + connection.execute(create_sql) + self._append(lines, f"表 {table} 不存在，已按当前 schema 创建") + continue + + current_columns = self._column_names(connection, table) + if current_columns == expected_columns: + self._append(lines, f"表 {table} schema 已匹配，无需清理") + continue + + temp_table = f"__fcl_rebuild_{table}" + connection.execute(f"DROP TABLE IF EXISTS {temp_table}") + connection.execute(create_sql.replace(f"CREATE TABLE {table}", f"CREATE TABLE {temp_table}", 1)) + + common_columns = [column for column in expected_columns if column in current_columns] + insert_columns = ", ".join(self._quote_identifier(column) for column in expected_columns) + select_expressions = ", ".join( + self._quote_identifier(column) if column in current_columns else DEFAULT_EXPRESSIONS[column] + for column in expected_columns + ) + connection.execute( + f"INSERT INTO {self._quote_identifier(temp_table)} ({insert_columns}) " + f"SELECT {select_expressions} FROM {self._quote_identifier(table)}" + ) + + connection.execute(f"DROP TABLE {self._quote_identifier(table)}") + connection.execute( + f"ALTER TABLE {self._quote_identifier(temp_table)} RENAME TO {self._quote_identifier(table)}" + ) + self._append(lines, f"表 {table} 已重建为当前 schema，保留字段: {', '.join(common_columns)}") + connection.commit() + + def _dump_all_tables(self, connection: sqlite3.Connection, lines: list[str]) -> None: + tables = self._table_names(connection) + self._append(lines, "SQLite 全量数据开始") + for table in tables: + rows = connection.execute(f"SELECT * FROM {self._quote_identifier(table)}").fetchall() + self._append(lines, f"表 {table} 行数: {len(rows)}") + for index, row in enumerate(rows, start=1): + row_data = {key: row[key] for key in row.keys()} + self._append(lines, f"表 {table} 第 {index} 行: {json.dumps(row_data, ensure_ascii=False)}") + self._append(lines, "SQLite 全量数据结束") + + @staticmethod + def _table_names(connection: sqlite3.Connection) -> list[str]: + rows = connection.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name" + ).fetchall() + return [row[0] for row in rows] + + @staticmethod + def _column_names(connection: sqlite3.Connection, table: str) -> list[str]: + rows = connection.execute(f"PRAGMA table_info({SQLiteDebugDumper._quote_identifier(table)})").fetchall() + return [row[1] for row in rows] + + @staticmethod + def _quote_identifier(identifier: str) -> str: + return '"' + identifier.replace('"', '""') + '"' + + @staticmethod + def _append(lines: list[str], message: str) -> None: + lines.append(message) + + @staticmethod + def _flush(lines: list[str]) -> str: + output = "\n".join(lines) + for line in lines: + logging.info(line) + return output diff --git a/friend_circle_lite/storage/sqlite_store.py b/friend_circle_lite/storage/sqlite_store.py new file mode 100644 index 00000000000..90086bb441e --- /dev/null +++ b/friend_circle_lite/storage/sqlite_store.py @@ -0,0 +1,521 @@ +"""Persistent RSS cache and article tracking storage. + +SQLite is used for both feed cache and article tracking because it is more robust +than hand-edited text formats for internal state: + +- schema is explicit and stable; +- writes are transactional; +- corruption risk from accidental manual edits is lower; +- Python ships with `sqlite3`, so no extra dependency is required. + +For smooth upgrades, this store can also migrate legacy cache data from the old +JSON cache file and the intermediate YAML cache file if they exist. +""" + +from __future__ import annotations + +import json +import logging +import sqlite3 +from contextlib import closing +from datetime import datetime +from pathlib import Path + +import yaml + +from friend_circle_lite.domain.models import Article, CacheRecord, LinkCheckRecord, LinkMethodStatus, normalize_homepage_url + + +class FeedCacheStore: + """Persist and load discovered RSS endpoints using SQLite.""" + + def __init__(self, cache_path: str | Path | None): + self.cache_path = Path(cache_path) if cache_path else None + + def load_records(self) -> list[CacheRecord]: + """Load cache records from SQLite, migrating legacy formats if needed.""" + if not self.cache_path: + return [] + + if self.cache_path.exists(): + return self._load_from_sqlite() + + migrated_records = self._load_legacy_records() + if migrated_records: + if self.save_records(migrated_records): + logging.info(f"[RSS 缓存] 已从旧格式迁移 {len(migrated_records)} 条 RSS 缓存到 SQLite") + return migrated_records + + logging.info("[RSS 缓存] RSS 缓存文件不存在，将在首次抓取后自动创建") + return [] + + def save_records(self, records: list[CacheRecord]) -> bool: + """Persist cache records to the SQLite database.""" + if not self.cache_path: + return True + + try: + self.cache_path.parent.mkdir(parents=True, exist_ok=True) + with closing(sqlite3.connect(self.cache_path)) as connection: + self._ensure_schema(connection) + connection.execute("DELETE FROM feed_cache") + connection.executemany( + "INSERT INTO feed_cache(name, url, source) VALUES (?, ?, ?)", + [(record.name, record.url, record.source) for record in sorted(records, key=lambda item: item.name)], + ) + connection.commit() + logging.info(f"[RSS 缓存] RSS 缓存已保存（{len(records)} 条）") + return True + except Exception as exc: + logging.error(f"[RSS 缓存] 保存 RSS 缓存失败: {exc}") + return False + + def _load_from_sqlite(self) -> list[CacheRecord]: + """Load records from the current SQLite cache file.""" + try: + with closing(sqlite3.connect(self.cache_path)) as connection: + self._ensure_schema(connection) + connection.commit() + rows = connection.execute( + "SELECT name, url, source FROM feed_cache ORDER BY name" + ).fetchall() + except Exception as exc: + logging.warning(f"[RSS 缓存] 读取 RSS 缓存失败: {exc}") + return [] + + return [ + CacheRecord(name=name, url=url, source=source or "cache") + for name, url, source in rows + if name and url + ] + + @staticmethod + def _ensure_schema(connection: sqlite3.Connection) -> None: + """Create the cache table when it does not exist yet.""" + connection.execute( + """ + CREATE TABLE IF NOT EXISTS feed_cache ( + name TEXT PRIMARY KEY, + url TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'cache' + ) + """ + ) + + def _load_legacy_records(self) -> list[CacheRecord]: + """Read old cache formats for seamless upgrades.""" + json_records = self._load_legacy_json_cache() + if json_records: + return json_records + + yaml_records = self._load_legacy_yaml_cache() + if yaml_records: + return yaml_records + + return [] + + def _load_legacy_json_cache(self) -> list[CacheRecord]: + """Read the previous JSON cache file format.""" + if not self.cache_path: + return [] + + legacy_path = self.cache_path.with_name("cache.json") + if not legacy_path.exists(): + return [] + + try: + with open(legacy_path, "r", encoding="utf-8") as file: + payload = json.load(file) + except Exception as exc: + logging.warning(f"读取旧 JSON 缓存失败: {exc}") + return [] + + if not isinstance(payload, list): + return [] + + return self._normalize_legacy_items(payload) + + def _load_legacy_yaml_cache(self) -> list[CacheRecord]: + """Read the temporary YAML cache format used during refactoring.""" + if not self.cache_path: + return [] + + legacy_path = self.cache_path.with_name("feed_cache.yaml") + if not legacy_path.exists(): + return [] + + try: + with open(legacy_path, "r", encoding="utf-8") as file: + payload = yaml.safe_load(file) or {} + except Exception as exc: + logging.warning(f"读取旧 YAML 缓存失败: {exc}") + return [] + + items = payload.get("feeds", []) if isinstance(payload, dict) else [] + return self._normalize_legacy_items(items) + + @staticmethod + def _normalize_legacy_items(items: list[object]) -> list[CacheRecord]: + """Normalize legacy cache items into typed cache records.""" + records: list[CacheRecord] = [] + for item in items: + if not isinstance(item, dict): + continue + name = str(item.get("name", "")).strip() + url = str(item.get("url", "")).strip() + source = str(item.get("source", "cache")).strip() or "cache" + if name and url: + records.append(CacheRecord(name=name, url=url, source=source)) + return records + + +class ArticleTrackingStore: + """Persist and load article tracking data using SQLite.""" + + def __init__(self, storage_path: str | Path | None, max_tracked_articles: int = 10): + self.storage_path = Path(storage_path) if storage_path else None + self.max_tracked_articles = max_tracked_articles + + def load_articles(self) -> list[Article]: + """Load tracked articles from SQLite, migrating from legacy JSON if needed.""" + if not self.storage_path: + return [] + + if self.storage_path.exists(): + return self._load_from_sqlite() + + # Try to migrate from legacy JSON format + migrated_articles = self._load_legacy_json() + if migrated_articles: + if self.save_articles(migrated_articles): + logging.info(f"[文章追踪] 已从旧 JSON 格式迁移 {len(migrated_articles)} 篇文章记录到 SQLite") + return migrated_articles + + logging.info("[文章追踪] 文章追踪数据不存在，这是首次运行") + return [] + + def save_articles(self, articles: list[Article]) -> bool: + """Persist articles to SQLite, keeping only the most recent max_tracked_articles.""" + if not self.storage_path: + return True + + try: + # Sort by date and keep only the most recent articles + valid_articles = [article for article in articles if article.published] + valid_articles.sort( + key=lambda item: datetime.strptime(item.published, "%Y-%m-%d %H:%M"), + reverse=True + ) + articles_to_save = valid_articles[:self.max_tracked_articles] + + self.storage_path.parent.mkdir(parents=True, exist_ok=True) + with closing(sqlite3.connect(self.storage_path)) as connection: + self._ensure_schema(connection) + connection.execute("DELETE FROM article_tracking") + connection.executemany( + """INSERT INTO article_tracking(title, author, link, published, summary, content) + VALUES (?, ?, ?, ?, ?, ?)""", + [ + ( + article.title, + article.author, + article.link, + article.published, + article.summary, + article.content, + ) + for article in articles_to_save + ], + ) + connection.commit() + return True + except Exception as exc: + logging.error(f"[文章追踪] 保存文章追踪数据失败: {exc}") + return False + + def _load_from_sqlite(self) -> list[Article]: + """Load articles from the SQLite database.""" + try: + with closing(sqlite3.connect(self.storage_path)) as connection: + self._ensure_schema(connection) + connection.commit() + rows = connection.execute( + """SELECT title, author, link, published, summary, content + FROM article_tracking + ORDER BY published DESC""" + ).fetchall() + except Exception as exc: + logging.warning(f"[文章追踪] 读取文章追踪数据失败: {exc}") + return [] + + return [ + Article( + title=title or "", + author=author or "", + link=link or "", + published=published or "", + summary=summary or "", + content=content or "", + ) + for title, author, link, published, summary, content in rows + ] + + @staticmethod + def _ensure_schema(connection: sqlite3.Connection) -> None: + """Create the article tracking table when it does not exist yet.""" + connection.execute( + """ + CREATE TABLE IF NOT EXISTS article_tracking ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + title TEXT NOT NULL, + author TEXT NOT NULL, + link TEXT NOT NULL, + published TEXT NOT NULL, + summary TEXT, + content TEXT + ) + """ + ) + + def _load_legacy_json(self) -> list[Article]: + """Read the old JSON format for seamless upgrades.""" + if not self.storage_path: + return [] + + legacy_path = self.storage_path.with_name("newest_posts.json") + if not legacy_path.exists(): + return [] + + try: + with open(legacy_path, "r", encoding="utf-8") as file: + payload = json.load(file) + except Exception as exc: + logging.warning(f"读取旧 JSON 文章追踪文件失败: {exc}") + return [] + + articles_data = payload.get("articles", []) if isinstance(payload, dict) else [] + articles: list[Article] = [] + for item in articles_data: + if not isinstance(item, dict): + continue + articles.append( + Article( + title=item.get("title", ""), + author=item.get("author", ""), + link=item.get("link", ""), + published=item.get("published", ""), + summary=item.get("summary", ""), + content=item.get("content", ""), + ) + ) + return articles + + +class LinkCheckStore: + """Persist friend link reachability checks using SQLite.""" + + def __init__(self, cache_path: str | Path | None): + self.cache_path = Path(cache_path) if cache_path else None + + def load_records(self, urls: list[str] | None = None) -> dict[str, LinkCheckRecord]: + if not self.cache_path or not self.cache_path.exists(): + return {} + + try: + with closing(sqlite3.connect(self.cache_path)) as connection: + self._ensure_schema(connection) + connection.commit() + rows = connection.execute( + """ + SELECT url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, + best_method, best_latency, fail_count, backlink_checked, has_author_link, + rss_crawl_reason, last_post_published, last_post_days_ago, + unreachable_since, rss_unavailable_since, + direct_success, direct_status_code, direct_latency, + proxy_success, proxy_status_code, proxy_latency, api_success, + api_status_code, api_latency + FROM link_check_state + """ + ).fetchall() + except Exception as exc: + logging.warning(f"[友链检测] 读取友链检测缓存失败: {exc}") + return {} + + allowed_urls = {normalize_homepage_url(url) for url in (urls or [])} + records: dict[str, LinkCheckRecord] = {} + for row in rows: + ( + url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, + best_method, best_latency, _legacy_fail_count, backlink_checked, has_author_link, + rss_crawl_reason, last_post_published, last_post_days_ago, + unreachable_since, rss_unavailable_since, + direct_success, direct_status_code, direct_latency, + proxy_success, proxy_status_code, proxy_latency, api_success, + api_status_code, api_latency, + ) = row + normalized_url = normalize_homepage_url(url or "") + if allowed_urls and normalized_url not in allowed_urls: + continue + records[normalized_url] = LinkCheckRecord( + name=name or "", + url=normalized_url, + avatar=avatar or "", + linkpage=linkpage or "", + checked_at=checked_at or "", + reachable=bool(reachable), + crawl_allowed=bool(crawl_allowed), + best_method=best_method or "none", + best_latency=best_latency if best_latency is not None else -1, + backlink_checked=bool(backlink_checked), + has_author_link=bool(has_author_link), + rss_crawl_reason=rss_crawl_reason or "blocked_unreachable", + last_post_published=last_post_published or "", + last_post_days_ago=last_post_days_ago, + unreachable_since=unreachable_since or (checked_at or "" if not bool(reachable) else ""), + rss_unavailable_since=rss_unavailable_since or (checked_at or "" if bool(reachable) and not bool(crawl_allowed) else ""), + direct=LinkMethodStatus(bool(direct_success), direct_status_code, direct_latency if direct_latency is not None else -1), + proxy=LinkMethodStatus(bool(proxy_success), proxy_status_code, proxy_latency if proxy_latency is not None else -1), + api=LinkMethodStatus(bool(api_success), api_status_code, api_latency if api_latency is not None else -1), + ) + return records + + def save_records(self, records: list[LinkCheckRecord]) -> bool: + if not self.cache_path: + return True + + try: + self.cache_path.parent.mkdir(parents=True, exist_ok=True) + with closing(sqlite3.connect(self.cache_path)) as connection: + self._ensure_schema(connection) + connection.executemany( + """ + INSERT INTO link_check_state( + url, name, avatar, linkpage, checked_at, reachable, crawl_allowed, + best_method, best_latency, fail_count, backlink_checked, has_author_link, + rss_crawl_reason, last_post_published, last_post_days_ago, + unreachable_since, rss_unavailable_since, + direct_success, direct_status_code, direct_latency, + proxy_success, proxy_status_code, proxy_latency, api_success, + api_status_code, api_latency + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(url) DO UPDATE SET + name = excluded.name, + avatar = excluded.avatar, + linkpage = excluded.linkpage, + checked_at = excluded.checked_at, + reachable = excluded.reachable, + crawl_allowed = excluded.crawl_allowed, + best_method = excluded.best_method, + best_latency = excluded.best_latency, + fail_count = excluded.fail_count, + backlink_checked = excluded.backlink_checked, + has_author_link = excluded.has_author_link, + rss_crawl_reason = excluded.rss_crawl_reason, + last_post_published = excluded.last_post_published, + last_post_days_ago = excluded.last_post_days_ago, + unreachable_since = excluded.unreachable_since, + rss_unavailable_since = excluded.rss_unavailable_since, + direct_success = excluded.direct_success, + direct_status_code = excluded.direct_status_code, + direct_latency = excluded.direct_latency, + proxy_success = excluded.proxy_success, + proxy_status_code = excluded.proxy_status_code, + proxy_latency = excluded.proxy_latency, + api_success = excluded.api_success, + api_status_code = excluded.api_status_code, + api_latency = excluded.api_latency + """, + [self._record_to_row(record) for record in records], + ) + connection.commit() + logging.info(f"[友链检测] 友链检测缓存已保存（{len(records)} 条）") + return True + except Exception as exc: + logging.error(f"[友链检测] 保存友链检测缓存失败: {exc}") + return False + + @staticmethod + def is_fresh(record: LinkCheckRecord, max_age_hours: int) -> bool: + if not record.checked_at: + return False + try: + checked_at = datetime.strptime(record.checked_at, "%Y-%m-%d %H:%M:%S") + except ValueError: + return False + age_seconds = (datetime.now() - checked_at).total_seconds() + return age_seconds < max_age_hours * 3600 + + @staticmethod + def _record_to_row(record: LinkCheckRecord) -> tuple: + return ( + normalize_homepage_url(record.url), + record.name, + record.avatar, + record.linkpage, + record.checked_at, + int(record.reachable), + int(record.crawl_allowed), + record.best_method, + record.best_latency, + 0, + int(record.backlink_checked), + int(record.has_author_link), + record.rss_crawl_reason, + record.last_post_published, + record.last_post_days_ago, + record.unreachable_since, + record.rss_unavailable_since, + int(record.direct.success), + record.direct.status_code, + record.direct.latency, + int(record.proxy.success), + record.proxy.status_code, + record.proxy.latency, + int(record.api.success), + record.api.status_code, + record.api.latency, + ) + + @staticmethod + def _ensure_schema(connection: sqlite3.Connection) -> None: + connection.execute( + """ + CREATE TABLE IF NOT EXISTS link_check_state ( + url TEXT PRIMARY KEY, + name TEXT NOT NULL, + avatar TEXT DEFAULT '', + linkpage TEXT DEFAULT '', + checked_at TEXT NOT NULL, + reachable INTEGER NOT NULL DEFAULT 0, + crawl_allowed INTEGER NOT NULL DEFAULT 0, + best_method TEXT NOT NULL DEFAULT 'none', + best_latency REAL DEFAULT -1, + fail_count INTEGER NOT NULL DEFAULT 0, + backlink_checked INTEGER NOT NULL DEFAULT 0, + has_author_link INTEGER NOT NULL DEFAULT 0, + rss_crawl_reason TEXT NOT NULL DEFAULT '', + last_post_published TEXT DEFAULT '', + last_post_days_ago INTEGER, + unreachable_since TEXT DEFAULT '', + rss_unavailable_since TEXT DEFAULT '', + direct_success INTEGER NOT NULL DEFAULT 0, + direct_status_code INTEGER, + direct_latency REAL DEFAULT -1, + proxy_success INTEGER NOT NULL DEFAULT 0, + proxy_status_code INTEGER, + proxy_latency REAL DEFAULT -1, + api_success INTEGER NOT NULL DEFAULT 0, + api_status_code INTEGER, + api_latency REAL DEFAULT -1 + ) + """ + ) + columns = {row[1] for row in connection.execute("PRAGMA table_info(link_check_state)").fetchall()} + if "last_post_published" not in columns: + connection.execute("ALTER TABLE link_check_state ADD COLUMN last_post_published TEXT DEFAULT ''") + if "last_post_days_ago" not in columns: + connection.execute("ALTER TABLE link_check_state ADD COLUMN last_post_days_ago INTEGER") + if "unreachable_since" not in columns: + connection.execute("ALTER TABLE link_check_state ADD COLUMN unreachable_since TEXT DEFAULT ''") + if "rss_unavailable_since" not in columns: + connection.execute("ALTER TABLE link_check_state ADD COLUMN rss_unavailable_since TEXT DEFAULT ''") diff --git a/push_rss_update/__init__.py b/friend_circle_lite/utils/__init__.py similarity index 100% rename from push_rss_update/__init__.py rename to friend_circle_lite/utils/__init__.py diff --git a/friend_circle_lite/utils/cache.py b/friend_circle_lite/utils/cache.py new file mode 100644 index 00000000000..6647090081a --- /dev/null +++ b/friend_circle_lite/utils/cache.py @@ -0,0 +1,24 @@ +"""Backward-compatible cache helpers. + +The crawler now stores feed cache records as YAML through `FeedCacheStore`. +These wrappers keep the old function names available for any external callers. +""" + +from friend_circle_lite.domain.models import CacheRecord +from friend_circle_lite.storage.sqlite_store import FeedCacheStore + + +def load_cache(cache_file: str): + """Load cache records and expose the legacy list-of-dicts structure.""" + records = FeedCacheStore(cache_file).load_records() + return [{"name": item.name, "url": item.url, "source": item.source} for item in records] + + +def save_cache(cache_file: str, cache_items: list[dict]): + """Persist cache records while accepting the legacy input structure.""" + records = [ + CacheRecord(name=item["name"], url=item["url"], source=item.get("source", "cache")) + for item in cache_items + if item.get("name") and item.get("url") + ] + return FeedCacheStore(cache_file).save_records(records) diff --git a/friend_circle_lite/utils/config.py b/friend_circle_lite/utils/config.py new file mode 100644 index 00000000000..3a4d92d8be7 --- /dev/null +++ b/friend_circle_lite/utils/config.py @@ -0,0 +1,100 @@ +"""Configuration loading utilities.""" + +from __future__ import annotations + +import copy +import logging +import os + +import yaml + +from friend_circle_lite.config.models import ApplicationConfig + + +CONFIG_OVERRIDE_ENV_NAME = "FCL_CONFIG_OVERRIDES" + + +def _config_overrides_from_env() -> str: + """Return the first configured environment override block.""" + return os.getenv(CONFIG_OVERRIDE_ENV_NAME, "") + + +def _parse_config_overrides(raw: str) -> dict: + """Parse dotted YAML paths from a line-oriented environment value.""" + overrides: dict = {} + for line_number, line in enumerate(raw.splitlines(), start=1): + line = line.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + logging.warning("配置覆盖第 %s 行格式无效，已忽略", line_number) + continue + path, value_text = (part.strip() for part in line.split("=", 1)) + path_parts = [part.strip() for part in path.split(".") if part.strip()] + if not path_parts or any(part.startswith("__") for part in path_parts): + logging.warning("配置覆盖第 %s 行路径无效，已忽略", line_number) + continue + try: + value = yaml.safe_load(value_text) if value_text else "" + except yaml.YAMLError: + logging.warning("配置覆盖第 %s 行的值无法解析，已忽略", line_number) + continue + target = overrides + for part in path_parts[:-1]: + current = target.get(part) + if not isinstance(current, dict): + current = {} + target[part] = current + target = current + target[path_parts[-1]] = value + return overrides + + +def _apply_config_overrides(config: dict, raw: str) -> dict: + """Apply environment overrides without mutating the YAML-loaded object.""" + overrides = _parse_config_overrides(raw) + if not overrides: + return config + merged = copy.deepcopy(config) + + def merge(target: dict, source: dict) -> None: + for key, value in source.items(): + if isinstance(value, dict) and isinstance(target.get(key), dict): + merge(target[key], value) + else: + target[key] = value + + merge(merged, overrides) + logging.info("已应用 %s 个环境配置覆盖项", sum(1 for _ in _flatten_override_paths(overrides))) + return merged + + +def _flatten_override_paths(value: dict, prefix: str = ""): + for key, child in value.items(): + path = f"{prefix}.{key}" if prefix else key + if isinstance(child, dict): + yield from _flatten_override_paths(child, path) + else: + yield path + +def load_raw_config(config_file: str) -> dict: + """Load the raw YAML config dictionary from disk.""" + try: + with open(config_file, 'r', encoding='utf-8') as file: + config = yaml.safe_load(file) or {} + override_block = _config_overrides_from_env() + return _apply_config_overrides(config, override_block) if override_block else config + except FileNotFoundError: + logging.error(f"配置文件 {config_file} 未找到") + return {} + except yaml.YAMLError as e: + logging.error(f"YAML解析错误: {str(e)}") + return {} + except Exception as e: + logging.error(f"加载配置文件时发生未知错误: {str(e)}") + return {} + + +def load_config(config_file: str) -> ApplicationConfig: + """Load and validate the application configuration as typed objects.""" + return ApplicationConfig.from_dict(load_raw_config(config_file)) diff --git a/friend_circle_lite/utils/github.py b/friend_circle_lite/utils/github.py new file mode 100644 index 00000000000..fce69727ace --- /dev/null +++ b/friend_circle_lite/utils/github.py @@ -0,0 +1,6 @@ +"""Backward-compatible GitHub notification helpers. + +New code should import from `friend_circle_lite.notifications.github`. +""" + +from friend_circle_lite.notifications.github import * # noqa: F401,F403 diff --git a/friend_circle_lite/utils/json.py b/friend_circle_lite/utils/json.py new file mode 100644 index 00000000000..7a33d6c3355 --- /dev/null +++ b/friend_circle_lite/utils/json.py @@ -0,0 +1,59 @@ +import json +import logging +from pathlib import Path +from typing import Any, Optional + +_VOLATILE_COMPARE_KEYS = frozenset({ + "last_updated_time", + "link_last_checked_time", +}) + + +def read_json(file_path: str | Path) -> Optional[dict | list]: + """安全读取 JSON 文件，如果文件不存在或格式错误则返回 None""" + try: + with open(file_path, 'r', encoding='utf-8') as f: + return json.load(f) + except FileNotFoundError: + logging.warning(f"文件不存在: {file_path}") + return None + except json.JSONDecodeError: + logging.warning(f"JSON 格式错误: {file_path}") + return None + except Exception as e: + logging.warning(f"读取 JSON 文件时发生错误: {file_path}, 错误信息: {str(e)}") + return None + + +def _content_for_compare(value: Any) -> Any: + if isinstance(value, dict): + return { + key: _content_for_compare(item) + for key, item in value.items() + if key not in _VOLATILE_COMPARE_KEYS + } + if isinstance(value, list): + return [_content_for_compare(item) for item in value] + return value + + +def write_json(file_path: str | Path, data: Any) -> bool: + """安全写入 JSON 文件，返回是否写入成功""" + try: + path = Path(file_path) + if path.is_file(): + try: + with path.open('r', encoding='utf-8') as f: + existing = json.load(f) + if _content_for_compare(existing) == _content_for_compare(data): + return True + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + pass + + path.parent.mkdir(parents=True, exist_ok=True) + with path.open('w', encoding='utf-8') as f: + json.dump(data, f, ensure_ascii=False, separators=(',', ':')) + return True + except Exception as e: + logging.warning(f"写入 JSON 文件时发生错误: {file_path}, 错误信息: {str(e)}") + return False diff --git a/friend_circle_lite/utils/mail.py b/friend_circle_lite/utils/mail.py new file mode 100644 index 00000000000..ea8071b31f1 --- /dev/null +++ b/friend_circle_lite/utils/mail.py @@ -0,0 +1,6 @@ +"""Backward-compatible mail helpers. + +New code should import from `friend_circle_lite.notifications.mail`. +""" + +from friend_circle_lite.notifications.mail import * # noqa: F401,F403 diff --git a/friend_circle_lite/utils/time.py b/friend_circle_lite/utils/time.py new file mode 100644 index 00000000000..fc55391b3e8 --- /dev/null +++ b/friend_circle_lite/utils/time.py @@ -0,0 +1,42 @@ +import logging +from dateutil import parser +from datetime import datetime, timezone, timedelta + +def format_published_time(time_str): + """ + 格式化发布时间为统一格式 YYYY-MM-DD HH:MM + + 参数: + time_str (str): 输入的时间字符串，可能是多种格式。 + + 返回: + str: 格式化后的时间字符串，若解析失败返回空字符串。 + """ + # 尝试自动解析输入时间字符串 + try: + parsed_time = parser.parse(time_str, fuzzy=True) + except (ValueError, parser.ParserError): + # 定义支持的时间格式 + time_formats = [ + '%a, %d %b %Y %H:%M:%S %z', # Mon, 11 Mar 2024 14:08:32 +0000 + '%a, %d %b %Y %H:%M:%S GMT', # Wed, 19 Jun 2024 09:43:53 GMT + '%Y-%m-%dT%H:%M:%S%z', # 2024-03-11T14:08:32+00:00 + '%Y-%m-%dT%H:%M:%SZ', # 2024-03-11T14:08:32Z + '%Y-%m-%d %H:%M:%S', # 2024-03-11 14:08:32 + '%Y-%m-%d' # 2024-03-11 + ] + for fmt in time_formats: + try: + parsed_time = datetime.strptime(time_str, fmt) + break + except ValueError: + continue + else: + logging.warning(f"无法解析时间字符串：{time_str}") + return '' + + # 处理时区转换 + if parsed_time.tzinfo is None: + parsed_time = parsed_time.replace(tzinfo=timezone.utc) + shanghai_time = parsed_time.astimezone(timezone(timedelta(hours=8))) + return shanghai_time.strftime('%Y-%m-%d %H:%M') \ No newline at end of file diff --git a/friend_circle_lite/utils/url.py b/friend_circle_lite/utils/url.py new file mode 100644 index 00000000000..3674055be44 --- /dev/null +++ b/friend_circle_lite/utils/url.py @@ -0,0 +1,43 @@ +import logging +from urllib.parse import urlparse, urljoin +import re + +def replace_non_domain(link: str, blog_url: str) -> str: + """ + 检测并处理相对地址、非正常域名（如 IP 地址或 localhost）。 + - 如果是相对地址（无协议和域名），自动拼接 blog_url + - 如果是 localhost 或 IP 地址，替换为 blog_url + - 如果是正常的绝对地址，直接返回 + + :param link: 原始地址字符串 + :param blog_url: 博客的基础地址 + :return: 处理后的完整地址字符串 + """ + if not link: + return link + + try: + parsed = urlparse(link) + + # 情况1: 相对地址（没有 scheme 和 netloc） + # 例如: "/post/article.html" 或 "post/article.html" + if not parsed.scheme and not parsed.netloc: + # 使用 urljoin 来正确处理相对路径 + return urljoin(blog_url, link) + + # 情况2: localhost 或 IP 地址 + if 'localhost' in parsed.netloc or re.match(r'^\d{1,3}(\.\d{1,3}){3}$', parsed.netloc): + # 提取 path + query + fragment + path = parsed.path or '/' + if parsed.query: + path += '?' + parsed.query + if parsed.fragment: + path += '#' + parsed.fragment + return urljoin(blog_url.rstrip('/') + '/', path.lstrip('/')) + + # 情况3: 正常的绝对地址，直接返回 + return link + + except Exception as e: + logging.warning(f"替换链接时出错：{link}, error: {e}") + return link diff --git a/main/fclite.css b/main/fclite.css index 52567f3b095..a1efaed9b6a 100644 --- a/main/fclite.css +++ b/main/fclite.css @@ -47,74 +47,254 @@ #random-article { display: flex; + flex-direction: column; + gap: 16px; position: relative; width: 100%; - margin: 8px 0; + margin: 0 0 16px 0; background-color: var(--container-bg-color); - border-radius: 10px; - border:1px solid var(--border-color); - height: 210px; - transition: border 0.3s; + border-radius: 8px; + border: 1px solid var(--border-color); + padding: 20px; + transition: all 0.3s ease; + overflow: hidden; + box-sizing: border-box; } -.random-container { - position: relative; - margin: 20px; - width: 90%; - height: 170px; +.random-top { + width: 100%; +} + +.random-stats { + display: grid; + grid-template-columns: repeat(4, 1fr); + gap: 12px; +} + +.stat-item { + text-align: center; + padding: 12px; + background-color: var(--background-color); + border-radius: 6px; + border: 1px solid var(--border-color); + transition: all 0.3s ease; } -.random-container:hover .random-title { - font-size: 32px; +.stat-item:hover { + border-color: var(--hover-color); } -.random-author { - font-size: 14px; +.stat-num { + font-size: 24px; + font-weight: 700; + color: var(--hover-color); + line-height: 1; + margin-bottom: 4px; +} + +.stat-text { + font-size: 11px; color: var(--author-color); - margin-bottom: 10px; + text-transform: uppercase; + letter-spacing: 0.5px; +} + +.random-content { + display: flex; + align-items: flex-end; + justify-content: space-between; + gap: 20px; +} + +.random-container { + flex: 1; + min-width: 0; } .random-container-title { - font-size: 20px; + font-size: 11px; font-weight: 700; - margin-bottom: 20px; + margin-bottom: 8px; + color: var(--text-color); + opacity: 0.6; + text-transform: uppercase; + letter-spacing: 1px; } .random-title { - margin-bottom: 10px; - font-size: 30px; - transition: font-size 0.3s ease-in-out; - white-space: nowrap; + font-size: 18px; + font-weight: 600; + line-height: 1.3; + margin-bottom: 8px; + transition: all 0.3s ease; + display: -webkit-box; + -webkit-line-clamp: 2; + -webkit-box-orient: vertical; overflow: hidden; text-overflow: ellipsis; } +.random-meta { + display: flex; + gap: 8px; + font-size: 12px; +} + +.random-author, +.random-date { + display: inline-flex; + align-items: center; + gap: 4px; + padding: 4px 10px; + background-color: var(--background-color); + border: 1px solid var(--border-color); + border-radius: 12px; + color: var(--author-color); + transition: all 0.3s ease; +} + +.random-author:hover, +.random-date:hover { + border-color: var(--hover-color); +} + .random-button-container { - position: absolute; - bottom: 20px; - right: 20px; - border: none; - border-radius: 20px; - cursor: pointer; - font-size: 14px; - transition: background-color 0.3s ease-in-out; + display: flex; + gap: 10px; + align-items: center; + flex-shrink: 0; } .random-button-container a { - margin-right: 10px; - color: #aaaaaa !important; + padding: 8px 16px; + color: var(--text-color) !important; text-decoration: none !important; + border: 1px solid var(--border-color); + border-radius: 6px; + background-color: var(--background-color); + transition: all 0.3s ease; + font-size: 13px; + white-space: nowrap; +} + +.random-button-container a:hover { + border-color: var(--hover-color); + color: var(--hover-color) !important; } .random-link-button { - padding: 10px 20px; + padding: 8px 20px; border: none; - border-radius: 20px; + border-radius: 6px; background-color: var(--hover-color); color: #fff; cursor: pointer; + font-size: 13px; + font-weight: 500; + transition: all 0.3s ease; + white-space: nowrap; +} + +.random-link-button:hover { + opacity: 0.9; +} + +/* 响应式设计 */ +@media screen and (max-width: 768px) { + .random-stats { + grid-template-columns: repeat(2, 1fr); + } +} + +@media screen and (max-width: 600px) { + .random-content { + flex-direction: column; + align-items: center; + gap: 16px; + } + + .random-container { + text-align: center; + } + + .random-meta { + justify-content: center; + } + + .random-button-container { + justify-content: center; + width: 100%; + } +} + +@media screen and (max-width: 400px) { + .random-stats { + gap: 8px; + } + + .stat-item { + padding: 10px 8px; + } + + .stat-num { + font-size: 20px; + } +} + +/* 加载和错误状态样式 */ +.loading-placeholder, +.error-placeholder { + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + padding: 40px 20px; + gap: 12px; +} + +.loading-spinner { + width: 40px; + height: 40px; + border: 3px solid var(--border-color); + border-top-color: var(--hover-color); + border-radius: 50%; + animation: spin 1s linear infinite; +} + +@keyframes spin { + to { transform: rotate(360deg); } +} + +.loading-text { font-size: 14px; - transition: background-color 0.3s ease-in-out; + color: var(--author-color); +} + +.error-icon { + font-size: 48px; +} + +.error-text { + font-size: 14px; + color: var(--author-color); + text-align: center; +} + +.retry-button { + margin-top: 8px; + padding: 8px 20px; + border: none; + border-radius: 6px; + background-color: var(--hover-color); + color: #fff; + cursor: pointer; + font-size: 13px; + font-weight: 500; + transition: all 0.3s ease; +} + +.retry-button:hover { + opacity: 0.9; } .modal { @@ -180,6 +360,7 @@ height: 250px; right: -20px; bottom: -20px; + object-fit: cover; transition: transform 0.6s ease !important; } @@ -189,6 +370,7 @@ border-radius: 50% !important; width: 110px; height: 110px; + object-fit: cover; } #modal-author-name-link { @@ -258,36 +440,50 @@ .articles-container { display: grid; grid-template-columns: repeat(auto-fill, minmax(220px, 1fr)); - gap: 8px; + gap: 10px; width: 100%; } .card { background-color: var(--container-bg-color); - border-radius: 10px; - padding: 10px; + border-radius: 8px; + padding: 12px; border: 1px solid var(--border-color); position: relative; overflow: hidden; display: flex; flex-direction: column; justify-content: space-between; - height: 150px; - transition: border 0.3s; + height: 120px; + transition: all 0.3s ease; + opacity: 0; + transform: translateY(20px); + animation: fadeInUp 0.5s ease forwards; +} + +@keyframes fadeInUp { + to { + opacity: 1; + transform: translateY(0); + } +} + +.card:hover { + border-color: var(--hover-color); + transform: translateY(-4px); } -.card:hover, #random-article:hover { - border: 1px solid var(--hover-color); + border-color: var(--hover-color); } .card-title { z-index: 1; - font-size: 17px; + font-size: 16px; color: var(--text-color); - font-weight: 520; + font-weight: 600; cursor: pointer; - margin-bottom: 10px; + margin-bottom: 12px; line-height: 1.5; max-height: 4.5em; overflow: hidden; @@ -296,36 +492,35 @@ -webkit-line-clamp: 3; line-clamp: 3; -webkit-box-orient: vertical; - transition: color 0.3s; + transition: all 0.3s ease; } .card-title:hover { color: var(--hover-color); - text-decoration: underline; } .card-author, .card-date { font-size: 12px; color: var(--author-color); - padding: 5px; + padding: 4px 10px 4px 2px; line-height: 15px; } .card-author:hover { - box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2); + border-color: var(--hover-color); } .card-author { cursor: pointer; background-color: var(--background-color); border: 1px solid var(--border-color); - border-radius: 15px; + border-radius: 12px; display: flex; padding-right: 10px; width: fit-content; align-items: center; - transition: box-shadow 0.2s; + transition: all 0.3s ease; } #friend-circle-lite-root .card-author img { @@ -357,6 +552,7 @@ width: 140px; height: 140px; opacity: 0.4; + object-fit: cover; transition: transform 0.6s ease, bottom 0.3s ease, right 0.3s ease; } @@ -368,22 +564,26 @@ #load-more-btn { color: var(--text-color); - font-size: 15px; + font-size: 13px; + font-weight: 500; background-color: var(--container-bg-color); cursor: pointer; - width: 200px; - border-radius: 10px; + width: 140px; + max-width: 100%; + border-radius: 20px; border: 1px solid var(--border-color); - padding: 3px; - transition: all 0.3s; + padding: 8px 24px; + transition: all 0.3s ease; margin: 20px auto; display: block; } #load-more-btn:hover { background-color: var(--hover-color); - width: 300px; + border-color: var(--hover-color); color: white; + width: 180px; + max-width: 100%; } #stats-container { diff --git a/main/fclite.js b/main/fclite.js index 453a73904ae..d4cad5ea750 100644 --- a/main/fclite.js +++ b/main/fclite.js @@ -4,8 +4,8 @@ function initialize_fc_lite() { // 设置默认配置 UserConfig = { private_api_url: UserConfig?.private_api_url || "", - page_turning_number: UserConfig?.page_turning_number || 20, // 默认20篇 - error_img: UserConfig?.error_img || "https://fastly.jsdelivr.net/gh/willow-god/Friend-Circle-Lite@latest/static/favicon.ico" // 默认头像 + page_turning_number: UserConfig?.page_turning_number || 24, // 默认24篇 + error_img: UserConfig?.error_img || "https://fastly.jsdelivr.net/gh/willow-god/Friend-Circle-Lite/static/favicon.ico" // 默认头像 }; const root = document.getElementById('friend-circle-lite-root'); @@ -17,6 +17,12 @@ function initialize_fc_lite() { const randomArticleContainer = document.createElement('div'); randomArticleContainer.id = 'random-article'; + randomArticleContainer.innerHTML = ` +
+ +
加载中...
+
+ `; root.appendChild(randomArticleContainer); const container = document.createElement('div'); @@ -29,7 +35,6 @@ function initialize_fc_lite() { loadMoreBtn.innerText = '再来亿点'; root.appendChild(loadMoreBtn); - // 创建统计信息容器 const statsContainer = document.createElement('div'); statsContainer.id = 'stats-container'; root.appendChild(statsContainer); @@ -51,40 +56,69 @@ function initialize_fc_lite() { } } + // 设置10秒超时 + const timeoutId = setTimeout(() => { + showError('加载超时，请刷新页面重试'); + }, 10000); + fetch(`${UserConfig.private_api_url}all.json`) - .then(response => response.json()) + .then(response => { + clearTimeout(timeoutId); + if (!response.ok) { + throw new Error('网络响应错误'); + } + return response.json(); + }) .then(data => { localStorage.setItem(cacheKey, JSON.stringify(data)); localStorage.setItem(cacheTimeKey, now.toString()); processArticles(data); }) + .catch(error => { + clearTimeout(timeoutId); + console.error('加载失败:', error); + showError('加载失败，请检查网络连接'); + }) .finally(() => { loadMoreBtn.innerText = '再来亿点'; // 恢复按钮文本 }); } + function showError(message) { + randomArticleContainer.innerHTML = ` +
+
⚠️
+
${message}
+ +
+ `; + } + function processArticles(data) { - allArticles = data.article_data; + allArticles = data.article_data || []; + // 处理统计数据 const stats = data.statistical_data; + statsContainer.innerHTML = `
Powered by: FriendCircleLite
Designed By: LiuShen
-
订阅:${stats.friends_num} 活跃:${stats.active_num} 总文章数:${stats.article_num}
更新时间:${stats.last_updated_time}
`; - displayRandomArticle(); // 显示随机友链卡片 + displayRandomArticle(stats); // 显示随机友链卡片，传入统计数据 const articles = allArticles.slice(start, start + UserConfig.page_turning_number); - articles.forEach(article => { + articles.forEach((article, index) => { const card = document.createElement('div'); card.className = 'card'; + card.style.animationDelay = `${index * 0.05}s`; const title = document.createElement('div'); title.className = 'card-title'; title.innerText = article.title; + title.title = article.title; // 添加完整标题的提示 card.appendChild(title); title.onclick = () => window.open(article.link, '_blank'); @@ -92,8 +126,8 @@ function initialize_fc_lite() { author.className = 'card-author'; const authorImg = document.createElement('img'); authorImg.className = 'no-lightbox'; - authorImg.src = article.avatar || UserConfig.error_img; // 使用默认头像 - authorImg.onerror = () => authorImg.src = UserConfig.error_img; // 头像加载失败时使用默认头像 + authorImg.src = article.avatar || UserConfig.error_img; + authorImg.onerror = () => authorImg.src = UserConfig.error_img; author.appendChild(authorImg); author.appendChild(document.createTextNode(article.author)); card.appendChild(author); @@ -110,7 +144,7 @@ function initialize_fc_lite() { const bgImg = document.createElement('img'); bgImg.className = 'card-bg no-lightbox'; bgImg.src = article.avatar || UserConfig.error_img; - bgImg.onerror = () => bgImg.src = UserConfig.error_img; // 头像加载失败时使用默认头像 + bgImg.onerror = () => bgImg.src = UserConfig.error_img; card.appendChild(bgImg); container.appendChild(card); @@ -124,25 +158,62 @@ function initialize_fc_lite() { } // 显示随机文章的逻辑 - function displayRandomArticle() { + function displayRandomArticle(stats) { const randomArticle = allArticles[Math.floor(Math.random() * allArticles.length)]; + if (!randomArticle) { + randomArticleContainer.innerHTML = ` +
+
暂无可展示文章
+
+ `; + return; + } randomArticleContainer.innerHTML = ` -
-
随机钓鱼
-
${randomArticle.title}
-
作者: ${randomArticle.author}
+
+
+
+
${stats.friends_num}
+
订阅
+
+
+
${stats.active_num}
+
活跃
+
+
+
${stats.article_num}
+
文章
+
+
+
${stats.error_num}
+
失败
+
+
-
- 刷新 - +
+
+
🎲 随便转转
+
${randomArticle.title}
+
+ ✍️ ${randomArticle.author} + 📅 ${randomArticle.created.substring(0, 10)} +
+
+
+ 🔄 换一篇 + +
`; // 为刷新按钮添加事件监听器 const refreshBtn = document.getElementById('refresh-random-article'); refreshBtn.addEventListener('click', function (event) { - event.preventDefault(); // 阻止默认的跳转行为 - displayRandomArticle(); // 调用显示随机文章的逻辑 + event.preventDefault(); + randomArticleContainer.style.opacity = '0.5'; + setTimeout(() => { + displayRandomArticle(stats); + randomArticleContainer.style.opacity = '1'; + }, 200); }); } @@ -235,4 +306,4 @@ function whenDOMReady() { } whenDOMReady(); -document.addEventListener("pjax:complete", initialize_fc_lite); +document.addEventListener("pjax:complete", initialize_fc_lite); \ No newline at end of file diff --git a/rss_subscribe/email_template.html b/push_templates/default.html similarity index 100% rename from rss_subscribe/email_template.html rename to push_templates/default.html diff --git a/readme.md b/readme.md index 6ff5caddcec..e3d4758462d 100644 --- a/readme.md +++ b/readme.md @@ -11,7 +11,64 @@ ## 开发进度 -### 2026-07-23 +### 2026-06-27 - 检测缓存与重试节奏整理 + +本次主要整理了友链检测缓存和重试节奏，目标是减少长期异常站点带来的重复请求，尤其是 RSS 探测请求。 + +- **不可达时长改为按时间计算**：不再把失败次数当作“不可达天数”。SQLite 记录 `unreachable_since`，`link.json` 输出 `unreachable_days`，按持续时间向上取整，首次不可达显示为 1 天。 +- **RSS 不可用状态单独入库**：新增 `rss_unavailable_since`，用于记录“站点可达，但 RSS 探测或解析失败”的持续起点。该字段只用于内部缓存和重试判断，不输出到 `link.json`。 +- **动态降低重试频率**：长期异常站点会逐步降低检测频率：未满 10 天按默认缓存周期，10 天后 5 天检测一次，30 天后 10 天检测一次，60 天后最多 15 天检测一次。 +- **减少 RSS 探测请求**：缓存复用发生在 RSS 探测之前。只要站点已有“RSS 不可用”缓存，并且未到动态重试窗口，就不会再次扫描 `/feed`、`/rss.xml`、`/atom.xml` 等候选地址。 +- **输出保持克制**：`link.json` 继续面向前端展示，只保留可达性、可抓取性、不可达天数、最近文章时间等字段；RSS 不可用起点不暴露给前端。 +- **SQLite 兼容升级**：旧缓存库会自动补齐新增字段；旧的 `fail_count` 列仅作为兼容遗留列保留，不再参与业务逻辑和 public JSON 输出。 + +
+查看更多 + +### 2026-06-08 - v2.1.0 优化更新 + +* **友链检测逻辑简化**：RSS 探测、主页可达性、反链检测统一由友链可达性检测流程维护，友圈抓取只读取可抓取 RSS 的缓存结果，减少重复请求。 +* **配置项整理**：代理配置独立为 `proxy_settings.proxy_url`，建议通过仓库环境变量 `PROXY_URL` 覆盖；`link_check.enable` 旧字段仍兼容读取，但当前检测流程始终启用。 +* **数据输出稳定**：`all.json` 保持友链朋友圈文章格式不变，友链可达性数据集中输出到 `link.json`；`link.json` 移除冗余的 `method`、`reason`、单站 `checked_at` 等展示字段。 +* **调试能力增强**：新增 `debug` / `FCL_DEBUG` 开关，开启后会全量打印 SQLite 缓存并清理核心表残留旧字段，便于定位缓存问题。 +* **首页展示重写**：`static/index.html` 改为国风单文件展示页，可在顶部切换友链可达性与友链朋友圈；`main/fclite.js` 和 `main/fclite.css` 保持不变，外部引用不受影响。 + +### 🎉 2026-05-31 - v2.0.0 重大更新 + +> **⚠️ 重要更新**：本次更新包含**数据结构变更**和**配置文件调整**，请查看 **[完整更新日志 (CHANGELOG.md)](./CHANGELOG.md)** 了解详情。 +> +> **✅ 兼容性更新**：保持向后兼容，旧版引用者无需修改代码，`all.json` 结构保持兼容，`main/fclite.js` 和 `main/fclite.css` 不变。 + +* **✨ 新增友链可达性检测功能**：合并 [check-flink](https://github.com/willow-god/check-flink) 项目，支持直连、代理、API 三种检测方式，并支持反链检测。 +* **📊 数据结构重构**：友链可达性数据独立输出到 `link.json`，`all.json` 仅保留友圈文章数据，实现数据分离。 +* **⚙️ 配置优化**：`merge_settings` 独立配置，友圈文章和友链可达性数据分别控制合并。 +* **🎨 前端页面重写**：新首页继承 check-flink 风格，同时展示友链状态和友圈文章。 +* **🔄 智能数据合并**：支持国内外数据源智能合并，可达性优先级、延迟、反链、失败次数均智能选择最优结果。 + +**详细说明**：[查看完整更新日志 →](./CHANGELOG.md) + +### 请求节奏说明 + +默认 GitHub Actions 每 4 小时执行一次友链朋友圈任务，但友链可达性检测与 RSS 探测结果会按 `link_check.max_age_hours` 缓存，默认 24 小时。 + +也就是说： + +- 友链可达性检测、RSS 探测、主页可达性、反链检测默认每天才会对同一站点重新检查一次。 +- 如果某个站点没有 RSS，检测结果会缓存为不可抓取，接下来 24 小时内的友圈任务会直接跳过它，不会每 4 小时重复探测。 +- 每 4 小时执行一次的友圈抓取会复用这份检测缓存，只请求 `crawlable=true` 且有 RSS 缓存的站点。 +- 新增友链如果还没有进入缓存，会在当前轮次按友链数据单独参与匹配；完整检测结果会等到下一个 24 小时检测窗口统一写入缓存。 + +这样可以保留较高的文章更新频率，同时尽量减少对友链站点、RSS 地址和代理/API 的请求数量。 + +### 2026-05-25 + +* 合并友链可达性检测能力，检测结果会写入独立的 `link.json` 供前端按需获取，不可达或仅 API 可达的友链会跳过 RSS 抓取。 + +### 2026-05-24 + +* 移除原先基于 FastAPI 的简陋后端部署方式，后续自部署统一采用生成静态文件后作为纯静态网站托管的纯净态方式。 + +### 2025-07-23 * 添加缓存文件，防止由于缓存导致多次请求 * 添加feed.php后缀的适配 @@ -30,9 +87,6 @@ * 添加随机文章刷新按钮 * 完善邮件通知模板自定义程度 -
-查看更多 -

2024-10-07

* 更新自部署的api地址，统一为all.json，提高js兼容性 @@ -93,24 +147,27 @@ ## 展示页面 -* [清羽飞扬の友链朋友圈](https://blog.liushen.fun/fcircle/) +* [清羽飞扬](https://blog.liushen.fun/fcircle/) + +* [星港](https://blog.starsharbor.com/fcircle/) -* [❖星港◎Star☆ 的友链朋友圈](https://blog.starsharbor.com/fcircle/) +* [梦爱吃鱼](https://blog.bsgun.cn/fcircle) -* [梦爱吃鱼的友链朋友圈](https://blog.bsgun.cn/fcircle/) +* [Aroes](https://homulilly.com/friends/) * 欢迎在issue中[提交](https://github.com/willow-god/Friend-Circle-Lite/issues/20)以展示你独特的设计！ ## 项目介绍 -- **爬取文章**: 爬取所有友链的文章，结果放置在根目录的all.json文件中，方便读取并部署到前端。 +- **爬取文章**: 爬取可抓取 RSS 的友链文章，结果放置在根目录的 `all.json` 文件中，方便读取并部署到前端。 +- **友链可达性检测**: 抓取前会统一维护 RSS 探测、主页可达性和反链检测缓存；检测结果单独输出到 `link.json`，友圈抓取会根据 `crawlable` 与 RSS 缓存决定是否抓取。 - **邮箱推送更新(对作者推送所有友链更新)**: 作者可以通过邮箱订阅所有rss的更新（未来开发）。 - **issue邮箱订阅(对访客实时推送最新文章邮件)**: 基于`GitHub issue`的博客更新邮件订阅功能，游客可以通过简单的提交`issue`进行邮箱订阅站点更新，删除对应`issue`即可取消订阅。 -- **文件分离**: 将前后端分离，前端文件放在page分支，后端文件放在主分支 +- **文件分离**: 将生成任务和静态展示分离，前端文件与生成后的 `all.json`、`link.json` 可直接作为静态网站托管。 ## 特点介绍 -* **轻量化**：对比原版友链朋友圈的功能，该友圈功能简洁，去掉了设置和fastAPI的臃肿，仅保留关键内容。 +* **轻量化**：对比原版友链朋友圈的功能，该友圈功能简洁，去掉了设置和 FastAPI 的臃肿，仅保留关键内容。 * **无数据库**：因为内容较少，我采用`json`直接存储文章信息，减少数据库操作，提升`action`运行效率。 * **部署简单**：原版友链朋友圈由于功能多，导致部署较为麻烦，本方案仅需简单的部署action即可使用，vercel仅用于部署前端静态页面和实时获取最新内容。 * **文件占用**：对比原版`4MB`的`bundle.js`文件大小，本项目仅需要`5.50KB`的`fclite.min.js`文件即可轻量的展示到前端。 @@ -118,6 +175,8 @@ ## 功能概览 * 文章爬取 +* 友链可达性检测 +* 友链状态前端展示 * 暗色适配 * 显示作者所有文章 * 获取丢失友链数据 @@ -157,103 +216,178 @@ ### 配置选项 -1. 如果需要修改爬虫设置或邮件模板等配置，需要修改仓库中的 `config.yaml` 文件： - - - **爬虫相关配置** - 使用 `requests` 库实现友链文章的爬取，并将结果存储到根目录下的 `all.json` 文件中。 - - ```yaml - spider_settings: - enable: true - json_url: "https://blog.qyliu.top/friend.json" - article_count: 5 - merge_result: - enable: true - merge_json_url: "https://fc.liushen.fun" - ``` - - `enable`：开启或关闭，默认开启； - - `json_url`：友链朋友圈通用爬取格式第一种（下方有配置方法）; - - `article_count`：每个作者留存文章个数。 - - `marge_result`：是否合并多个json文件，若为true则会合并指定网络地址和本地地址的json文件并去重 - - - `enable`：是否启用合并功能，该功能提供与自部署的友链合并功能，可以解决服务器部分国外网站，服务器无法访问的问题 - - - `marge_json_path`：请填写网络地址的json文件，用于合并，不带空格！！！ - - - **邮箱推送功能配置** - 暂未实现，预留用于将每天的友链文章更新推送给指定邮箱。 - - ```yaml - email_push: - enable: false - to_email: recipient@example.com - subject: "今天的 RSS 订阅更新" - body_template: "rss_template.html" - ``` - - **暂未实现**：该部分暂未实现，由于感觉用处不大，保留接口后期酌情更新。 - - - **邮箱 issue 订阅功能配置** - 通过 GitHub issue 实现向提取的所有邮箱推送博客更新的功能。 - - ```yaml - rss_subscribe: - enable: true - github_username: willow-god - github_repo: Friend-Circle-Lite - your_blog_url: https://blog.qyliu.top/ - ``` - - `enable`：开启或关闭，默认开启，如果没有配置请关闭。 - - `github_username`：github用户名，用来拼接github api地址 - - `github_repo`：仓库名称，作用同上。 - - `your_blog_url`：用来定时检测是否有最新文章，请确保你的网站可以被FCLite抓取到 - - - **SMTP 配置** - 使用配置中的相关信息实现邮件发送功能。 - - ```yaml - smtp: - email: 3162475700@qq.com - server: smtp.qq.com - port: 587 - use_tls: true - ``` - - `email`：发件人邮箱地址 - - `server`：`SMTP` 服务器地址 - - `port`：`SMTP` 端口号 - - `use_tls`：是否使用 `tls` 加密 - - 这部分配置较为复杂，请自行学习使用。 - - - **特定 RSS 配置** - - 用于指定特定友链特殊RSS，样例如下： +如果需要修改爬虫、合并、友链检测或邮件模板等配置，请修改仓库中的 `conf.yaml` 文件。下面只列常用项，完整注释可以直接查看 `conf.yaml`。 + +#### 使用 Repository variables 覆盖配置 + +为了避免 fork 后频繁修改公开的 `conf.yaml`，GitHub Actions 支持使用 Repository variable `FCL_CONFIG_OVERRIDES` 在运行时覆盖配置。变量值按“一行一个配置项”填写，使用点号表示嵌套路径，值按 YAML 类型解析： + +```text +merge_settings.enable=true +merge_settings.remote_base_url=https://fc.liiiu.cn +spider_settings.article_count=8 +link_check.enable_backlink_check=false +``` + +没有写入变量的配置项继续使用 `conf.yaml` 中的值；配置文件本身不会被修改。空行和 `#` 开头的行会被忽略，错误格式的行会跳过并记录警告。 + +- **调试开关** + + ```yaml + debug: false + ``` + + 默认关闭。开启后，程序结束前会全量打印 SQLite 缓存表结构与所有数据，并保守清理核心表残留旧字段。也可以使用仓库环境变量 `FCL_DEBUG=1` 临时开启。 + +- **爬虫相关配置** + + ```yaml + spider_settings: + enable: true + json_url: "https://blog.liushen.fun/friend.json" + article_count: 5 + ``` + + `enable`：是否启用友链朋友圈抓取。 + + `json_url`：你的友链 JSON 地址，仅支持网络地址。 + + `article_count`：每个站点最多抓取的文章数量。 + +- **代理配置** + + ```yaml + proxy_settings: + proxy_url: "" + ``` + + 程序会先直连，请求失败且配置了代理时自动走代理。`proxy_url` 涉及一定违规风险和隐私风险，请尽量不要直接写入配置文件，推荐在仓库环境变量 `PROXY_URL` 中配置。 + + 推荐填写代理前缀，例如： + + ```text + https://proxy.example.com/ + ``` + + 程序会自动拼接为： - ```yaml - specific_RSS: - - name: "Redish101" - url: "https://reblog.redish101.top/api/feed" - # - name: "無名小栈" - # url: "https://blog.imsyy.top/rss.xml" - ``` + ```text + https://proxy.example.com/https://example.com/feed.xml + ``` - `name`：友链名称，需要严格匹配 + 如果需要自行搭建代理，可以参考：[使用 CF Workers 搭建反代加速器](https://blog.liushen.fun/posts/dd89adc9/)。 - `url`：该友链对应RSS地址 + 也兼容 `https://proxy.example.com?url={url}` 这类高级格式，但普通反代场景不推荐这样写。 - 可以添加多个，如果不需要也可以置空。 +- **数据合并配置** + + ```yaml + merge_settings: + enable: false + remote_base_url: "https://fc.liushen.fun" + merge_article_data: true + merge_link_check_data: true + ``` + + `enable`：是否启用数据合并。 + + `remote_base_url`：远程数据源基础 URL，程序会自动拼接 `/all.json`、`/link.json`、`/errors.json`。 + + `merge_article_data`：是否合并友链朋友圈文章数据。 + + `merge_link_check_data`：是否合并友链可达性数据。 + +- **友链可达性检测配置** + + ```yaml + link_check: + max_age_hours: 24 + timeout: 15 + max_workers: 10 + status_api_url: "https://v2.xxapi.cn/api/status?url={url}" + enable_backlink_check: true + author_url: "blog.liushen.fun" + ``` + + 友圈抓取依赖友链可达性检测，因此当前检测流程始终启用；旧配置中的 `link_check.enable` 会被兼容读取，但不再作为有效开关。 + + `max_age_hours`：同一友链检测结果缓存时间，默认 24 小时。缓存未过期时会复用 RSS、主页可达性、反链等结果；没有 RSS 的站点也会在缓存期内直接跳过，避免每次友圈抓取都重新探测。 + + `timeout`：单次网页请求超时时间。 + + `max_workers`：并发检测数量。 + + `status_api_url`：兜底状态码 API。API 只能确认状态码，无法提供页面内容，所以 API-only 结果只用于可达性展示，不参与 RSS 抓取。 + + `enable_backlink_check`：是否检测对方友链页是否包含你的站点链接。 + + `author_url`：你的站点域名，用于反链检测，建议只填写域名。 + + 友链数据兼容旧三字段和新四字段格式： + + ```json + ["站点名称", "站点地址", "头像地址"] + ["站点名称", "站点地址", "友链页地址", "头像地址"] + ``` + +- **邮箱推送功能配置** + + 暂未实现，预留用于将每天的友链文章更新推送给指定邮箱。 + + ```yaml + email_push: + enable: false + to_email: recipient@example.com + subject: "今天的 RSS 订阅更新" + body_template: "rss_template.html" + ``` + +- **邮箱 issue 订阅功能配置** + + 通过 GitHub issue 实现向提取的所有邮箱推送博客更新的功能。 + + ```yaml + rss_subscribe: + enable: true + github_username: willow-god + github_repo: Friend-Circle-Lite + your_blog_url: https://blog.liushen.fun/ + email_template: "./push_templates/default.html" + website_info: + title: "清羽飞扬" + ``` + + `enable`：开启或关闭，如果没有配置请关闭。 + + `github_username`：GitHub 用户名，用来拼接 GitHub API 地址。 + + `github_repo`：仓库名称，作用同上。 + + `your_blog_url`：用来定时检测是否有最新文章，请确保你的网站可以被 FCLite 抓取到。 + +- **SMTP 配置** + + 使用配置中的相关信息实现邮件发送功能。SMTP 密码不写入配置文件，Action 部署请使用 `SMTP_PWD` Secret。 + + ```yaml + smtp: + email: notify@example.com + server: smtp.example.com + port: 465 + use_tls: true + ``` + +- **特定 RSS 配置** + + 用于指定特殊友链 RSS，名称需要与友链名称严格匹配。 + + ```yaml + specific_RSS: + - name: "阮一峰" + url: "http://feeds.feedburner.com/ruanyifeng" + ``` + + 如果不需要也可以置空，但不要删除此项。 2. **贡献与定制:** 欢迎对仓库进行贡献或根据需要进行定制。 @@ -336,7 +470,7 @@ 部署完成后，你将获得一个地址，如果是通过vercel部署的，建议自行绑定域名。 -检查 `https://example.com/all.json` 是否有数据，如果有，则部署成功。 +检查 `https://example.com/all.json` 和 `https://example.com/link.json` 是否有数据，如果都有，则部署成功。 ## 部署到你的页面 @@ -366,15 +500,13 @@ ## 自部署使用方法 +自部署后续统一采用纯静态方式：本项目只负责定时生成 `all.json`、`link.json`、`errors.json` 等数据文件，生成完成后把 `static`、`main` 和数据文件作为静态网站托管即可，不再启动 FastAPI 后端服务。友链检测缓存会保存在 `temp/cache.sqlite3` 中，用于判断同一友链是否需要在 24 小时后重新检测；静态网站只需要发布生成后的 JSON 和静态资源。 + 如果你有一台境内服务器，你也可以通过以下操作将其部署到你的服务器上，操作如下： ### 前置工作 -确保你的服务器有定时任务 `crontab` 功能包，一般是linux自带，如果你没有宝塔等可以管理定时任务的面板工具，可能需要你自行了解定时工具并导入，本教程提供了简单的介绍。 - -### 宿主机环境 - -> 适用于宝塔面板等宿主机直接有Python环境的场景下 +确保你的服务器有 Python 运行环境，以及定时任务 `crontab`、宝塔、1Panel 等任意一种可定时执行命令的工具。 首先克隆仓库并进入对应路径： @@ -383,40 +515,40 @@ git clone https://github.com/willow-god/Friend-Circle-Lite.git cd Friend-Circle-Lite ``` -由于不存在issue，所以不支持邮箱推送(主要是懒得分类写了，要不然还得从secret中获取密码的功能剥离QAQ)，请将除第一部分抓取以外的功能均设置为false +由于不存在 issue，所以不支持邮箱推送(主要是懒得分类写了，要不然还得从secret中获取密码的功能剥离QAQ)，请将除第一部分抓取以外的功能均设置为false。 -下载服务相关包，其中 `requirements-server.txt` 是部署API服务所用包， `requirements.txt` 是抓取服务所用包，请均下载一遍。 +安装抓取服务所需依赖： ```bash pip install -r ./requirements.txt -pip install -r ./server/requirements-server.txt ``` -#### 部署API服务 +### 生成静态文件 -如果环境配置完毕，你可以进入目录路径后直接运行`deploy.sh`脚本启动API服务： +执行一次抓取命令： ```bash -chmod +x ./deploy.sh -./deploy.sh +python run.py ``` -其中的注释应该是较为详细的，如果部署成功你可以使用以下命令进行测试，如果获取到了首页html内容则成功： +执行完成后，根目录会生成或更新 `all.json`、`link.json`、`errors.json` 等数据文件。将以下内容放到你的静态网站目录即可： -```bash -curl 127.0.0.1:1223 -``` +- `static/` 目录中的静态页面和资源 +- `main/` 目录中的前端样式与脚本 +- `all.json`、`link.json`、`errors.json` 数据文件 -这个端口号可以修改，在server.py最后一行修改数字即可，如果你想删除该API服务，可以使用ps找到对应进程并使用Kill命令杀死进程： +如果希望在宿主机上直接整理出可发布目录，也可以运行： ```bash -ps aux | grep python -kill -9 [这里填写上面查询结果中对应的进程号] +chmod +x ./deploy.sh +./deploy.sh ``` -### Docker环境 +脚本会执行 `python3 run.py`，并将 `main`、`static`、`all.json`、`link.json`、`errors.json` 复制到 `pages/` 目录。将 `pages/` 目录作为静态网站根目录部署即可。部署完成后，检查 `/all.json` 和 `/link.json` 是否有数据，如果都有，则部署成功。 -> 目前Docker部署方式和1Panel强相关，如果有其他需要docker环境部署的，可以参考，但是可能需要自行摸索 +### 1Panel / Docker 环境 + +> 目前 Docker 部署方式和 1Panel 强相关；这里的思路是先在面板里跑一次生成命令，把生成后的静态文件放到网站目录，再按静态网站托管。 由于主包也用上了1Panel，所以捣鼓了一下，得益于1Panel可以方便的创建运行环境，所以可以基本做到无占用的API，因为除了执行action的时候，其他时间完全是纯静态的。 @@ -437,21 +569,23 @@ kill -9 [这里填写上面查询结果中对应的进程号] 其中运行目录为刚才看到的以网站命名的文件夹，其中名称可以按照自己的要求随意修改，仅为Docker名称，执行命令如下： ```bash -pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && pip install -r requirements.txt && python run.py && mv -f all.json errors.json index/ +pip config set global.index-url https://pypi.tuna.tsinghua.edu.cn/simple && pip install -r requirements.txt && python run.py && mv -f all.json link.json errors.json index/ ``` 点击运行，如果正常，应该会执行一个docker实现抓取，并且在抓取完成后，docker关闭，而你的网站下的index文件夹内，应该就是最新的爬取的内容，直接静态部署即可，后面只需要定时启动docker，会自动执行抓取命令，抓取完成后也会自动关闭。 #### 合并github数据 -你是不是以为github数据没用了？并不是！因为有很多站长是使用的GitHub page等服务部署的，这种服务可能无法被你的服务器抓取，此时你就需要合并两个的爬取数据。修改第一个配置中的以下部分： +你是不是以为 GitHub 数据没用了？并不是！因为有很多站长使用 GitHub Pages 等服务部署博客，这种服务可能无法被你的服务器抓取，此时你可以同时维护国内外两条线路，并合并文章数据与可达性数据。修改 `conf.yaml` 中的以下部分： ```yaml -merge_result: - enable: true - merge_json_url: "https://fc.liushen.fun" +merge_settings: + enable: true + remote_base_url: "https://fc.liushen.fun" + merge_article_data: true + merge_link_check_data: true ``` -其中地址项不要添加最后的斜杠，这样就会在本地爬取结束后合并远程的数据，以做到更高的准确率！ +其中 `remote_base_url` 不需要填写具体 JSON 文件名，程序会自动请求 `/all.json`、`/link.json`、`/errors.json` 并合并结果。 ### 定时抓取文章 @@ -459,7 +593,7 @@ merge_result: > 宿主机环境只需要直接执行python ./run.py即可执行抓取 -由于原生的crontab可能较为复杂，如果有兴趣可以查看./deploy.sh文件中，屏蔽掉的部分，这里我不会细讲，这里我主要讲解宝塔面板添加定时任务，这样可以最大程度减少内存占用，其他面板服务类似： +由于原生的 crontab 可能较为复杂，这里主要讲解宝塔面板添加定时任务，这样可以最大程度减少内存占用，其他面板服务类似： ![](./static/baota.png) @@ -486,4 +620,4 @@ docker restart Friend-Circle-Lite ## Star增长曲线 -[![Star History Chart](https://api.star-history.com/svg?repos=willow-god/Friend-Circle-Lite&type=Timeline)](https://star-history.com/#willow-god/Friend-Circle-Lite&Timeline) \ No newline at end of file +[![Star History Chart](https://api.star-history.com/svg?repos=willow-god/Friend-Circle-Lite&type=Timeline)](https://star-history.com/#willow-god/Friend-Circle-Lite&Timeline) diff --git a/requirements.txt b/requirements.txt index bef98e81b15..873fccbba39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ -datetime python-dateutil==2.9.0.post0 requests feedparser==6.0.11 PyYAML==6.0.1 -jinja2==3.1.2 \ No newline at end of file +jinja2==3.1.2 diff --git a/rss_subscribe/__init__.py b/rss_subscribe/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/rss_subscribe/push_article_update.py b/rss_subscribe/push_article_update.py deleted file mode 100644 index 813b2c3ac96..00000000000 --- a/rss_subscribe/push_article_update.py +++ /dev/null @@ -1,105 +0,0 @@ -import logging -import requests -import re -from friend_circle_lite.get_info import check_feed, parse_feed -import json -import os - -# 标准化的请求头 -HEADERS_JSON = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/123.0.0.0 Safari/537.36 " - "(Friend-Circle-Lite/1.0; +https://github.com/willow-god/Friend-Circle-Lite)" - ), - "X-Friend-Circle": "1.0" -} - - -def extract_emails_from_issues(api_url): - """ - 从GitHub issues API中提取以[e-mail]开头的title中的邮箱地址。 - - 参数： - api_url (str): GitHub issues API的URL。 - - 返回： - dict: 包含所有提取的邮箱地址的字典。 - { - "emails": [ - "3162475700@qq.com" - ] - } - """ - try: - response = requests.get(api_url, headers=HEADERS_JSON, timeout=10) - response.raise_for_status() - issues = response.json() - except Exception as e: - logging.error(f"无法获取 GitHub issues 数据，错误信息: {e}") - return None - - email_pattern = re.compile(r'^\[邮箱订阅\](.+)$') - emails = [] - - for issue in issues: - title = issue.get("title", "") - match = email_pattern.match(title) - if match: - email = match.group(1).strip() - emails.append(email) - - return {"emails": emails} - -def get_latest_articles_from_link(url, count=5, last_articles_path="./temp/newest_posts.json"): - """ - 从指定链接获取最新的文章数据并与本地存储的上次的文章数据进行对比。 - - 参数： - url (str): 用于获取文章数据的链接。 - count (int): 获取文章数的最大数。如果小于则全部获取，如果文章数大于则只取前 count 篇文章。 - - 返回： - list: 更新的文章列表，如果没有更新的文章则返回 None。 - """ - # 本地存储上次文章数据的文件 - local_file = last_articles_path - - # 检查和解析 feed - session = requests.Session() - feed_type, feed_url = check_feed(url, session) - if feed_type == 'none': - logging.error(f"无法获取 {url} 的文章数据") - return None - - # 获取最新的文章数据 - latest_data = parse_feed(feed_url, session ,count) - latest_articles = latest_data['articles'] - - # 读取本地存储的上次的文章数据 - if os.path.exists(local_file): - with open(local_file, 'r', encoding='utf-8') as file: - last_data = json.load(file) - else: - last_data = {'articles': []} - - last_articles = last_data['articles'] - - # 找到更新的文章 - updated_articles = [] - last_titles = {article['link'] for article in last_articles} - - for article in latest_articles: - if article['link'] not in last_titles: - updated_articles.append(article) - - logging.info(f"从 {url} 获取到 {len(latest_articles)} 篇文章，其中 {len(updated_articles)} 篇为新文章") - - # 更新本地存储的文章数据 - with open(local_file, 'w', encoding='utf-8') as file: - json.dump({'articles': latest_articles}, file, ensure_ascii=False, indent=4) - - # 如果有更新的文章，返回这些文章，否则返回 None - return updated_articles if updated_articles else None - diff --git a/run.py b/run.py index 8816cc9b0fc..de7ef9857f4 100644 --- a/run.py +++ b/run.py @@ -1,165 +1,27 @@ -import logging -import json -import sys -import os - -from friend_circle_lite.get_info import ( - fetch_and_process_data, - marge_data_from_json_url, - marge_errors_from_json_url, - deal_with_large_data -) -from friend_circle_lite.get_conf import load_config -from rss_subscribe.push_article_update import ( - get_latest_articles_from_link, - extract_emails_from_issues -) -from push_rss_update.send_email import send_emails - -# ========== 日志设置 ========== -logging.basicConfig( - level=logging.INFO, - format='😋 %(levelname)s: %(message)s' -) - -# ========== 加载配置 ========== -config = load_config("./conf.yaml") - -# ========== 爬虫模块 ========== -if config["spider_settings"]["enable"]: - logging.info("✅ 爬虫已启用") - - json_url = config['spider_settings']['json_url'] - article_count = config['spider_settings']['article_count'] - specific_rss = config['specific_RSS'] - - logging.info(f"📥 正在从 {json_url} 获取数据，每个博客获取 {article_count} 篇文章") - result, lost_friends = fetch_and_process_data( - json_url=json_url, - specific_RSS=specific_rss, - count=article_count, - cache_file="./temp/cache.json" - ) # type: ignore - - if config["spider_settings"]["merge_result"]["enable"]: - merge_url = config['spider_settings']["merge_result"]['merge_json_url'] - logging.info(f"🔀 合并功能开启，从 {merge_url} 获取外部数据") - - result = marge_data_from_json_url(result, f"{merge_url}/all.json") - lost_friends = marge_errors_from_json_url(lost_friends, f"{merge_url}/errors.json") - - article_count = len(result.get("article_data", [])) - logging.info(f"📦 数据获取完毕，共有 {article_count} 位好友的动态，正在处理数据") - - result = deal_with_large_data(result) +"""Friend-Circle-Lite application entrypoint.""" - with open("all.json", "w", encoding="utf-8") as f: - json.dump(result, f, ensure_ascii=False, indent=2) +from __future__ import annotations - with open("errors.json", "w", encoding="utf-8") as f: - json.dump(lost_friends, f, ensure_ascii=False, indent=2) - -# ========== 邮箱推送准备 ========== -SMTP_isReady = False - -sender_email = "" -server = "" -port = 0 -use_tls = False -password = "" - -if config["email_push"]["enable"] or config["rss_subscribe"]["enable"]: - logging.info("📨 推送功能已启用，正在准备中...") - - smtp_conf = config["smtp"] - sender_email = smtp_conf["email"] - server = smtp_conf["server"] - port = smtp_conf["port"] - use_tls = smtp_conf["use_tls"] - password = os.getenv("SMTP_PWD") - - logging.info(f"📡 SMTP 服务器：{server}:{port}") - if not password or not sender_email or not server or not port: - logging.error("❌ 环境变量 SMTP_PWD 未设置，无法发送邮件") - else: - logging.info(f"🔐 密码(部分)：{password[:3]}*****") - SMTP_isReady = True - -# ========== 邮件推送（待实现）========== -if config["email_push"]["enable"] and SMTP_isReady: - logging.info("📧 邮件推送已启用") - logging.info("⚠️ 抱歉，目前尚未实现邮件推送功能") - -# ========== RSS 订阅推送 ========== -if config["rss_subscribe"]["enable"] and SMTP_isReady: - logging.info("📰 RSS 订阅推送已启用") - - # 获取 GitHub 仓库信息 - fcl_repo = os.getenv('FCL_REPO') # 仓库内置 - if fcl_repo: - github_username, github_repo = fcl_repo.split('/') - else: - github_username = str(config["rss_subscribe"]["github_username"]).strip() - github_repo = str(config["rss_subscribe"]["github_repo"]).strip() +import logging - logging.info(f"👤 GitHub 用户名：{github_username}") - logging.info(f"📁 GitHub 仓库：{github_repo}") +from friend_circle_lite.cli import FriendCircleLiteApplication +from friend_circle_lite.utils.config import load_config - your_blog_url = config["rss_subscribe"]["your_blog_url"] - email_template = config["rss_subscribe"]["email_template"] - website_title = config["rss_subscribe"]["website_info"]["title"] - latest_articles = get_latest_articles_from_link( - url=your_blog_url, - count=5, - last_articles_path="./temp/newest_posts.json" # 存储上一次的文章 +def configure_logging() -> None: + """Configure the global logging style used by the CLI entrypoint.""" + logging.basicConfig( + level=logging.INFO, + format="😋 %(levelname)s: %(message)s", ) - if not latest_articles: - logging.info("📭 无新文章，无需推送") - else: - logging.info(f"🆕 获取到的最新文章：{latest_articles}") - - github_api_url = ( - f"https://api.github.com/repos/{github_username}/{github_repo}/issues" - f"?state=closed&label=subscribed&per_page=200" - ) - logging.info(f"🔎 正在从 GitHub 获取订阅邮箱：{github_api_url}") - email_list = extract_emails_from_issues(github_api_url) - - if not email_list: - logging.info("⚠️ 无订阅邮箱，请检查格式或是否有订阅者") - sys.exit(0) - logging.info(f"📬 获取到邮箱列表：{email_list}") +def main() -> None: + """Load configuration and run the application.""" + configure_logging() + app = FriendCircleLiteApplication(load_config("./conf.yaml")) + app.run() - for article in latest_articles: - template_data = { - "title": article["title"], - "summary": article["summary"], - "published": article["published"], - "link": article["link"], - "website_title": website_title, - "github_issue_url": ( - f"https://github.com/{github_username}/{github_repo}" - "/issues?q=is%3Aissue+is%3Aclosed" - ), - } - send_emails( - emails=email_list["emails"], - sender_email=sender_email, - smtp_server=server, - port=port, - password=password, - subject=f"{website_title} の最新文章：{article['title']}", - body=( - f"📄 文章标题：{article['title']}\n" - f"🔗 链接：{article['link']}\n" - f"📝 简介：{article['summary']}\n" - f"🕒 发布时间：{article['published']}" - ), - template_path=email_template, - template_data=template_data, - use_tls=use_tls - ) +if __name__ == "__main__": + main() diff --git a/server.py b/server.py deleted file mode 100644 index 2fb50f4dc2d..00000000000 --- a/server.py +++ /dev/null @@ -1,95 +0,0 @@ -from fastapi import FastAPI -from fastapi.staticfiles import StaticFiles -from fastapi.responses import FileResponse, HTMLResponse, JSONResponse -from starlette.middleware.cors import CORSMiddleware -import json -import random - -from friend_circle_lite.get_info import fetch_and_process_data, sort_articles_by_time -from friend_circle_lite.get_conf import load_config - -app = FastAPI() - -# 设置静态文件目录 -app.mount("/static", StaticFiles(directory="static"), name="static") -app.mount("/main", StaticFiles(directory="main"), name="main") - -# 添加 CORS 中间件 -app.add_middleware( - CORSMiddleware, - allow_origins=["*"], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], -) -# 返回图标图片 -@app.get("/favicon.ico", response_class=HTMLResponse) -async def favicon(): - return FileResponse('static/favicon.ico') - -# 返回背景图片 -@app.get("/bg-light.webp", response_class=HTMLResponse) -async def bg_light(): - return FileResponse('static/bg-light.webp') - -# 返回背景图片 -@app.get("/bg-dark.webp", response_class=HTMLResponse) -async def bg_dark(): - return FileResponse('static/bg-dark.webp') - -# 返回资源文件 -# 返回 CSS 文件 -@app.get("/fclite.css", response_class=HTMLResponse) -async def get_fclite_css(): - return FileResponse('./main/fclite.css') - -# 返回 JS 文件 -@app.get("/fclite.js", response_class=HTMLResponse) -async def get_fclite_js(): - return FileResponse('./main/fclite.js') - -@app.get("/", response_class=HTMLResponse) -async def root(): - return FileResponse('./static/index.html') - -@app.get('/all.json') -async def get_all_articles(): - try: - with open('./all.json', 'r', encoding='utf-8') as f: - articles_data = json.load(f) - return JSONResponse(content=articles_data) - except FileNotFoundError: - return JSONResponse(content={"error": "File not found"}, status_code=404) - except json.JSONDecodeError: - return JSONResponse(content={"error": "Failed to decode JSON"}, status_code=500) - -@app.get('/errors.json') -async def get_error_friends(): - try: - with open('./errors.json', 'r', encoding='utf-8') as f: - errors_data = json.load(f) - return JSONResponse(content=errors_data) - except FileNotFoundError: - return JSONResponse(content={"error": "File not found"}, status_code=404) - except json.JSONDecodeError: - return JSONResponse(content={"error": "Failed to decode JSON"}, status_code=500) - -@app.get('/random') -async def get_random_article(): - try: - with open('./all.json', 'r', encoding='utf-8') as f: - articles_data = json.load(f) - if articles_data.get("article_data"): - random_article = random.choice(articles_data["article_data"]) - return JSONResponse(content=random_article) - else: - return JSONResponse(content={"error": "No articles available"}, status_code=404) - except FileNotFoundError: - return JSONResponse(content={"error": "File not found"}, status_code=404) - except json.JSONDecodeError: - return JSONResponse(content={"error": "Failed to decode JSON"}, status_code=500) - -if __name__ == '__main__': - # 启动FastAPI应用 - import uvicorn - uvicorn.run(app, host='0.0.0.0', port=1223) diff --git a/server/deploy-home.html b/server/deploy-home.html deleted file mode 100644 index 49148026ac5..00000000000 --- a/server/deploy-home.html +++ /dev/null @@ -1,65 +0,0 @@ - - - - - - - 轻量友链朋友圈 - - - - -
- Avatar -

Friend-Circle-Lite
服务已运行

-
- 查看文档 - 测试接口 -
-
- - diff --git a/server/requirements-server.txt b/server/requirements-server.txt deleted file mode 100644 index f0615cfd093..00000000000 --- a/server/requirements-server.txt +++ /dev/null @@ -1,2 +0,0 @@ -fastapi -uvicorn \ No newline at end of file diff --git a/static/_headers b/static/_headers new file mode 100644 index 00000000000..97f1890683d --- /dev/null +++ b/static/_headers @@ -0,0 +1,4 @@ +/* + Access-Control-Allow-Origin: * + Access-Control-Allow-Methods: GET, POST, OPTIONS, PUT, DELETE + Access-Control-Allow-Headers: * \ No newline at end of file diff --git a/static/edgeone.json b/static/edgeone.json new file mode 100644 index 00000000000..1d4df629b8d --- /dev/null +++ b/static/edgeone.json @@ -0,0 +1,21 @@ +{ + "headers": [ + { + "source": "/*", + "headers": [ + { + "key": "Access-Control-Allow-Origin", + "value": "*" + }, + { + "key": "Access-Control-Allow-Methods", + "value": "GET, POST, OPTIONS, PUT, DELETE" + }, + { + "key": "Access-Control-Allow-Headers", + "value": "*" + } + ] + } + ] +} diff --git a/static/index.html b/static/index.html index c6500042755..80d336cad96 100644 --- a/static/index.html +++ b/static/index.html @@ -1,220 +1,66 @@ - - - + + - - - - - - Friend-Circle-Lite + + + + + 友链清册 · Friend Circle Lite + + + - - -
- Avatar -

Friend-Circle-Lite
服务已运行

-
- 查看文档 - 测试接口 -
- -
-
-
-
- - - - - +
+ +

A quiet index of good places

友链清册

把值得访问的站点与近来的文字，安静地收在同一页。

+
+

友链可达性数据

+ +
+ - - \ No newline at end of file + diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000000..011a32e9287 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test package for Friend-Circle-Lite.""" diff --git a/tests/test_refactor_contracts.py b/tests/test_refactor_contracts.py new file mode 100644 index 00000000000..53ba5f774c3 --- /dev/null +++ b/tests/test_refactor_contracts.py @@ -0,0 +1,1336 @@ +import unittest +import json +import sqlite3 +import tempfile +from contextlib import closing +from datetime import datetime, timedelta +from pathlib import Path +from unittest.mock import patch + +import requests + +from friend_circle_lite.config.models import ProxySettings +from friend_circle_lite.config.printer import print_startup_config +from friend_circle_lite.crawler.http_client import WebFetchClient +from friend_circle_lite.crawler.service import FeedResolver, FriendCircleCrawlService, SingleSiteCrawler +from friend_circle_lite.all_friends import deal_with_large_data, merge_link_data_from_json_url +from friend_circle_lite.app_config import ApplicationConfig +from friend_circle_lite.cli import FriendCircleLiteApplication +from friend_circle_lite.link_checker.service import LinkReachabilityService, RetryBackoffPolicy +from friend_circle_lite.models import Article, CacheRecord, FeedEndpoint, LinkCheckRecord, LinkMethodStatus, Website +from friend_circle_lite.outputs.legacy_api import _to_public_link +from friend_circle_lite.storage.diagnostics import SQLiteDebugDumper +from friend_circle_lite.utils.json import write_json +from friend_circle_lite.utils.config import load_raw_config + + +class RefactorContractsTest(unittest.TestCase): + def test_repository_variable_overrides_nested_config_without_editing_yaml(self): + from friend_circle_lite.utils.config import _apply_config_overrides + + original = { + "merge_settings": { + "enable": False, + "remote_base_url": "https://default.example", + }, + "spider_settings": {"article_count": 5}, + } + + with patch.dict( + "os.environ", + { + "FCL_CONFIG_OVERRIDES": ( + "merge_settings.enable=true\n" + "merge_settings.remote_base_url=https://remote.example\n" + "spider_settings.article_count=8\n" + "# ignored comment\n" + ) + }, + clear=True, + ): + loaded = load_raw_config("conf.yaml") + overridden = _apply_config_overrides(original, "merge_settings.enable=true\nspider_settings.article_count=8") + + self.assertTrue(loaded["merge_settings"]["enable"]) + self.assertEqual(loaded["merge_settings"]["remote_base_url"], "https://remote.example") + self.assertEqual(loaded["spider_settings"]["article_count"], 8) + self.assertFalse(original["merge_settings"]["enable"]) + self.assertEqual(original["spider_settings"]["article_count"], 5) + self.assertTrue(overridden["merge_settings"]["enable"]) + self.assertEqual(overridden["spider_settings"]["article_count"], 8) + + def test_github_action_schedule_uses_22_minute_offset(self): + workflow = Path(".github/workflows/friend_circle_lite.yml").read_text(encoding="utf-8") + + self.assertIn('cron: "22 */4 * * *"', workflow) + self.assertNotIn('cron: "0 */4 * * *"', workflow) + + def test_github_actions_opt_into_node24_runtime(self): + workflow_paths = [ + Path(".github/workflows/friend_circle_lite.yml"), + Path(".github/workflows/deal_subscribe_issue.yml"), + ] + + for workflow_path in workflow_paths: + with self.subTest(workflow=str(workflow_path)): + workflow = workflow_path.read_text(encoding="utf-8") + self.assertIn("FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true", workflow) + + def test_publish_workflow_skips_unchanged_page_tree(self): + workflow = Path(".github/workflows/friend_circle_lite.yml").read_text(encoding="utf-8") + + self.assertIn('git fetch origin "${PAGE_BRANCH}:refs/remotes/origin/${PAGE_BRANCH}"', workflow) + self.assertIn('published_tree=$(git rev-parse "origin/${PAGE_BRANCH}^{tree}")', workflow) + self.assertIn("generated_tree=$(git write-tree)", workflow) + self.assertIn('[ "$generated_tree" = "$published_tree" ]', workflow) + self.assertIn("No static asset changes; skipping page branch update.", workflow) + + def test_static_index_is_standalone_dashboard_with_view_switch(self): + html = Path("static/index.html").read_text(encoding="utf-8") + + self.assertIn('data-view="links"', html) + self.assertIn('data-view="articles"', html) + self.assertIn('["first24", "前 24 条"]', html) + link_config_start = html.index("links:", html.index("viewConfig")) + link_config = html[link_config_start:html.index("articles:", link_config_start)] + self.assertIn('defaultFilter: "unreachable"', link_config) + expected_link_filters = [ + '["unreachable", "不可达"]', + '["uncrawlable", "不可抓取"]', + '["noBacklink", "无反链"]', + '["stale", "久未更新"]', + '["all", "全部"]', + ] + for item in expected_link_filters: + self.assertIn(item, link_config) + positions = [link_config.index(item) for item in expected_link_filters] + self.assertEqual(positions, sorted(positions)) + self.assertNotIn('["crawlable", "可抓取"]', link_config) + self.assertNotIn('["backlink", "反链"]', link_config) + self.assertLess( + html.index('["unreachable", "不可达"]', link_config_start), + html.index('["all", "全部"]', link_config_start), + ) + article_config_start = html.index("articles:", html.index("viewConfig")) + self.assertLess( + html.index('["first24", "前 24 条"]', article_config_start), + html.index('["all", "全部"]', article_config_start), + ) + self.assertIn("scrollbar-width: thin", html) + self.assertIn("::-webkit-scrollbar", html) + self.assertIn(".dashboard {\n display: grid;\n gap: 14px;", html) + self.assertIn(".action-strip", html) + self.assertIn(".summary-row", html) + self.assertIn(".control-row", html) + self.assertIn(".metrics-row", html) + self.assertIn("summary-row", html) + self.assertIn("control-row", html) + self.assertIn("metrics metrics-row", html) + self.assertIn("toolbar top-toolbar", html) + self.assertIn("--control-height: 44px", html) + self.assertIn("height: var(--control-height)", html) + self.assertIn(".content-grid {\n display: grid;\n align-items: start;", html) + self.assertIn(".content-grid.is-links {\n grid-template-columns: repeat(3, minmax(0, 1fr));", html) + self.assertIn('content.className = `content-grid ${state.view === "links" ? "is-links" : "is-articles"}`;', html) + self.assertIn(".link-card.featured", html) + self.assertIn('if (state.filter === "uncrawlable") return !link.rss;', html) + self.assertIn('if (state.filter === "noBacklink") return link.backlink !== true;', html) + self.assertIn("sortLinks", html) + self.assertIn("visibleDetails", html) + self.assertIn("renderSurfaceFacts", html) + self.assertIn("primaryInsight", html) + self.assertIn("formatUnreachableDays", html) + self.assertIn("const unreachableDays = formatUnreachableDays(link);", html) + self.assertIn('label: "不可达时长"', html) + self.assertIn("unreachable_days", html) + self.assertNotIn("fail_count", html) + self.assertNotIn("fails:", html) + self.assertNotIn("formatFailureAge", html) + self.assertNotIn("fa-solid fa-rotate", html) + self.assertNotIn("--leaf", html) + self.assertNotIn("--amber", html) + self.assertNotIn("--red", html) + self.assertNotIn(".icon-badge.good", html) + self.assertNotIn(".icon-badge.warn", html) + self.assertNotIn(".icon-badge.bad", html) + self.assertIn("align-self: start", html) + self.assertIn("-webkit-line-clamp: 2", html) + self.assertIn("article-avatar-mark", html) + self.assertIn("article.avatar", html) + self.assertIn("https://github.com/willow-god/Friend-Circle-Lite", html) + self.assertIn("poem-background", html) + self.assertIn("link-status", html) + self.assertIn("surface-facts", html) + self.assertIn("title=", html) + self.assertIn("Promise.all", html) + self.assertIn("link.json", html) + self.assertIn("all.json", html) + self.assertNotIn(".metric::before", html) + self.assertNotIn(".link-card::before", html) + self.assertNotIn(".article-card::before", html) + self.assertNotIn("fclite.js", html) + self.assertNotIn("fclite.css", html) + + def test_link_check_uses_cached_rss_without_homepage_request(self): + class Store: + def load_records(self, urls): + return {} + + def save_records(self, records): + return True + + class Parser: + def parse(self, feed_url, count=1, blog_url=""): + return [Article(title="Post", author="Site", link="https://site.example/post", published="2026-06-07 10:00")] + + class Fetcher: + calls = [] + + def get(self, *args, **kwargs): + self.calls.append(args[0]) + raise AssertionError("主页不应该在 RSS 可解析时被请求") + + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"enable": True}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + feed_records=[CacheRecord(name="Site", url="https://site.example/rss.xml", source="cache")], + feed_parser=Parser(), + fetcher=Fetcher(), + ) + + records = service.check_websites([Website(name="Site", url="https://site.example", avatar="avatar.png")]) + + self.assertTrue(records[0].reachable) + self.assertTrue(records[0].crawl_allowed) + self.assertEqual(records[0].best_method, "rss_cache") + + def test_link_check_records_latest_rss_article_staleness(self): + class Store: + def load_records(self, urls): + return {} + + def save_records(self, records): + return True + + published = (datetime.now() - timedelta(days=8, hours=1)).strftime("%Y-%m-%d %H:%M") + + class Parser: + last_latency = 0.42 + + def parse(self, feed_url, count=1, blog_url=""): + return [Article(title="Latest", author="Site", link="https://site.example/post", published=published)] + + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"enable": True}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + feed_records=[CacheRecord(name="Site", url="https://site.example/rss.xml", source="cache")], + feed_parser=Parser(), + ) + + record = service.check_websites([Website(name="Site", url="https://site.example", avatar="avatar.png")])[0] + public_link = record.to_link_dict() + + self.assertEqual(record.last_post_published, published) + self.assertEqual(record.last_post_days_ago, 8) + self.assertEqual(public_link["updated"], published) + self.assertEqual(public_link["stale_days"], 8) + + def test_unreachable_public_link_keeps_latency_unknown(self): + public_link = _to_public_link({ + "name": "Blocked", + "url": "https://blocked.example/", + "reachable": False, + "crawl_allowed": False, + "best_latency": 0.31, + "unreachable_since": "2026-06-26 12:00:00", + }) + + self.assertEqual(public_link["latency"], -1) + self.assertNotIn("fail_count", public_link) + self.assertEqual(public_link["unreachable_since"], "2026-06-26 12:00:00") + + def test_unreachable_record_tracks_since_time_and_rounds_days_up(self): + record = LinkCheckRecord( + name="Blocked", + url="https://blocked.example/", + checked_at="2026-06-27 10:00:00", + reachable=False, + unreachable_since="2026-06-26 12:00:00", + ) + + with patch("friend_circle_lite.domain.models.datetime") as fake_datetime: + fake_datetime.now.return_value = datetime(2026, 6, 27, 10, 0, 0) + fake_datetime.strptime.side_effect = datetime.strptime + public_link = record.to_link_dict() + + self.assertEqual(public_link["unreachable_days"], 1) + self.assertEqual(public_link["unreachable_since"], "2026-06-26 12:00:00") + self.assertNotIn("fail_count", public_link) + + def test_first_unreachable_check_uses_current_time_as_since_time(self): + class Store: + def load_records(self, urls): + return {} + + def save_records(self, records): + return True + + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"enable": True}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + ) + website = Website(name="Blocked", url="https://blocked.example/", avatar="avatar.png") + + with patch.object(service, "_now_text", return_value="2026-06-27 10:00:00"): + record = service._compose_non_rss_record( + website, + cached=None, + homepage=LinkMethodStatus(False, None, -1), + api=LinkMethodStatus(False, None, -1), + ) + + self.assertFalse(record.reachable) + self.assertEqual(record.unreachable_since, "2026-06-27 10:00:00") + + def test_unreachable_cache_can_be_reused_within_max_age(self): + class Store: + def is_fresh(self, record, max_age_hours): + return True + + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"max_age_hours": 24}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + ) + cached = LinkCheckRecord( + name="Blocked", + url="https://blocked.example/", + checked_at="2026-06-27 08:00:00", + reachable=False, + best_latency=-1, + unreachable_since="2026-06-26 12:00:00", + ) + + self.assertTrue(service._can_reuse_cached_record(cached, Website(name="Blocked", url="https://blocked.example/"))) + + def test_retry_backoff_policy_scales_with_continuous_failure_days(self): + with patch("friend_circle_lite.domain.models.datetime") as fake_datetime: + fake_datetime.now.return_value = datetime(2026, 6, 27, 12, 0, 0) + fake_datetime.strptime.side_effect = datetime.strptime + + self.assertEqual(RetryBackoffPolicy.effective_max_age_hours("2026-06-18 12:00:00", 24), 24) + self.assertEqual(RetryBackoffPolicy.effective_max_age_hours("2026-06-17 12:00:00", 24), 120) + self.assertEqual(RetryBackoffPolicy.effective_max_age_hours("2026-05-28 12:00:00", 24), 240) + self.assertEqual(RetryBackoffPolicy.effective_max_age_hours("2026-04-28 12:00:00", 24), 360) + + def test_rss_unavailable_record_tracks_since_time_without_public_json_output(self): + class Store: + def load_records(self, urls): + return {} + + def save_records(self, records): + return True + + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"enable": True}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + ) + website = Website(name="NoRSS", url="https://norss.example/", avatar="avatar.png") + + with patch.object(service, "_now_text", return_value="2026-06-27 10:00:00"): + record = service._compose_non_rss_record( + website, + cached=None, + homepage=LinkMethodStatus(True, 200, 0.2), + api=LinkMethodStatus(False, None, -1), + ) + + self.assertTrue(record.reachable) + self.assertFalse(record.crawl_allowed) + self.assertEqual(record.rss_unavailable_since, "2026-06-27 10:00:00") + self.assertEqual(record.unreachable_since, "") + public_link = record.to_link_dict() + self.assertNotIn("rss_unavailable_since", public_link) + + def test_rss_unavailable_cache_uses_dynamic_backoff_interval(self): + class Store: + calls = [] + + def is_fresh(self, record, max_age_hours): + self.calls.append(max_age_hours) + return max_age_hours == 120 + + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"max_age_hours": 24}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + ) + cached = LinkCheckRecord( + name="NoRSS", + url="https://norss.example/", + checked_at="2026-06-26 08:00:00", + reachable=True, + crawl_allowed=False, + best_method="homepage", + best_latency=0.2, + rss_unavailable_since="2026-06-17 08:00:00", + ) + + with patch("friend_circle_lite.link_checker.service.datetime") as fake_datetime: + fake_datetime.now.return_value = datetime(2026, 6, 27, 8, 0, 0) + fake_datetime.strptime.side_effect = datetime.strptime + reused = service._can_reuse_cached_record(cached, Website(name="NoRSS", url="https://norss.example/")) + + self.assertTrue(reused) + self.assertEqual(service.store.calls, [120]) + + def test_link_check_logs_total_cached_and_actual_check_counts(self): + class Store: + def load_records(self, urls): + return { + "https://cached.example/": LinkCheckRecord( + name="Cached", + url="https://cached.example/", + checked_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + reachable=True, + crawl_allowed=False, + best_method="homepage", + best_latency=0.2, + ) + } + + def save_records(self, records): + return True + + def is_fresh(self, record, max_age_hours): + return True + + class Discovery: + def discover(self, website_url): + return None + + class Response: + status_code = 200 + + class Fetcher: + def get(self, *args, **kwargs): + return type("Result", (), {"response": Response(), "latency": 0.1, "success": True})() + + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"max_age_hours": 24}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + feed_parser=type("Parser", (), {"parse": lambda self, *args, **kwargs: [], "last_latency": 0.01})(), + feed_discovery=Discovery(), + fetcher=Fetcher(), + ) + + with patch("logging.info") as info: + service.check_websites([ + Website(name="Cached", url="https://cached.example/", avatar="cached.png"), + Website(name="Fresh", url="https://fresh.example/", avatar="fresh.png"), + ]) + + messages = "\n".join(str(call.args[0]) for call in info.call_args_list) + self.assertIn("[友链检测]", messages) + self.assertIn("友链总数 2 个", messages) + self.assertIn("缓存复用 1 个", messages) + self.assertIn("本次实际检测 1 个", messages) + + def test_crawler_entry_logs_source_and_article_limit_with_module_label(self): + config = ApplicationConfig.from_dict({ + "spider_settings": { + "enable": True, + "json_url": "https://example.com/friends.json", + "article_count": 3, + }, + "runtime_paths": { + "all_json_file": "./tmp/all.json", + "errors_json_file": "./tmp/errors.json", + "link_json_file": "./tmp/link.json", + }, + }) + payload = ({"statistical_data": {}, "article_data": []}, [], {"statistical_data": {}, "link_data": []}) + + with patch("friend_circle_lite.cli.fetch_and_process_data", return_value=payload), \ + patch("friend_circle_lite.cli.write_json"), \ + patch("logging.info") as info: + FriendCircleLiteApplication(config).run_crawler_if_enabled() + + messages = "\n".join(str(call.args[0]) for call in info.call_args_list) + self.assertIn("[爬虫入口]", messages) + self.assertIn("https://example.com/friends.json", messages) + self.assertIn("每站最多 3 篇文章", messages) + + def test_link_check_reuses_cached_linkpage_when_only_trailing_slash_differs(self): + class Store: + def load_records(self, urls): + return { + "https://wcowin.work/": LinkCheckRecord( + name="Wcowin", + url="https://wcowin.work/", + linkpage="https://wcowin.work/link/", + checked_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + reachable=True, + crawl_allowed=False, + best_method="homepage", + best_latency=0.2, + ) + } + + def save_records(self, records): + raise AssertionError("只差末尾斜杠时不应该重新检测并写入缓存") + + def is_fresh(self, record, max_age_hours): + return True + + class Fetcher: + def get(self, *args, **kwargs): + raise AssertionError("只差末尾斜杠时不应该重新请求网站") + + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({ + "link_check": { + "enable_backlink_check": True, + "author_url": "blog.liushen.fun", + } + }).link_check, + proxy_settings=ProxySettings(), + store=Store(), + fetcher=Fetcher(), + ) + + records = service.check_websites([ + Website( + name="Wcowin", + url="https://wcowin.work/", + avatar="avatar.png", + linkpage="https://wcowin.work/link", + ) + ]) + + self.assertEqual(len(records), 1) + self.assertTrue(records[0].reachable) + self.assertFalse(records[0].crawl_allowed) + + def test_linkpage_change_refreshes_backlink_only(self): + saved_records = [] + + class Store: + def load_records(self, urls): + return { + "https://site.example/": LinkCheckRecord( + name="Site", + url="https://site.example/", + linkpage="https://site.example/old-links/", + checked_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + reachable=True, + crawl_allowed=False, + best_method="homepage", + best_latency=0.2, + backlink_checked=True, + has_author_link=False, + ) + } + + def save_records(self, records): + saved_records.extend(records) + return True + + def is_fresh(self, record, max_age_hours): + return True + + class Response: + status_code = 200 + text = '清羽飞扬' + + class Fetcher: + calls = [] + + def get(self, url, *args, **kwargs): + self.calls.append(url) + if url != "https://site.example/new-links/": + raise AssertionError("反链页变化时只应请求新的友链页") + return type("Result", (), {"response": Response(), "latency": 0.2, "success": True})() + + fetcher = Fetcher() + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({ + "link_check": { + "enable_backlink_check": True, + "author_url": "blog.liushen.fun", + } + }).link_check, + proxy_settings=ProxySettings(), + store=Store(), + fetcher=fetcher, + ) + + records = service.check_websites([ + Website( + name="Site", + url="https://site.example/", + avatar="avatar.png", + linkpage="https://site.example/new-links/", + ) + ]) + + self.assertEqual(fetcher.calls, ["https://site.example/new-links/"]) + self.assertEqual(len(saved_records), 1) + self.assertTrue(records[0].has_author_link) + self.assertEqual(records[0].linkpage, "https://site.example/new-links/") + + def test_link_check_revalidates_legacy_crawlable_cache_without_rss_method(self): + class Store: + def load_records(self, urls): + return { + "https://site.example": LinkCheckRecord( + name="Site", + url="https://site.example", + checked_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + reachable=True, + crawl_allowed=True, + best_method="direct", + ) + } + + def save_records(self, records): + return True + + def is_fresh(self, record, max_age_hours): + return True + + class Parser: + def parse(self, feed_url, count=1, blog_url=""): + return [Article(title="Post", author="Site", link="https://site.example/post", published="2026-06-07 10:00")] + + class Discovery: + calls = 0 + + def discover(self, website_url): + self.calls += 1 + return FeedEndpoint(url="https://site.example/rss.xml", feed_type="specific", source="auto") + + discovery = Discovery() + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"max_age_hours": 24}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + feed_records=[], + feed_parser=Parser(), + feed_discovery=discovery, + ) + + records = service.check_websites([Website(name="Site", url="https://site.example", avatar="avatar.png")]) + + self.assertEqual(discovery.calls, 1) + self.assertTrue(records[0].crawl_allowed) + self.assertEqual(records[0].best_method, "rss") + + def test_link_check_revalidates_fresh_cache_without_measured_latency(self): + saved_records = [] + + class Store: + def load_records(self, urls): + return { + "https://site.example/": LinkCheckRecord( + name="Site", + url="https://site.example/", + checked_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + reachable=True, + crawl_allowed=True, + best_method="rss_cache", + best_latency=-1, + ) + } + + def save_records(self, records): + saved_records.extend(records) + return True + + def is_fresh(self, record, max_age_hours): + return True + + class Parser: + last_latency = 0.23 + + def parse(self, feed_url, count=1, blog_url=""): + return [Article(title="Post", author="Site", link="https://site.example/post", published="2026-06-07 10:00")] + + class Discovery: + calls = 0 + + def discover(self, website_url): + self.calls += 1 + return FeedEndpoint(url="https://site.example/rss.xml", feed_type="specific", source="auto") + + discovery = Discovery() + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"max_age_hours": 24}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + feed_records=[], + feed_parser=Parser(), + feed_discovery=discovery, + ) + + records = service.check_websites([Website(name="Site", url="https://site.example", avatar="avatar.png")]) + + self.assertEqual(discovery.calls, 1) + self.assertEqual(len(saved_records), 1) + self.assertEqual(records[0].best_latency, 0.23) + self.assertGreater(records[0].to_link_dict()["latency"], 0) + + def test_link_check_revalidates_fresh_homepage_cache_without_measured_latency(self): + saved_records = [] + + class Store: + def load_records(self, urls): + return { + "https://site.example/": LinkCheckRecord( + name="Site", + url="https://site.example/", + checked_at=datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + reachable=True, + crawl_allowed=False, + best_method="homepage", + best_latency=-1, + ) + } + + def save_records(self, records): + saved_records.extend(records) + return True + + def is_fresh(self, record, max_age_hours): + return True + + class Discovery: + def discover(self, website_url): + return None + + class Response: + status_code = 200 + + class Fetcher: + calls = 0 + + def get(self, *args, **kwargs): + self.calls += 1 + return type("Result", (), {"response": Response(), "latency": 0.31, "success": True})() + + fetcher = Fetcher() + service = LinkReachabilityService( + config=ApplicationConfig.from_dict({"link_check": {"max_age_hours": 24}}).link_check, + proxy_settings=ProxySettings(), + store=Store(), + feed_parser=type("Parser", (), {"parse": lambda self, *args, **kwargs: [], "last_latency": 0.01})(), + feed_discovery=Discovery(), + fetcher=fetcher, + ) + + records = service.check_websites([Website(name="Site", url="https://site.example", avatar="avatar.png")]) + + self.assertEqual(fetcher.calls, 1) + self.assertEqual(len(saved_records), 1) + self.assertEqual(records[0].best_latency, 0.31) + + def test_article_crawl_does_not_rediscover_invalid_cached_rss(self): + class Parser: + def parse(self, feed_url, count=1, blog_url=""): + return [] + + class Discovery: + calls = 0 + + def discover(self, website_url): + self.calls += 1 + return FeedEndpoint(url="https://site.example/new-rss.xml", feed_type="specific", source="auto") + + discovery = Discovery() + resolver = FeedResolver( + discovery_service=discovery, + configured_feeds=[CacheRecord(name="Site", url="https://site.example/rss.xml", source="cache")], + ) + crawler = SingleSiteCrawler(parser_service=Parser(), resolver=resolver) + + result = crawler.crawl(Website(name="Site", url="https://site.example", avatar="avatar.png"), count=1) + + self.assertEqual(discovery.calls, 0) + self.assertEqual(result.status, "error") + self.assertEqual(result.cache_update.action, "none") + + def test_web_fetch_client_falls_back_to_proxy_url(self): + calls = [] + + class Response: + status_code = 200 + text = "ok" + + class Session: + def get(self, url, headers=None, timeout=None): + calls.append(url) + if len(calls) == 1: + raise requests.RequestException("direct failed") + return Response() + + client = WebFetchClient(Session(), ProxySettings(proxy_url="https://proxy.example/{url}")) + + result = client.get("https://site.example/feed.xml", desc="RSS") + + self.assertTrue(result.success) + self.assertTrue(result.used_proxy) + self.assertEqual(calls, [ + "https://site.example/feed.xml", + "https://proxy.example/https://site.example/feed.xml", + ]) + + def test_web_fetch_client_records_positive_latency_for_request_exception(self): + class Session: + def get(self, url, headers=None, timeout=None): + raise requests.RequestException("direct failed") + + client = WebFetchClient(Session(), ProxySettings()) + + with patch("time.time", side_effect=[10.0, 10.0]): + result = client.get("https://site.example/feed.xml", desc="RSS") + + self.assertFalse(result.success) + self.assertGreater(result.latency, 0) + self.assertNotEqual(result.latency, 0.0) + + def test_web_fetch_client_does_not_log_proxy_service_url(self): + class Session: + def __init__(self): + self.calls = 0 + + def get(self, url, headers=None, timeout=None): + self.calls += 1 + if self.calls == 1: + raise requests.RequestException("direct failed") + raise requests.RequestException("HTTPSConnectionPool(host='proxy.example', port=443)") + + client = WebFetchClient(Session(), ProxySettings(proxy_url="https://proxy.example/")) + + with patch("logging.warning") as warning: + client.get("https://site.example/feed.xml", desc="RSS") + + messages = "\n".join(str(call.args[0]) for call in warning.call_args_list) + self.assertNotIn("https://proxy.example", messages) + self.assertNotIn("proxy.example", messages) + self.assertIn("https://site.example/feed.xml", messages) + + def test_startup_config_does_not_log_proxy_service_url(self): + config = ApplicationConfig.from_dict({ + "proxy_settings": {"proxy_url": "https://proxy.example/"}, + }) + + with patch("logging.info") as info: + print_startup_config(config) + + messages = "\n".join(str(call.args[0]) for call in info.call_args_list) + self.assertNotIn("https://proxy.example", messages) + self.assertIn("代理", messages) + + def test_config_keeps_existing_yaml_keys(self): + config = ApplicationConfig.from_dict({ + "debug": True, + "spider_settings": { + "enable": True, + "json_url": "https://example.com/friends.json", + "article_count": 3, + }, + "proxy_settings": {"proxy_url": "https://proxy.example/"}, + "merge_settings": { + "enable": True, + "remote_base_url": "https://remote.example", + "merge_article_data": False, + "merge_link_check_data": True, + }, + "link_check": { + "enable": False, + "max_age_hours": 6, + "timeout": 9, + "max_workers": 2, + "status_api_url": "https://status.example?url={url}", + "enable_backlink_check": True, + "author_url": "example.com", + }, + "runtime_paths": { + "cache_file": "./tmp/state.sqlite3", + "all_json_file": "./public/all.json", + "errors_json_file": "./public/errors.json", + "link_json_file": "./public/link.json", + }, + "specific_RSS": [{"name": "Manual", "url": "https://example.com/feed.xml"}], + }) + + self.assertEqual(config.spider_settings.json_url, "https://example.com/friends.json") + self.assertTrue(config.debug) + self.assertEqual(config.spider_settings.article_count, 3) + self.assertEqual(config.proxy_settings.proxy_url, "https://proxy.example/") + self.assertTrue(config.merge_settings.enable) + self.assertFalse(config.merge_settings.merge_article_data) + self.assertTrue(config.link_check.enable) + self.assertEqual(config.link_check.author_url, "example.com") + self.assertEqual(config.runtime_paths.cache_file, "./tmp/state.sqlite3") + self.assertEqual(config.specific_rss[0]["name"], "Manual") + + def test_debug_env_enables_sqlite_dump(self): + with patch.dict("os.environ", {"FCL_DEBUG": "1"}): + config = ApplicationConfig.from_dict({}) + + self.assertTrue(config.debug) + + def test_debug_string_false_stays_disabled(self): + config = ApplicationConfig.from_dict({"debug": "false"}) + + self.assertFalse(config.debug) + + def test_website_and_link_record_public_shapes_are_stable(self): + website = Website.from_friend_item(["Alice", "https://alice.example", "https://alice.example/links", "avatar.png"]) + self.assertEqual(website.to_error_payload(), ["Alice", "https://alice.example/", "avatar.png"]) + + record = LinkCheckRecord( + name=website.name, + url=website.url, + avatar=website.avatar, + linkpage=website.linkpage, + checked_at="2026-06-06 12:00:00", + reachable=True, + crawl_allowed=True, + best_method="proxy", + best_latency=1.2, + backlink_checked=True, + has_author_link=True, + rss_crawl_reason="allowed_by_proxy", + direct=LinkMethodStatus(False, 403, 2.0), + proxy=LinkMethodStatus(True, 200, 1.2), + ) + + self.assertEqual(record.to_link_dict(), { + "name": "Alice", + "link": "https://alice.example/", + "link_page": "https://alice.example/links", + "avatar": "avatar.png", + "reachable": True, + "crawlable": True, + "latency": 1.2, + "unreachable_days": None, + "unreachable_since": "", + "has_backlink": True, + "updated": "", + "stale_days": None, + }) + + def test_public_link_uses_negative_latency_when_unreachable(self): + public_link = _to_public_link({ + "name": "Legacy", + "url": "https://legacy.example/", + "reachable": False, + "crawl_allowed": False, + "best_latency": -1, + }) + + self.assertEqual(public_link["latency"], -1) + + def test_homepage_url_normalization_adds_trailing_slash_to_paths(self): + website = Website.from_friend_item(["PathSite", "https://example.com/blog", "avatar.png"]) + + self.assertEqual(website.url, "https://example.com/blog/") + + def test_load_websites_deduplicates_by_normalized_homepage_url(self): + service = FriendCircleCrawlService(json_url="https://example.com/friends.json", count=1) + + class Response: + def raise_for_status(self): + return None + + def json(self): + return { + "friends": [ + ["Wcowin", "https://wcowin.work", "https://wcowin.work/link", "old.png"], + ["Wcowin", "https://wcowin.work/", "https://wcowin.work/link/", "new.png"], + ] + } + + class Session: + def get(self, *args, **kwargs): + return Response() + + websites = service._load_websites(Session()) + + self.assertEqual(len(websites), 1) + self.assertEqual(websites[0].url, "https://wcowin.work/") + self.assertEqual(websites[0].avatar, "new.png") + + def test_large_data_sorting_keeps_public_article_schema(self): + payload = { + "statistical_data": {"article_num": 0}, + "article_data": [ + {"title": "Old", "created": "2024-01-01 00:00", "link": "https://old", "author": "A", "avatar": "a.png"}, + {"title": "New", "created": "2024-01-02 00:00", "link": "https://new", "author": "B", "avatar": "b.png"}, + ], + } + + result = deal_with_large_data(payload) + + self.assertEqual([article["title"] for article in result["article_data"]], ["New", "Old"]) + self.assertEqual(result["statistical_data"]["article_num"], 2) + self.assertEqual(set(result["article_data"][0].keys()), {"title", "created", "link", "author", "avatar"}) + + def test_write_json_minifies_generated_payloads(self): + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "link.json" + + self.assertTrue(write_json(path, {"statistical_data": {"link_total_num": 1}, "link_data": [{"name": "站点"}]})) + + text = path.read_text(encoding="utf-8") + + self.assertEqual(text, '{"statistical_data":{"link_total_num":1},"link_data":[{"name":"站点"}]}') + self.assertEqual(json.loads(text)["link_data"][0]["name"], "站点") + + def test_write_json_preserves_semantically_unchanged_file(self): + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "link.json" + original = '{\n "link_data": [{"name": "站点"}],\n "statistical_data": {"link_total_num": 1}\n}\n' + path.write_text(original, encoding="utf-8") + + self.assertTrue(write_json(path, { + "statistical_data": {"link_total_num": 1}, + "link_data": [{"name": "站点"}], + })) + + self.assertEqual(path.read_text(encoding="utf-8"), original) + + def test_write_json_replaces_changed_file_with_minified_json(self): + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "link.json" + path.write_text('{"link_data":[]}', encoding="utf-8") + + self.assertTrue(write_json(path, { + "statistical_data": {"link_total_num": 1}, + "link_data": [{"name": "站点"}], + })) + + self.assertEqual( + path.read_text(encoding="utf-8"), + '{"statistical_data":{"link_total_num":1},"link_data":[{"name":"站点"}]}', + ) + + def test_write_json_replaces_invalid_or_undecodable_file(self): + invalid_files = { + "truncated": b'{"link_data":', + "non_utf8": b'\xff\xfe', + } + + for name, existing in invalid_files.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "link.json" + path.write_bytes(existing) + + self.assertTrue(write_json(path, {"link_data": []})) + self.assertEqual(path.read_text(encoding="utf-8"), '{"link_data":[]}') + + def test_write_json_skips_when_only_volatile_timestamps_change(self): + cases = { + "all_json_last_updated_time": { + "original": ( + '{"statistical_data":{"friends_num":1,"article_num":1,' + '"last_updated_time":"2026-08-01 10:00:00"},' + '"article_data":[{"title":"A","created":"2026-08-01 09:00","link":"https://a","author":"x","avatar":""}]}' + ), + "payload": { + "statistical_data": { + "friends_num": 1, + "article_num": 1, + "last_updated_time": "2026-08-15 23:59:59", + }, + "article_data": [{ + "title": "A", + "created": "2026-08-01 09:00", + "link": "https://a", + "author": "x", + "avatar": "", + }], + }, + }, + "link_json_last_checked_time": { + "original": ( + '{"statistical_data":{"link_total_num":1,"link_last_checked_time":"2026-08-01 10:00:00"},' + '"link_data":[{"name":"站点","link":"https://site.example","reachable":true}]}' + ), + "payload": { + "statistical_data": { + "link_total_num": 1, + "link_last_checked_time": "2026-08-15 23:59:59", + }, + "link_data": [{ + "name": "站点", + "link": "https://site.example", + "reachable": True, + }], + }, + }, + } + + for name, case in cases.items(): + with self.subTest(name=name), tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / f"{name}.json" + path.write_text(case["original"], encoding="utf-8") + + self.assertTrue(write_json(path, case["payload"])) + self.assertEqual(path.read_text(encoding="utf-8"), case["original"]) + + def test_write_json_rewrites_when_content_changes_even_if_time_matches(self): + with tempfile.TemporaryDirectory() as temp_dir: + path = Path(temp_dir) / "all.json" + path.write_text( + '{"statistical_data":{"friends_num":1,"last_updated_time":"2026-08-01 10:00:00"},' + '"article_data":[{"title":"Old"}]}', + encoding="utf-8", + ) + + self.assertTrue(write_json(path, { + "statistical_data": { + "friends_num": 1, + "last_updated_time": "2026-08-01 10:00:00", + }, + "article_data": [{"title": "New"}], + })) + + self.assertEqual( + path.read_text(encoding="utf-8"), + '{"statistical_data":{"friends_num":1,"last_updated_time":"2026-08-01 10:00:00"},' + '"article_data":[{"title":"New"}]}', + ) + + def test_link_merge_keeps_best_reachability_shape(self): + local = { + "statistical_data": {}, + "link_data": [{ + "name": "Site", + "link": "https://site.example", + "link_page": "", + "avatar": "", + "reachable": True, + "crawlable": False, + "method": "api", + "latency": 3.0, + "unreachable_since": "2026-06-05 12:00:00", + "unreachable_days": 2, + "checked_at": "2026-06-05 12:00:00", + "has_backlink": None, + "reason": "blocked_api_only", + }], + } + remote = { + "statistical_data": {}, + "link_data": [{ + "name": "Site", + "link": "https://site.example", + "link_page": "", + "avatar": "", + "reachable": True, + "crawlable": True, + "method": "proxy", + "latency": 1.0, + "unreachable_since": "", + "unreachable_days": None, + "checked_at": "2026-06-06 12:00:00", + "has_backlink": True, + "reason": "allowed_by_proxy", + }], + } + + class Response: + def json(self): + return remote + + with patch("requests.get", return_value=Response()): + merged = merge_link_data_from_json_url(local, "https://remote.example/link.json") + + self.assertNotIn("method", merged["link_data"][0]) + self.assertNotIn("checked_at", merged["link_data"][0]) + self.assertNotIn("reason", merged["link_data"][0]) + self.assertEqual(merged["link_data"][0]["latency"], 1.0) + self.assertNotIn("fail_count", merged["link_data"][0]) + self.assertIsNone(merged["link_data"][0]["unreachable_days"]) + self.assertTrue(merged["link_data"][0]["has_backlink"]) + self.assertEqual(merged["statistical_data"]["link_total_num"], 1) + self.assertNotIn("stats", merged) + self.assertNotIn("links", merged) + + def test_all_json_statistics_do_not_include_link_statistics(self): + with tempfile.TemporaryDirectory() as temp_dir: + service = FriendCircleCrawlService( + json_url="https://example.com/friends.json", + count=1, + cache_file=str(Path(temp_dir) / "cache.sqlite3"), + ) + website = Website(name="Site", url="https://site.example", avatar="avatar.png") + + def load_websites(_session): + return [website] + + def check_links(_websites, _feed_records, _manual_names): + return [ + LinkCheckRecord( + name="Site", + url="https://site.example", + avatar="avatar.png", + checked_at="2026-06-07 12:00:00", + reachable=False, + crawl_allowed=False, + ) + ] + + service._load_websites = load_websites + service._check_links = check_links + + all_payload, _errors, link_payload = service.run() + + self.assertEqual(set(all_payload["statistical_data"].keys()), { + "friends_num", + "active_num", + "error_num", + "article_num", + "last_updated_time", + }) + self.assertIn("link_total_num", link_payload["statistical_data"]) + self.assertIn("link_data", link_payload) + self.assertNotIn("stats", link_payload) + self.assertNotIn("links", link_payload) + + def test_crawl_filter_uses_crawl_allowed_and_feed_cache_not_best_method(self): + with tempfile.TemporaryDirectory() as temp_dir: + service = FriendCircleCrawlService( + json_url="https://example.com/friends.json", + count=1, + specific_rss=[{"name": "WithRSS", "url": "https://with.example/rss.xml"}], + cache_file=str(Path(temp_dir) / "cache.sqlite3"), + ) + websites = [ + Website(name="WithRSS", url="https://with.example", avatar="with.png"), + Website(name="NoRSS", url="https://no.example", avatar="no.png"), + ] + + def load_websites(_session): + return websites + + def check_links(_websites, _feed_records, _manual_names): + return [ + LinkCheckRecord( + name="WithRSS", + url="https://with.example", + avatar="with.png", + checked_at="2026-06-07 12:00:00", + reachable=True, + crawl_allowed=True, + best_method="homepage", + ), + LinkCheckRecord( + name="NoRSS", + url="https://no.example", + avatar="no.png", + checked_at="2026-06-07 12:00:00", + reachable=True, + crawl_allowed=True, + best_method="rss", + ), + ] + + crawled_names = [] + + def crawl(_crawler, website, count): + crawled_names.append(website.name) + return type("Result", (), { + "website": website, + "status": "active", + "articles": [ + Article( + title=f"{website.name} Post", + author=website.name, + link=f"{website.url}/post", + published="2026-06-07 10:00", + avatar=website.avatar, + ) + ], + "feed_url": "https://with.example/rss.xml", + "feed_type": "specific", + "source_used": "manual", + "cache_update": type("Update", (), {"name": None, "action": "none", "url": None})(), + })() + + service._load_websites = load_websites + service._check_links = check_links + + with patch("friend_circle_lite.crawler.service.SingleSiteCrawler.crawl", crawl): + service.run() + + self.assertEqual(crawled_names, ["WithRSS"]) + + def test_sqlite_debug_dumper_prints_all_rows_and_cleans_extra_columns(self): + with tempfile.TemporaryDirectory() as temp_dir: + db_path = Path(temp_dir) / "cache.sqlite3" + with closing(sqlite3.connect(db_path)) as connection: + connection.execute( + """ + CREATE TABLE feed_cache ( + name TEXT PRIMARY KEY, + url TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'cache', + old_column TEXT + ) + """ + ) + connection.execute( + "INSERT INTO feed_cache(name, url, source, old_column) VALUES (?, ?, ?, ?)", + ("Site", "https://site.example/rss.xml", "cache", "legacy"), + ) + connection.commit() + + output = SQLiteDebugDumper(db_path).run() + + self.assertIn("feed_cache", output) + self.assertIn("https://site.example/rss.xml", output) + self.assertIn("old_column", output) + with closing(sqlite3.connect(db_path)) as connection: + columns = [row[1] for row in connection.execute("PRAGMA table_info(feed_cache)").fetchall()] + self.assertEqual(columns, ["name", "url", "source"]) + + def test_sqlite_debug_dumper_rebuilds_tables_with_missing_columns(self): + with tempfile.TemporaryDirectory() as temp_dir: + db_path = Path(temp_dir) / "cache.sqlite3" + with closing(sqlite3.connect(db_path)) as connection: + connection.execute( + """ + CREATE TABLE link_check_state ( + url TEXT PRIMARY KEY, + name TEXT NOT NULL, + reachable INTEGER NOT NULL DEFAULT 0 + ) + """ + ) + connection.execute( + "INSERT INTO link_check_state(url, name, reachable) VALUES (?, ?, ?)", + ("https://site.example", "Site", 1), + ) + connection.commit() + + output = SQLiteDebugDumper(db_path).run() + + self.assertIn("缺少当前字段", output) + with closing(sqlite3.connect(db_path)) as connection: + row = connection.execute( + "SELECT url, name, checked_at, crawl_allowed, best_method, rss_unavailable_since FROM link_check_state" + ).fetchone() + self.assertEqual(row, ("https://site.example", "Site", "", 0, "none", "")) + + +if __name__ == "__main__": + unittest.main()