diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index b64c3e60..d4c99905 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,75 +1,16 @@ # html2rss-web AI Agent Instructions -## Overview +This repository uses a centralized documentation structure. All AI agents (including Copilot, Gemini CLI, etc.) must follow the constraints and rules defined in the following files: -- Ruby web app that converts websites into RSS 2.0 feeds. -- Built with **Roda** backend + **Preact** frontend, using the **html2rss** gem (+ `html2rss-configs`). -- **Frontend:** Vite-built Preact UI, served alongside Ruby backend. +## Canonical Documentation -## Documentation website of core dependencies +- **Agent Constraints**: [AGENTS.md](../../AGENTS.md) (Execution rules, verification, and UI principles) +- **Contributor Guide**: [docs/README.md](../../docs/README.md) (Architecture, security, setup, and coding style) +- **Design System**: [docs/design-system.md](../../docs/design-system.md) (Visual and CSS rules) -Search these pages before using them. Find examples, plugins, UI components, and configuration options. +## Quick Reference for Agents -### Roda - -1. https://roda.jeremyevans.net/documentation.html - -### Preact & Vite - -1. https://preactjs.com/guide/v10/getting-started/ -2. https://vite.dev/guide/ - -### html2rss - -1. If available, find source locally in: `../html2rss`. -2. source code on github: https://github.com/html2rss/html2rss - -### Test and Linters - -1. https://docs.rubocop.org/rubocop/cops.html -2. https://docs.rubocop.org/rubocop-rspec/cops_rspec.html -3. https://rspec.info/features/3-13/rspec-expectations/built-in-matchers/ -4. https://www.betterspecs.org/ - -Fix rubocop `RSpec/MultipleExpectations` adding rspec tag `:aggregate_failures`. - -## Core Rules - -- ✅ Organise Roda routes via dedicated modules (e.g. `Html2rss::Web::Routes::*`), keeping the main app class thin. -- ✅ Keep helper modules minimal: define entrypoints with `class << self` and push implementation helpers under `private`; avoid `module_function` unless mirroring existing conventions. -- ✅ Validate all inputs. Pass outbound requests through **SSRF filter**. -- ✅ Add caching headers where appropriate (`Rack::Cache`). -- ✅ Errors: friendly messages for users, detailed logging internally. -- ✅ **Frontend**: Use Preact components in `frontend/src/`. Keep it simple. -- ✅ **CSS**: Use the app-owned frontend styles in `frontend/src/styles/`. -- ✅ **Specs**: RSpec for Ruby, build tests for frontend. -- ✅ When a spec needs to tweak environment variables, wrap the example in `ClimateControl.modify` so state is restored automatically. - -## Don't - -- ❌ Don't use Ruby's URI class or addressable gem directly. Strictly use `Html2rss::Url` only. -- ❌ Don't bypass SSRF filter or weaken CSP. -- ❌ Don't add databases, ORMs, or background jobs. -- ❌ Don't leak stack traces or secrets in responses. -- ❌ Don't reach into private API with `send(...)`; expose what you need at the module level instead. -- ❌ Don't modify `frontend/dist/` - it's generated by build process. -- ❌ NEVER expose the auth token a user provides. - -## Environment - -- `RACK_ENV` – environment -- `AUTO_SOURCE_ENABLED`, `AUTO_SOURCE_USERNAME`, `AUTO_SOURCE_PASSWORD`, `AUTO_SOURCE_ALLOWED_ORIGINS` -- `HEALTH_CHECK_USERNAME`, `HEALTH_CHECK_PASSWORD` -- `SENTRY_DSN` (optional) - -### Verification Steps - -- Run `ruby -c app.rb` to check syntax -- Run `bundle exec rspec` to verify tests -- Check `bundle install` removes unused dependencies - -## Style - -- Add `# frozen_string_literal: true` -- Follow RuboCop style -- YARD doc comments for public methods +- **Environment**: All commands MUST run inside the Dev Container. +- **Verification**: Run `make ready` before any commit. +- **Security**: Follow strict [Security & Safety Rules](../../docs/README.md#security--safety-rules). +- **Style**: Follow [Architectural Constraints](../../docs/README.md#architectural-constraints) (YARD docs, Roda organization). diff --git a/AGENTS.md b/AGENTS.md index b7999f0b..5dae9788 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,84 +1,51 @@ -# Agent Workflow (Dev Container) +# Agent Workflow Constraints -## Start the Dev Container - -```text -docker compose -f .devcontainer/docker-compose.yml up -d -``` - -## Commands (run inside the container) - -```text -docker compose -f .devcontainer/docker-compose.yml exec -T app make setup - -docker compose -f .devcontainer/docker-compose.yml exec -T app make dev - -docker compose -f .devcontainer/docker-compose.yml exec -T app make test - -docker compose -f .devcontainer/docker-compose.yml exec -T app make ready - -docker compose -f .devcontainer/docker-compose.yml exec -T app bundle exec rubocop -F - -docker compose -f .devcontainer/docker-compose.yml exec -T app bundle exec rspec -``` - -Pre-commit gate (required): - -```text -docker compose -f .devcontainer/docker-compose.yml exec -T app make ready -``` - -If you need an interactive shell: - -```text -docker compose -f .devcontainer/docker-compose.yml exec app bash -``` - ---- +This document defines execution constraints for AI agents. For general contributor rules, setup commands, architectural constraints, and security policies, see [docs/README.md](docs/README.md). ## Collaboration Agreement (Agent ↔ User) -## Interview Answers (ID-able) + Expert Recommendations - -**DoD:** `make ready` in Dev Container; if applicable, user completes manual smoke test with agent-provided steps. -**Verification:** Always smoke Dev Container + `make ready`. -**Commits:** Group by logical unit after smoke-tested (feature / improvement / refactor). -**Responses:** Changes → Commands → Results → Next steps, ending with a concise one-line summary. -**KISS vs Refactor:** KISS by default; boy-scout refactors allowed if low-risk and simplifying. -**Ambiguity:** Proceed with safest assumption, then confirm. -**Non-negotiables:** Dev Container only; security first. - -Expert recommendation: keep workflows terminal-first and keyboard-focused (clear commands, no GUI-only steps). +- **DoD:** `make ready` in Dev Container; if applicable, user completes manual smoke test with agent-provided steps. +- **Verification:** Always smoke Dev Container + `make ready`. +- **Commits:** Group by logical unit after smoke-tested (feature / improvement / refactor). +- **Responses:** Changes → Commands → Results → Next steps, ending with a concise one-line summary. +- **KISS vs Refactor:** KISS by default; boy-scout refactors allowed if low-risk and simplifying. +- **Ambiguity:** Proceed with safest assumption, then confirm. +- **Non-negotiables:** Dev Container only; security first. -## Definition of Done - -- Run `make ready` inside the Dev Container. -- If applicable, user completes manual smoke test; agent provides clear instructions. - -## Verification Rules +## Agent-Specific Verification Rules - Always run Dev Container smoke + `make ready` for changes. +- For frontend changes, also verify in `chrome-devtools` MCP at `http://127.0.0.1:4001/` while the Dev Container is running. +- Capture a quick state check for all affected UI states (e.g., guest/member/result) to enforce state parity and avoid duplicate actions. -## Commit Granularity +### Frontend Smoke Checklist -- Group commits by logical units after they have grown and been smoke-tested (feature / improvement / refactor). +- Start the Dev Container and app (`make dev`). +- Open `http://127.0.0.1:4001/` with `chrome-devtools` MCP. +- Validate the primary user path touched by the change. +- Verify all affected states (e.g., guest/member/result) keep the same layout grammar. +- Confirm action uniqueness: one canonical control per outcome in each state. -## Response Format +## UI Execution Principles -- Default: Changes → Commands → Results → Next steps. -- End with a concise one-line summary. +See [docs/design-system.md](docs/design-system.md) for visual rules. -## KISS vs Refactor +- **Task Dominance:** Each UI state should make one user objective obvious and primary. Supporting surfaces and links must yield priority. +- **Copy Minimalism:** Remove text that repeats what the interface already communicates. Prefer action-oriented wording. +- **State Skeleton:** Adjacent UI states should read as the same frame with content changes, not as separate pages. +- **Focus Contract:** Verify browser autofocus and return-focus behavior on initial load and after transitions. +- **Support Compression:** When a user has advanced past setup, reduce the visual weight of support content. -- KISS by default. -- Boy-scout refactors are allowed when they reduce complexity and are low-risk. - -## Ambiguity Handling +## Response Format -- Proceed with the safest assumption, then confirm. +1. **Changes:** Briefly list files/symbols modified. +2. **Commands:** Show the verification commands run. +3. **Results:** Summarize the outcome. +4. **Next steps:** Propose the immediate follow-up. +5. **One-line Summary:** End with a concise summary. ## Non-Negotiables -- Security first. -- YARD docs are strict for public Ruby methods in `app/`: every public method must have a YARD docstring with typed `@param` tags (for all params) and typed `@return`. -- When touching non-public methods, add YARD docs when it improves maintenance or clarifies invariants/edge handling. +- **Security first:** No leaking secrets or insecure patterns. See [Security & Safety Rules](docs/README.md#security--safety-rules). +- **YARD docs:** Strict for public Ruby methods in `app/`. Every public method must have a YARD docstring with typed `@param` and `@return`. See [Architectural Constraints](docs/README.md#architectural-constraints). +- **No host execution:** All commands MUST run inside the Dev Container via `make` or `bundle exec`. diff --git a/Gemfile b/Gemfile index 8f9ae8c3..ac313275 100644 --- a/Gemfile +++ b/Gemfile @@ -16,7 +16,6 @@ gem 'parallel' gem 'rack-cache' gem 'rack-timeout' gem 'roda' -gem 'ssrf_filter' gem 'zeitwerk' gem 'puma', require: false diff --git a/Gemfile.lock b/Gemfile.lock index 7878935d..6eac5921 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,6 +1,6 @@ GIT remote: https://github.com/html2rss/html2rss - revision: e0dca5bf74b17c1e2a0618fc0a4af27c16da1883 + revision: 7672db3109769b059110d8b7bea55cf68ba36a39 branch: master specs: html2rss (0.17.0) @@ -65,7 +65,7 @@ GEM addressable (2.8.9) public_suffix (>= 2.0.2, < 8.0) ast (2.4.3) - async (2.38.0) + async (2.38.1) console (~> 1.29) fiber-annotation io-event (~> 1.11) @@ -167,7 +167,7 @@ GEM io-endpoint (0.17.2) io-event (1.14.4) io-stream (0.11.1) - json (2.19.1) + json (2.19.2) json-schema (6.2.0) addressable (~> 2.8) bigdecimal (>= 3.1, < 5) @@ -185,7 +185,7 @@ GEM mime-types (3.7.0) logger mime-types-data (~> 3.2025, >= 3.2025.0507) - mime-types-data (3.2026.0303) + mime-types-data (3.2026.0317) minitest (6.0.2) drb (~> 2.0) prism (~> 1.5) @@ -220,7 +220,7 @@ GEM protocol-http2 (0.24.0) protocol-hpack (~> 1.4) protocol-http (~> 0.47) - protocol-rack (0.21.1) + protocol-rack (0.22.0) io-stream (>= 0.10) protocol-http (~> 0.58) rack (>= 1.0) @@ -333,7 +333,6 @@ GEM simplecov_json_formatter (~> 0.1) simplecov-html (0.13.2) simplecov_json_formatter (0.1.4) - ssrf_filter (1.3.0) stackprof (0.2.28) thor (1.5.0) traces (0.18.2) @@ -386,7 +385,6 @@ DEPENDENCIES ruby-lsp sentry-ruby simplecov - ssrf_filter stackprof vcr webmock @@ -399,7 +397,7 @@ CHECKSUMS activesupport (8.1.2) sha256=88842578ccd0d40f658289b0e8c842acfe9af751afee2e0744a7873f50b6fdae addressable (2.8.9) sha256=cc154fcbe689711808a43601dee7b980238ce54368d23e127421753e46895485 ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383 - async (2.38.0) sha256=f95d00da2eb72e2c5340a6d78c321ec70cec65cbeceb0dc2cb2a32ff17a0f4cf + async (2.38.1) sha256=72ba6b7de04d852355458bfe891221226bb7d29f055f5cb043ae3345497f8cec async-http (0.94.2) sha256=c5ca94b337976578904a373833abe5b8dfb466a2946af75c4ae38c409c5c78b2 async-pool (0.11.2) sha256=0a43a17b02b04d9c451b7d12fafa9a50e55dc6dd00d4369aca00433f16a7e3ed async-websocket (0.30.0) sha256=55739954528ad8f87f7792d0452e1268d1ef2aa5b3719f79400a05a1a6202cdf @@ -441,7 +439,7 @@ CHECKSUMS io-endpoint (0.17.2) sha256=3feaf766c116b35839c11fac68b6aaadc47887bb488902a57bf8e1d288fb3338 io-event (1.14.4) sha256=455a9e4fb4613d12867b90461c297af6993b400a521bf62046f83b27f9c6aa3d io-stream (0.11.1) sha256=fa5f551fcff99581c1757b9d1cee2c37b124f07d2ca4f40b756a05ab9bd21b87 - json (2.19.1) sha256=dd94fdc59e48bff85913829a32350b3148156bc4fd2a95a2568a78b11344082d + json (2.19.2) sha256=e7e1bd318b2c37c4ceee2444841c86539bc462e81f40d134cf97826cb14e83cf json-schema (6.2.0) sha256=e8bff46ed845a22c1ab2bd0d7eccf831c01fe23bb3920caa4c74db4306813666 kramdown (2.5.2) sha256=1ba542204c66b6f9111ff00dcc26075b95b220b07f2905d8261740c82f7f02fa language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc @@ -451,7 +449,7 @@ CHECKSUMS mcp (0.8.0) sha256=ae8bd146bb8e168852866fd26f805f52744f6326afb3211e073f78a95e0c34fb metrics (0.15.0) sha256=61ded5bac95118e995b1bc9ed4a5f19bc9814928a312a85b200abbdac9039072 mime-types (3.7.0) sha256=dcebf61c246f08e15a4de34e386ebe8233791e868564a470c3fe77c00eed5e56 - mime-types-data (3.2026.0303) sha256=164af1de5824c5195d4b503b0a62062383b65c08671c792412450cd22d3bc224 + mime-types-data (3.2026.0317) sha256=77f078a4d8631d52b842ba77099734b06eddb7ad339d792e746d2272b67e511b minitest (6.0.2) sha256=db6e57956f6ecc6134683b4c87467d6dd792323c7f0eea7b93f66bd284adbc3d net-http (0.9.1) sha256=25ba0b67c63e89df626ed8fac771d0ad24ad151a858af2cc8e6a716ca4336996 nio4r (2.7.5) sha256=6c90168e48fb5f8e768419c93abb94ba2b892a1d0602cb06eef16d8b7df1dca1 @@ -470,7 +468,7 @@ CHECKSUMS protocol-http (0.60.0) sha256=ca1354947676d663b6f23c49654aee464288774e7867c4a6e406fecce9691cec protocol-http1 (0.37.0) sha256=5bdd739e28792b341134596f6f5ab21a9d4b395f67bae69e153743eb0e69d123 protocol-http2 (0.24.0) sha256=65327a019b7e36d2774e94050bf57a43bb60212775d2fcf02ae1d2ed4f01ef28 - protocol-rack (0.21.1) sha256=366ff16efbf4c2f8d2e3fad4e992effa2357610f70effbccfa2767d26fedc577 + protocol-rack (0.22.0) sha256=b7c49c0b597ca2c6d20f8bcd746c4415a1b750eacfbe64f828e780c978a4293d protocol-url (0.4.0) sha256=64d4c03b6b51ad815ac6fdaf77a1d91e5baf9220d26becb846c5459dacdea9e1 protocol-websocket (0.20.2) sha256=c41d93c35fba5dae85375c597f76975f3dbd75d8c5b2f21b33dab4dc22a5a511 public_suffix (7.0.5) sha256=1a8bb08f1bbea19228d3bed6e5ed908d1cb4f7c2726d18bd9cadf60bc676f623 @@ -513,7 +511,6 @@ CHECKSUMS simplecov (0.22.0) sha256=fe2622c7834ff23b98066bb0a854284b2729a569ac659f82621fc22ef36213a5 simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246 simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428 - ssrf_filter (1.3.0) sha256=66882d7de7d09c019098d6d7372412950ae184ebbc7c51478002058307aba6f2 stackprof (0.2.28) sha256=4ec2ace02f386012b40ca20ef80c030ad711831f59511da12e83b34efb0f9a04 thor (1.5.0) sha256=e3a9e55fe857e44859ce104a84675ab6e8cd59c650a49106a05f55f136425e73 traces (0.18.2) sha256=80f1649cb4daace1d7174b81f3b3b7427af0b93047759ba349960cb8f315e214 @@ -530,4 +527,4 @@ CHECKSUMS zlib (3.2.3) sha256=5bd316698b32f31a64ab910a8b6c282442ca1626a81bbd6a1674e8522e319c20 BUNDLED WITH - 4.0.8 + 4.0.6 diff --git a/README.md b/README.md index 9bf7d39b..85ec5130 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ html2rss-web converts arbitrary websites into RSS 2.0 feeds with a slim Ruby bac ## Links - Docs & feed directory: https://html2rss.github.io +- Contributor Guide: [docs/README.md](docs/README.md) - Discussions: https://github.com/orgs/html2rss/discussions - Sponsor: https://github.com/sponsors/gildesmarais @@ -15,29 +16,35 @@ html2rss-web converts arbitrary websites into RSS 2.0 feeds with a slim Ruby bac - Responsive Preact interface for demo, sign-in, conversion, and result flows. - Automatic source discovery with token-scoped permissions. - Signed public feed URLs that work in standard RSS readers. -- Built-in SSRF defences, input validation, and HMAC-protected tokens. +- Built-in URL validation, scoped feed access controls, and HMAC-protected tokens. -## Architecture +## Architecture Overview - **Backend:** Ruby + Roda, backed by the `html2rss` gem for extraction. - **Frontend:** Preact app built with Vite into `frontend/dist` and served at `/`. -- **Distribution:** Docker Compose by default; other deployments require manual wiring. -- [Project notes](docs/README.md) +- **Distribution:** Docker Compose by default. -## REST API Snapshot +For detailed architecture and internal rules, see [docs/README.md](docs/README.md). + +## Trial Run (Docker Pull And Run) + +The published image already includes a sample `config/feeds.yml`, so you can try the app without creating or mounting one first. ```bash -# List available strategies -curl -H "Authorization: Bearer " \ - "https://your-domain.com/api/v1/strategies" - -# Create a feed and capture the signed public URL -curl -X POST "https://your-domain.com/api/v1/feeds" \ - -H "Authorization: Bearer " \ - -H "Content-Type: application/json" \ - -d '{"url":"https://example.com","name":"Example Feed"}' +docker run --rm \ + -p 4000:4000 \ + -e RACK_ENV=production \ + -e HTML2RSS_SECRET_KEY=$(openssl rand -hex 32) \ + html2rss/web ``` +Then open: + +- `http://localhost:4000/` for the web UI +- `http://localhost:4000/microsoft.com/azure-products.rss` for a built-in Azure updates feed + +This trial run is intentionally minimal. Use Docker Compose for Browserless, auto-updates, or local feed overrides. + ## Deploy (Docker Compose) 1. Generate a key: `openssl rand -hex 32`. @@ -46,73 +53,12 @@ curl -X POST "https://your-domain.com/api/v1/feeds" \ UI + API run on `http://localhost:4000`. The app exits if the secret key is missing. -## Development (Dev Container) +## Development Use the repository's [Dev Container](.devcontainer/README.md) for all local development and tests. Running the app directly on the host is not supported. -Quick start inside the Dev Container: - -``` -make setup -make dev -make test -make ready -make yard-verify-public-docs -bundle exec rubocop -F -bundle exec rspec -make openapi -``` - -Dev URLs: Ruby app at `http://localhost:4000`, frontend dev server at `http://localhost:4001`. - -Backend code under the `Html2rss::Web` namespace now lives under `app/web/**`, so Zeitwerk can mirror constant paths directly instead of relying on directory-specific namespace wiring. -`make ready` also runs `rake zeitwerk:verify`, which eager-loads the app and fails on loader drift early. -For contributors and AI agents changing backend structure, follow the rules in [docs/README.md](docs/README.md). - -## Make Targets - -| Command | Purpose | -| ------------------------------ | ---------------------------------------------------------- | -| `make help` | List available shortcuts. | -| `make setup` | Install Ruby and Node dependencies. | -| `make dev` | Run Ruby (port 4000) and frontend (port 4001) dev servers. | -| `make dev-ruby` | Start only the Ruby server. | -| `make dev-frontend` | Start only the frontend dev server (port 4001). | -| `make test` | Run Ruby and frontend test suites. | -| `make test-ruby` | Run Ruby specs. | -| `make test-frontend` | Run frontend unit and contract tests. | -| `make lint` | Run all linters. | -| `make lintfix` | Auto-fix lint warnings where possible. | -| `make yard-verify-public-docs` | Enforce typed YARD docs for public methods in `app/`. | -| `make openapi` | Regenerate `public/openapi.yaml` from request specs. | -| `make openapi-verify` | Regenerate + fail if OpenAPI file is stale. | -| `make clean` | Remove build artefacts. | - -## OpenAPI Contract - -The OpenAPI file is generated from Ruby request specs only. - -- Regenerate: `make openapi` -- Verify drift (CI behavior): `make openapi-verify` - -## Frontend npm Scripts (inside Dev Container) - -| Command | Purpose | -| ----------------------- | -------------------------------------------- | -| `npm run dev` | Vite dev server with hot reload (port 4001). | -| `npm run build` | Build static assets into `frontend/dist/`. | -| `npm run test:run` | Unit tests (Vitest). | -| `npm run test:contract` | Contract tests with MSW. | - -## Testing Strategy - -| Layer | Tooling | Focus | -| ----------------- | ------------------------ | ---------------------------------------------------- | -| Ruby API | RSpec + Rack::Test | Feed creation, retrieval, auth paths. | -| Frontend unit | Vitest + Testing Library | Component rendering and hooks with mocked fetch. | -| Frontend contract | Vitest + MSW | End-to-end fetch flows against mocked API responses. | -| Docker smoke | RSpec (`:docker`) | Net::HTTP probes against the containerised service. | +See the [Contributor Guide](docs/README.md) for setup commands, testing strategy, and backend structure rules. ## Contributing diff --git a/app/web/api/v1/root_metadata.rb b/app/web/api/v1/root_metadata.rb index 2d936b5b..dfdd503c 100644 --- a/app/web/api/v1/root_metadata.rb +++ b/app/web/api/v1/root_metadata.rb @@ -7,6 +7,24 @@ module V1 ## # Builds the public metadata payload for the API root endpoint. module RootMetadata + FEATURED_FEEDS = [ + { + path: '/microsoft.com/azure-products.rss', + title: 'Azure product updates', + description: 'Follow Microsoft Azure product announcements from your own instance.' + }, + { + path: '/phys.org/weekly.rss', + title: 'Top science news of the week', + description: 'Try a high-signal feed with stable weekly headlines from the built-in config set.' + }, + { + path: '/softwareleadweekly.com/issues.rss', + title: 'Software Lead Weekly issues', + description: 'Follow a long-running newsletter archive from the embedded config catalog.' + } + ].freeze + class << self # @param router [Roda::RodaRequest] # @return [Hash{Symbol=>Object}] @@ -30,7 +48,8 @@ def instance_payload(_router) feed_creation: { enabled: AutoSource.enabled?, access_token_required: AutoSource.enabled? - } + }, + featured_feeds: FEATURED_FEEDS } end end diff --git a/app/web/api/v1/strategies.rb b/app/web/api/v1/strategies.rb index 12582b08..d34f409d 100644 --- a/app/web/api/v1/strategies.rb +++ b/app/web/api/v1/strategies.rb @@ -33,8 +33,8 @@ def index(_request) def display_name_for(name) case name.to_s - when 'ssrf_filter' then 'Standard (recommended)' - when 'browserless' then 'JavaScript pages' + when 'faraday' then 'Default' + when 'browserless' then 'JavaScript pages (recommended)' else name.to_s.split('_').map(&:capitalize).join(' ') end end diff --git a/app/web/boot/setup.rb b/app/web/boot/setup.rb index 2c22f364..7d5344f0 100644 --- a/app/web/boot/setup.rb +++ b/app/web/boot/setup.rb @@ -7,12 +7,11 @@ module Boot # Applies boot-time runtime configuration outside the Roda class body. module Setup class << self - # Validates environment configuration and wires the request service. + # Validates environment configuration. # # @return [void] def call! validate_environment! - configure_request_service! end private @@ -23,13 +22,6 @@ def validate_environment! EnvironmentValidator.validate_production_security! Flags.validate! end - - # @return [void] - def configure_request_service! - Html2rss::RequestService.register_strategy(:ssrf_filter, SsrfFilterStrategy) - Html2rss::RequestService.default_strategy_name = :ssrf_filter - Html2rss::RequestService.unregister_strategy(:faraday) - end end end end diff --git a/app/web/config/local_config.rb b/app/web/config/local_config.rb index 4b937d6c..4e7d0cea 100644 --- a/app/web/config/local_config.rb +++ b/app/web/config/local_config.rb @@ -1,6 +1,11 @@ # frozen_string_literal: true require 'yaml' +begin + require 'html2rss/configs' +rescue LoadError + nil +end module Html2rss module Web @@ -17,6 +22,7 @@ class NotFound < RuntimeError; end # raised when the local config shape is invalid class InvalidConfig < RuntimeError; end FEED_EXTENSION_PATTERN = /\.(json|rss|xml)\z/ + EMBEDDED_FEED_NAME_PATTERN = %r{\A[^/]+/.+\z} # Path to local feed configuration file. CONFIG_FILE = 'config/feeds.yml' @@ -27,10 +33,8 @@ class << self # @return [Hash] def find(name) normalized_name = normalize_name(name) - config = snapshot.feeds.fetch(normalized_name.to_sym) do - raise NotFound, "Did not find local feed config at '#{normalized_name}'" - end - config_hash = deep_dup(config.raw) + config_hash = local_feed_config(normalized_name) || embedded_feed_config(normalized_name) + raise NotFound, "Did not find local feed config at '#{normalized_name}'" unless config_hash apply_global_defaults(config_hash) end @@ -76,6 +80,26 @@ def reload!(reason: 'manual') private + # @param normalized_name [String] + # @return [Hash{Symbol=>Object}, nil] + def local_feed_config(normalized_name) + config = snapshot.feeds[normalized_name.to_sym] + return nil unless config + + deep_dup(config.raw) + end + + # @param normalized_name [String] + # @return [Hash{Symbol=>Object}, nil] + def embedded_feed_config(normalized_name) + return nil unless defined?(Html2rss::Configs) + return nil unless normalized_name.match?(EMBEDDED_FEED_NAME_PATTERN) + + deep_dup(Html2rss::Configs.find_by_name(normalized_name)) + rescue Html2rss::Configs::ConfigNotFound + nil + end + # Applies global defaults only when feed-level keys are absent. # # @param config [Hash{Symbol=>Object}] @@ -90,9 +114,9 @@ def apply_global_defaults(config) end # @param name [String, Symbol, #to_s] - # @return [String] basename without extension for feed lookup. + # @return [String] path without feed extension for feed lookup. def normalize_name(name) - File.basename(name.to_s).sub(FEED_EXTENSION_PATTERN, '') + name.to_s.delete_prefix('/').sub(FEED_EXTENSION_PATTERN, '') end # Deep-duplicates nested config structures to avoid mutating shared data. diff --git a/app/web/domain/auto_source.rb b/app/web/domain/auto_source.rb index 4b1d09c7..cafb0429 100644 --- a/app/web/domain/auto_source.rb +++ b/app/web/domain/auto_source.rb @@ -21,7 +21,7 @@ def enabled? # @param token_data [Hash{Symbol=>Object}] authenticated account data. # @param strategy [String] # @return [Html2rss::Web::Api::V1::FeedMetadata::Metadata, nil] - def create_stable_feed(name, url, token_data, strategy = 'ssrf_filter') + def create_stable_feed(name, url, token_data, strategy = Html2rss::RequestService.default_strategy_name.to_s) return nil unless token_data && FeedAccess.url_allowed_for_username?(token_data[:username], url) feed_token = Auth.generate_feed_token(token_data[:username], url, strategy: strategy) diff --git a/app/web/feeds/source_resolver.rb b/app/web/feeds/source_resolver.rb index 0bb33a6e..46310da4 100644 --- a/app/web/feeds/source_resolver.rb +++ b/app/web/feeds/source_resolver.rb @@ -36,6 +36,8 @@ def resolve_static(feed_request) generator_input: generator_input, ttl_seconds: CacheTtl.seconds_from_minutes(generator_input.dig(:channel, :ttl)) ) + rescue LocalConfig::NotFound + raise Html2rss::Web::NotFoundError, "Feed '#{feed_request.feed_name}' is not available on this instance" end # @param feed_request [Html2rss::Web::Feeds::Contracts::Request] @@ -69,7 +71,7 @@ def static_cache_identity(feed_name, params) def static_generator_input(config, params) generator_input = config.dup generator_input[:params] = merged_static_params(config, params) - generator_input[:strategy] ||= Html2rss::RequestService.default_strategy_name + generator_input[:strategy] ||= Html2rss::RequestService.default_strategy_name.to_sym generator_input end diff --git a/app/web/security/ssrf_filter_strategy.rb b/app/web/security/ssrf_filter_strategy.rb deleted file mode 100644 index 2ad3f76f..00000000 --- a/app/web/security/ssrf_filter_strategy.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -require 'ssrf_filter' -require 'html2rss' -module Html2rss - module Web - ## - # Strategy to fetch a URL using the SSRF filter. - class SsrfFilterStrategy < Html2rss::RequestService::Strategy - # Executes a URL fetch through `ssrf_filter` and adapts response shape. - # - # @return [Html2rss::RequestService::Response] - def execute - headers = LocalConfig.global.fetch(:headers, {}).merge( - ctx.headers.transform_keys(&:to_sym) - ) - response = SsrfFilter.get(ctx.url, headers:) - - Html2rss::RequestService::Response.new(body: response.body, - url: ctx.url, - headers: response.to_hash.transform_values(&:first)) - end - end - end -end diff --git a/docker-compose.yml b/docker-compose.yml index c901f2c8..38273eaa 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,11 +7,9 @@ services: restart: unless-stopped ports: - "127.0.0.1:4000:4000" - volumes: - - type: bind - source: ./config/feeds.yml - target: /app/config/feeds.yml - read_only: true + env_file: + - path: .env + required: false environment: RACK_ENV: production PORT: 4000 @@ -19,6 +17,13 @@ services: HEALTH_CHECK_TOKEN: ${HEALTH_CHECK_TOKEN:?set HEALTH_CHECK_TOKEN} BROWSERLESS_IO_WEBSOCKET_URL: ws://browserless:4002 BROWSERLESS_IO_API_TOKEN: ${BROWSERLESS_IO_API_TOKEN:?set BROWSERLESS_IO_API_TOKEN} + # Trial runs use the image's bundled config/feeds.yml. + # Uncomment the block below when you want to replace it with your own file. + # volumes: + # - type: bind + # source: ./config/feeds.yml + # target: /app/config/feeds.yml + # read_only: true watchtower: image: containrrr/watchtower diff --git a/docs/README.md b/docs/README.md index c98c8bae..be2aef63 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,44 +1,96 @@ -# Project Notes +# Project Documentation & Contributor Guide -This is the only hand-written project document in `docs/`. +Welcome! This is the canonical source of truth for contributing to `html2rss-web`. -Keep this file short, current, and operational. Do not add planning docs, migration diaries, redesign notes, or parallel architecture narratives back into this directory. +## Docs Index -The only generated artifact intentionally exposed by the app is [`public/openapi.yaml`](/Users/gil/versioned/html2rss/html2rss-web/public/openapi.yaml). +- **Start here for contributors**: This document. +- **Architecture & Request Lifecycle**: [docs/architecture.md](architecture.md) +- **UI/Design rules**: [docs/design-system.md](design-system.md) +- **Agent execution constraints**: [AGENTS.md](../AGENTS.md) +- **Generated contract artifacts**: `public/openapi.yaml` +- **Public-facing intro**: [README.md](../README.md) -## System Snapshot +--- -- Backend: Ruby + Roda under the `Html2rss::Web` namespace. -- Frontend: Preact + Vite, built into `frontend/dist` and served at `/`. -- Feed extraction: delegated to the `html2rss` gem. -- Distribution: Docker Compose / Dev Container first. +## System Snapshot & Architecture -## Source Of Truth +`html2rss-web` converts arbitrary websites into RSS 2.0 feeds. -- Runtime behavior: application code plus tests. -- HTTP contract: request specs plus generated OpenAPI. -- This file: contributor conventions and current project rules only. +- **Backend**: Ruby + Roda under the `Html2rss::Web` namespace. +- **Frontend**: Preact + Vite, built into `frontend/dist` and served at `/`. +- **Feed extraction**: Delegated to the `html2rss` gem. +- **Distribution**: Docker Compose / Dev Container first. -## Verification +### Source Of Truth -Primary local gate: +- **Runtime behavior**: Application code plus tests. +- **HTTP contract**: Request specs plus generated OpenAPI. +- **This file**: Contributor conventions and current project rules. -```text -docker compose -f .devcontainer/docker-compose.yml exec -T app make ready -``` +--- + +## Development Setup (Dev Container) + +Use the repository's [Dev Container](.devcontainer/README.md) for all local development and tests. +Running the app directly on the host is not supported. + +### Common Commands (Inside Dev Container) + +| Command | Purpose | +| ------------------------------ | ---------------------------------------------------------- | +| `make setup` | Install Ruby and Node dependencies. | +| `make dev` | Run Ruby (port 4000) and frontend (port 4001) dev servers. | +| `make ready` | Full pre-flight check (Lint + Test + OpenAPI + Zeitwerk). | +| `make test` | Run Ruby and frontend test suites. | +| `make lint` | Run all linters. | +| `make yard-verify-public-docs` | Enforce typed YARD docs for public methods in `app/`. | +| `make openapi` | Regenerate `public/openapi.yaml` from request specs. | + +### Frontend npm Scripts + +| Command | Purpose | +| ----------------------- | -------------------------------------------- | +| `npm run dev` | Vite dev server with hot reload (port 4001). | +| `npm run build` | Build static assets into `frontend/dist/`. | +| `npm run test:run` | Unit tests (Vitest). | +| `npm run test:contract` | Contract tests with MSW. | + +--- + +## Contract-Driven Development Loop -Useful commands: +To change or add API endpoints, follow this sequence: -```text -docker compose -f .devcontainer/docker-compose.yml exec -T app make setup -docker compose -f .devcontainer/docker-compose.yml exec -T app make dev -docker compose -f .devcontainer/docker-compose.yml exec -T app bundle exec rspec -docker compose -f .devcontainer/docker-compose.yml exec -T app make openapi -docker compose -f .devcontainer/docker-compose.yml exec -T app make openapi-verify -docker compose -f .devcontainer/docker-compose.yml exec -T app make openapi-lint +1. **Ruby Request Spec**: Define the new behavior or endpoint in `spec/html2rss/web/app_integration_spec.rb` or a dedicated request spec. +2. **OpenAPI Generation**: Run `make openapi` inside the Dev Container to regenerate `public/openapi.yaml` from the spec metadata. +3. **Verify Contract**: Run `make openapi-verify` and `make openapi-lint` to ensure the generated file matches the specs and is valid. +4. **Frontend Client**: The frontend generated client in `frontend/src/api/generated` is updated by the build process. + +Always verify the contract before committing API changes. + +--- + +## Verification & Testing Strategy + +### Local Verification Gate + +Always run this before pushing or committing: + +```bash +make ready ``` -Frontend verification lives at `http://127.0.0.1:4001/` while the dev container is running. +### Testing Layers + +| Layer | Tooling | Focus | +| ----------------- | ------------------------ | ---------------------------------------------------- | +| Ruby API | RSpec + Rack::Test | Feed creation, retrieval, auth paths. | +| Frontend unit | Vitest + Testing Library | Component rendering and hooks with mocked fetch. | +| Frontend contract | Vitest + MSW | End-to-end fetch flows against mocked API responses. | +| Docker smoke | RSpec (`:docker`) | Net::HTTP probes against the containerised service. | + +--- ## Backend Structure Rules @@ -48,11 +100,13 @@ Frontend verification lives at `http://127.0.0.1:4001/` while the dev container - Keep route composition in `app/web/routes/**`. - Keep `/api/v1` contract-specific code in `app/web/api/**`. - Keep feed fetching, caching, and orchestration in `app/web/feeds/**`. -- Keep auth, token handling, SSRF strategy, and security logging in `app/web/security/**`. +- Keep auth, token handling, URL validation, and security logging in `app/web/security/**`. - Keep request-scoped context in `app/web/request/**`. - Keep boot/runtime setup in `app/web/boot/**`. - Do not create generic buckets such as `services`, `helpers`, `utils`, or `concerns`. +--- + ## API Contract Rules - `public/openapi.yaml` is generated output, not hand-edited design prose. @@ -62,15 +116,59 @@ Frontend verification lives at `http://127.0.0.1:4001/` while the dev container - Quality must fail with `make openapi-lint`. - Frontend generated client code under `frontend/src/api/generated` is machine-generated only. -## Runtime Flags +--- + +## Core Dependencies + +Search these pages for examples, plugins, and configuration options: + +- **Roda**: [roda.jeremyevans.net](https://roda.jeremyevans.net/documentation.html) +- **Preact & Vite**: [preactjs.com](https://preactjs.com/guide/v10/getting-started/) and [vite.dev](https://vite.dev/guide/) +- **html2rss**: [github.com/html2rss/html2rss](https://github.com/html2rss/html2rss) +- **Testing (Ruby)**: [rspec.info](https://rspec.info/features/3-13/rspec-expectations/built-in-matchers/), [rubocop.org](https://docs.rubocop.org/rubocop/cops.html), [betterspecs.org](https://www.betterspecs.org/) + +--- + +## Security & Safety Rules + +- **URL Handling**: Never use Ruby's `URI` class or `addressable` gem directly. Use `Html2rss::Url` for all URL logic. +- **SSRF Protection**: Delegated to the `html2rss` gem's built-in security features. Do not bypass these protections or weaken CSP. +- **Secrets**: Never leak stack traces, auth tokens, or internal secrets in HTTP responses. +- **Data Protection**: Auth tokens provided by users must never be exposed or logged. + +--- -Managed flags: +## Architectural Constraints + +- **No Persistence**: Do not add databases, ORMs, or background job systems. +- **Backend Style**: + - Keep the main `app.rb` thin; organize routes in `Html2rss::Web::Routes::*`. + - For helpers, use `class << self` and `private` methods. Avoid `module_function`. + - Use YARD doc comments for all public methods in `app/`. + - Add `# frozen_string_literal: true` to all Ruby files. + - Do not use `send(...)` to reach into private APIs; expose what is needed at the module level. +- **Frontend Style**: + - Follow visual and CSS rules in [design-system.md](design-system.md). + - Use Preact components in `frontend/src/`. + - Use shared styles in `public/shared-ui.css` or app-specific styles in `frontend/src/styles/`. + - Do not modify `frontend/dist/` directly. +- **Testing**: + - Use `ClimateControl.modify` for tests that change environment variables. + - Use `:aggregate_failures` to resolve `RSpec/MultipleExpectations` warnings. + +--- + +## Environment & Runtime Flags + +Managed flags and environment keys: | Name | Env key | Type | Default | | --------------------------------- | --------------------------------- | -------------- | ---------------------------------------- | | `auto_source_enabled` | `AUTO_SOURCE_ENABLED` | boolean | `true` in development/test, else `false` | | `async_feed_refresh_enabled` | `ASYNC_FEED_REFRESH_ENABLED` | boolean | `false` | | `async_feed_refresh_stale_factor` | `ASYNC_FEED_REFRESH_STALE_FACTOR` | integer `>= 1` | `3` | +| `health_check_token` | `HEALTH_CHECK_TOKEN` | string | `nil` | +| `sentry_dsn` | `SENTRY_DSN` | string | `nil` | Rules: @@ -78,31 +176,15 @@ Rules: - Unknown managed feature-style env keys must fail fast at boot. - Add or change flags in code, tests, and this table together. -## Observability Contract - -Canonical event fields: +--- -- `event_name` -- `schema_version` -- `request_id` -- `route_group` -- `actor` -- `outcome` - -Optional request context fields: +## Observability Contract -- `path` -- `method` -- `strategy` -- `started_at` -- `details` +Canonical event fields: `event_name`, `schema_version`, `request_id`, `route_group`, `actor`, `outcome`. -Critical-path event families: +Critical-path event families: auth, feed create, feed render, request errors. -- auth -- feed create -- feed render -- request errors +--- ## Documentation Policy diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 00000000..4315c153 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,48 @@ +# Architecture & Request Lifecycle + +This document provides a mental model of how `html2rss-web` processes requests. + +## High-Level Data Flow + +```text +[ User / RSS Reader ] + | + v +[ Roda App (app/web/routes) ] <--- [ Auth / Security (app/web/security) ] + | + v +[ Feeds Service (app/web/feeds) ] <--- [ Cache (app/web/feeds/cache) ] + | + v +[ html2rss Gem ] <--- [ Request Strategies (Faraday / Browserless) ] + | + v +[ Target Website ] +``` + +## Request Lifecycle + +### 1. Routing & Auth +Requests enter via `app.rb` and are dispatched to `app/web/routes/`. +- **API v1**: Authenticated via `app/web/security/auth.rb`. +- **Public Feeds**: Validated via HMAC tokens in `app/web/security/feed_token.rb`. + +### 2. Resolution +The `Html2rss::Web::Feeds::SourceResolver` determines where the feed configuration comes from: +- **Static**: Pre-defined in `config/feeds.yml`. +- **Dynamic**: Generated on-the-fly via the `/api/v1/feeds` endpoint (AutoSource). + +### 3. Fetching & Rendering +The `Html2rss::Web::Feeds::Service` orchestrates the extraction: +1. Checks the `Html2rss::Web::Feeds::Cache`. +2. If stale/missing, calls the `html2rss` gem with the resolved strategy. +3. Renders the output using `RSSRenderer` (XML) or `JSONRenderer`. + +## Extension Points + +### Adding a Request Strategy +Strategies are defined by the `html2rss` gem but can be configured here. +- **Faraday**: Default HTTP client for static HTML. +- **Browserless**: Used for JavaScript-heavy websites. + +To add or configure strategies, see `app/web/feeds/source_resolver.rb` and the `html2rss` gem documentation. diff --git a/docs/design-system.md b/docs/design-system.md new file mode 100644 index 00000000..d9b243a3 --- /dev/null +++ b/docs/design-system.md @@ -0,0 +1,235 @@ +# Design System Manifest + +This document is the visual source of truth for `html2rss-web`. + +If you change the UI, read this first. If a design decision is not reflected here, update this file in the same change. The goal is to prevent slow visual drift between the app shell, result flow, and RSS/XSL presentation. + +## Core Rule + +There is one shared design language and one shared primitive layer. + +- Shared primitives live in [public/shared-ui.css](../public/shared-ui.css). +- App-specific composition lives in [frontend/src/styles/main.css](../frontend/src/styles/main.css). +- Feed-specific composition lives in [public/rss.xsl](../public/rss.xsl). + +Do not duplicate tokens, base canvas rules, card shells, rails, stack primitives, or brand-lockup styling in `main.css` or `rss.xsl`. If app and feed both need it, it belongs in `shared-ui.css`. + +## Visual Thesis + +The UI is not a generic SaaS dashboard. It should read as: + +- stark +- editorial +- quiet but deliberate +- dark, with restrained light accents +- compact, not crowded + +The experience should feel like one product across: + +- `/` +- the token gate +- the result page +- `/example.rss` and all XSL-rendered feeds + +If a page looks like it came from a different product, the change is wrong even if the CSS is technically valid. + +## Non-Negotiable Surface Rules + +- Background must use the same dark canvas and top-light treatment defined in `shared-ui.css`. +- Shared cards must use the same border, radius, and surface treatment. +- Serif display typography is reserved for major titles and the wordmark. +- Sans UI typography is the default for controls, supporting copy, and metadata. +- Mono is reserved for URLs, tokens, and machine-like values. +- Eyebrow text is uppercase, compact, and low-noise. +- Spacing should come from the token scale only. + +Do not introduce: + +- ad hoc colors +- page-local shadows that fight the shared card elevation +- one-off radii +- extra spacing scales +- component-specific typography systems + +## Architecture + +The CSS is intentionally split by responsibility, not by page count. + +### 1. Shared Primitive Layer + +Owned by [public/shared-ui.css](../public/shared-ui.css). + +This file owns: + +- tokens +- global box sizing and canvas behavior +- global typography baseline +- link behavior +- rails +- stack primitives +- card primitives +- eyebrow primitive +- brand lockup + +This file should stay small, boring, and reusable. + +### 2. App Composition Layer + +Owned by [frontend/src/styles/main.css](../frontend/src/styles/main.css). + +This file owns: + +- page-shell composition +- workspace layout +- form behavior +- dominant-field behavior +- button behavior +- notice state behavior +- token-gate composition +- result-page composition +- utility strip behavior + +This file should not redefine shared primitives. + +### 3. Feed Composition Layer + +Owned by [public/rss.xsl](../public/rss.xsl). + +This file owns only feed-page specifics: + +- feed hero composition +- feed metadata rows +- feed list/card content styling +- feed empty/error presentation + +This file should compose shared classes rather than restyle them. + +## Approved Primitive API + +Prefer composing these primitives before inventing new classes: + +- `layout-shell` +- `layout-rail-reading` +- `layout-rail-copy` +- `layout-stack` +- `layout-stack--tight` +- `ui-card` +- `ui-card--padded` +- `ui-card--roomy` +- `ui-card--notice` +- `ui-eyebrow` +- `brand-lockup` +- `input` +- `input--lg` +- `input--minimal` +- `input--mono` +- `btn` +- `btn--primary` +- `btn--ghost` +- `btn--quiet` +- `btn--linkish` + +Semantic state should prefer attributes over extra visual variants: + +- `data-tone="error"` +- `data-tone="success"` +- `data-state="loading"` + +This is deliberate. We want a small CSS API with composable primitives, not endless component-local variants. + +## Variant Discipline + +Before adding a new class or modifier, ask: + +1. Can this be expressed by composing existing primitives? +2. Is this a reusable primitive or only page-local composition? +3. Is this visual difference actually perceptible and meaningful? +4. Does this belong to structure, modifier, or semantic state? + +Default answers: + +- New primitive: rare +- New modifier: suspicious +- New component-specific variant: usually wrong +- New semantic attribute: acceptable when behavior or tone truly changes + +Avoid returning to patterns like: + +- `input--hero` +- `input--select` +- `status-card` +- multiple near-identical surface tokens + +Those create variant creep. + +## Color And Surface Rules + +Use only the shared tokens unless there is a strong system-level reason to extend them. + +Key expectations: + +- `--surface-base` is the default card plane. +- `--surface-elevated` is for stronger inputs and interactive surfaces. +- success and error backgrounds are semantic overlays, not new card systems. +- border strength should increase only for focus or meaningful emphasis. + +If you think you need another surface token, the burden of proof is high. + +## Typography Rules + +- `--font-family-display` is for primary titles and the wordmark only. +- `--font-family-ui` is the default everywhere else. +- `--font-family-mono` is for feed URLs, tokens, and similarly mechanical strings. +- `ui-eyebrow` is the preferred pattern for small uppercase metadata labels. + +Do not create alternate display systems per page. + +## Layout Rules + +The layout language is narrow on purpose. + +- Use rails to control readable width. +- Use stack primitives for vertical rhythm. +- Keep shells centered and calm. +- Prefer composition over custom grid declarations. + +If you add `display: grid`, be able to explain why an existing stack or rail primitive was insufficient. + +## Agent Checklist + +When changing UI, an agent must verify: + +1. Does the change reuse `shared-ui.css` where appropriate? +2. Did I avoid duplicating a shared primitive in `main.css` or `rss.xsl`? +3. Does the app still match the RSS/XSL rendering in overall tone and framing? +4. Did I avoid inventing a page-local variant for something that should be a modifier or attribute? +5. If I added a token, modifier, or primitive, did I justify it in this file? + +## Drift Triggers + +These are common signs that the system is drifting: + +- app and RSS page use different canvas/background treatment +- same content type gets different card shells +- page-local spacing values appear outside the token scale +- headings start mixing unrelated type styles +- new input or card variants appear with overlapping purpose +- semantic states are encoded as a growing list of presentational classes + +If you see one of these, consolidate instead of layering more CSS. + +## Change Policy + +When changing the design system: + +- update the shared primitive first if the rule is cross-surface +- update this manifest if the rule changes +- keep the primitive API smaller, not larger, when possible +- validate both app UI and RSS/XSL output + +The right direction is brutal clarity: + +- fewer primitives +- fewer variants +- stronger shared identity +- less local exception code diff --git a/frontend/index.html b/frontend/index.html index 1324d6e2..e2538ef7 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -8,6 +8,7 @@ name="description" content="html2rss converts fixed demo pages or operator-submitted URLs into feed endpoints." /> + html2rss diff --git a/frontend/src/__tests__/App.contract.test.tsx b/frontend/src/__tests__/App.contract.test.tsx index b4f08a7a..87d906a6 100644 --- a/frontend/src/__tests__/App.contract.test.tsx +++ b/frontend/src/__tests__/App.contract.test.tsx @@ -18,7 +18,7 @@ describe('App contract', () => { http.post('/api/v1/feeds', async ({ request }) => { const body = (await request.json()) as { url: string; strategy: string }; - expect(body).toEqual({ url: 'https://example.com/articles', strategy: 'ssrf_filter' }); + expect(body).toEqual({ url: 'https://example.com/articles', strategy: 'browserless' }); expect(request.headers.get('authorization')).toBe(`Bearer ${token}`); return HttpResponse.json( @@ -35,7 +35,14 @@ describe('App contract', () => { return HttpResponse.json( { - items: [{ title: 'Contract Item' }], + items: [ + { + title: 'Contract Item', + content_text: 'Contract preview excerpt.', + url: 'https://example.com/contract-item', + date_published: '2024-01-01T00:00:00Z', + }, + ], }, { headers: { 'content-type': 'application/feed+json' }, @@ -47,6 +54,9 @@ describe('App contract', () => { render(); await screen.findByLabelText('Page URL'); + await waitFor(() => { + expect(screen.getByRole('combobox')).toHaveValue('browserless'); + }); const urlInput = screen.getByLabelText('Page URL') as HTMLInputElement; fireEvent.input(urlInput, { target: { value: 'https://example.com/articles' } }); @@ -54,16 +64,18 @@ describe('App contract', () => { fireEvent.click(screen.getByRole('button', { name: 'Generate feed URL' })); await waitFor(() => { + expect(screen.getByText('Your feed is ready')).toBeInTheDocument(); expect(screen.getByText('Example Feed')).toBeInTheDocument(); expect(screen.getByLabelText('Feed URL')).toBeInTheDocument(); expect(screen.getByRole('button', { name: 'Copy feed URL' })).toBeInTheDocument(); expect(screen.getByRole('link', { name: 'Open feed' })).toBeInTheDocument(); - expect(screen.getByRole('link', { name: 'JSON Feed' })).toHaveAttribute( + expect(screen.getByRole('link', { name: 'Open JSON Feed' })).toHaveAttribute( 'href', 'http://localhost:3000/api/v1/feeds/generated-token.json' ); expect(screen.getByRole('button', { name: 'Create another feed' })).toBeInTheDocument(); - expect(screen.getByText('Feed preview')).toBeInTheDocument(); + expect(screen.getByText('Preview')).toBeInTheDocument(); + expect(screen.getByText('Latest items from this feed')).toBeInTheDocument(); expect(screen.getByText('Contract Item')).toBeInTheDocument(); }); }); @@ -89,6 +101,7 @@ describe('App contract', () => { enabled: true, access_token_required: true, }, + featured_feeds: [], }, }, }); @@ -135,6 +148,9 @@ describe('App contract', () => { render(); await screen.findByLabelText('Page URL'); + await waitFor(() => { + expect(screen.getByRole('combobox')).toHaveValue('browserless'); + }); fireEvent.input(screen.getByLabelText('Page URL'), { target: { value: 'https://example.com/articles' }, diff --git a/frontend/src/__tests__/App.test.tsx b/frontend/src/__tests__/App.test.tsx index c5b44a7f..3085bb7c 100644 --- a/frontend/src/__tests__/App.test.tsx +++ b/frontend/src/__tests__/App.test.tsx @@ -38,6 +38,7 @@ describe('App', () => { beforeEach(() => { vi.clearAllMocks(); + window.history.replaceState({}, '', 'http://localhost:3000/'); mockUseAccessToken.mockReturnValue({ token: null, @@ -60,6 +61,7 @@ describe('App', () => { enabled: true, access_token_required: true, }, + featured_feeds: [], }, }, isLoading: false, @@ -77,8 +79,8 @@ describe('App', () => { mockUseStrategies.mockReturnValue({ strategies: [ - { id: 'ssrf_filter', name: 'ssrf_filter', display_name: 'Standard (recommended)' }, - { id: 'browserless', name: 'browserless', display_name: 'JavaScript pages' }, + { id: 'faraday', name: 'faraday', display_name: 'Default' }, + { id: 'browserless', name: 'browserless', display_name: 'JavaScript pages (recommended)' }, ], isLoading: false, error: null, @@ -102,6 +104,50 @@ describe('App', () => { }); }); + it('prefers browserless as the default strategy when available', () => { + render(); + + return waitFor(() => { + expect(screen.getByRole('combobox')).toHaveValue('browserless'); + }); + }); + + it('falls back to the first available strategy when browserless is unavailable', () => { + mockUseStrategies.mockReturnValue({ + strategies: [{ id: 'faraday', name: 'faraday', display_name: 'Default' }], + isLoading: false, + error: null, + }); + + render(); + + return waitFor(() => { + expect(screen.getByRole('combobox')).toHaveValue('faraday'); + }); + }); + + it('auto-submits a prefilled url using the resolved default strategy', async () => { + mockUseAccessToken.mockReturnValue({ + token: 'saved-token', + hasToken: true, + saveToken: mockSaveToken, + clearToken: mockClearToken, + isLoading: false, + error: null, + }); + window.history.replaceState({}, '', 'http://localhost:3000/?url=https%3A%2F%2Fexample.com%2Farticles'); + + render(); + + await waitFor(() => { + expect(mockConvertFeed).toHaveBeenCalledWith( + 'https://example.com/articles', + 'browserless', + 'saved-token' + ); + }); + }); + it('shows inline token prompt when submitting without a token', async () => { render(); @@ -123,14 +169,55 @@ describe('App', () => { expect(mockConvertFeed).not.toHaveBeenCalled(); }); + it('promotes included feeds when feed creation is disabled', () => { + mockUseApiMetadata.mockReturnValue({ + metadata: { + api: { + name: 'html2rss-web API', + description: 'RESTful API for converting websites to RSS feeds', + openapi_url: 'http://example.test/openapi.yaml', + }, + instance: { + feed_creation: { + enabled: false, + access_token_required: false, + }, + featured_feeds: [ + { + path: '/microsoft.com/azure-products.rss', + title: 'Azure product updates', + description: 'Follow Microsoft Azure product announcements from your own instance.', + }, + ], + }, + }, + isLoading: false, + error: null, + }); + + render(); + + expect(screen.getByText('Try a working included feed')).toBeInTheDocument(); + expect(screen.getByRole('link', { name: 'Azure product updates' })).toHaveAttribute( + 'href', + '/microsoft.com/azure-products.rss' + ); + expect(screen.getByText('Custom feed generation is disabled for this instance.')).toBeInTheDocument(); + }); + it('renders the result panel when a feed is available', async () => { + vi.spyOn(window, 'fetch').mockResolvedValue({ + ok: true, + json: async () => ({ items: [] }), + } as Response); + mockUseFeedConversion.mockReturnValue({ isConverting: false, result: { id: 'feed-123', name: 'Example Feed', url: 'https://example.com/articles', - strategy: 'ssrf_filter', + strategy: 'faraday', feed_token: 'example-token', public_url: '/api/v1/feeds/example-token', json_public_url: '/api/v1/feeds/example-token.json', @@ -196,7 +283,7 @@ describe('App', () => { expect(mockSaveToken).toHaveBeenCalledWith('token-123'); expect(mockConvertFeed).toHaveBeenCalledWith( 'https://example.com/articles', - 'ssrf_filter', + 'browserless', 'token-123' ); }); @@ -281,4 +368,27 @@ describe('App', () => { expect(bookmarklet.getAttribute('href')).toContain('/?url='); expect(bookmarklet.getAttribute('href')).not.toContain('%27+encodeURIComponent'); }); + + it('shows the utility links in a user-focused order', () => { + render(); + + fireEvent.click(screen.getByRole('button', { name: 'More' })); + + const utilityLinks = screen.getAllByRole('link').map((link) => link.textContent); + expect(utilityLinks).toEqual([ + 'Try included feeds', + 'Bookmarklet', + 'OpenAPI spec', + 'Source code', + ]); + + expect(screen.getByRole('link', { name: 'OpenAPI spec' })).toHaveAttribute( + 'href', + 'http://example.test/openapi.yaml' + ); + expect(screen.getByRole('link', { name: 'Try included feeds' })).toHaveAttribute( + 'href', + 'https://html2rss.github.io/web-application/how-to/use-included-configs/' + ); + }); }); diff --git a/frontend/src/__tests__/ResultDisplay.test.tsx b/frontend/src/__tests__/ResultDisplay.test.tsx index c9ad54a5..b69fd86c 100644 --- a/frontend/src/__tests__/ResultDisplay.test.tsx +++ b/frontend/src/__tests__/ResultDisplay.test.tsx @@ -9,7 +9,7 @@ describe('ResultDisplay', () => { id: 'test-id', name: 'Test Feed', url: 'https://example.com', - strategy: 'ssrf_filter', + strategy: 'faraday', feed_token: 'test-feed-token', public_url: 'https://example.com/feed.xml', json_public_url: 'https://example.com/feed.json', @@ -21,28 +21,44 @@ describe('ResultDisplay', () => { ok: true, json: async () => ({ items: [ - { title: 'Item One' }, - { content_text: '56 points by canpan 1 hour ago | hide | 18 comments' }, - { content_text: '2. Item Two ( example.com )' }, + { + title: 'Item One', + content_text: '

First preview item with markup.

', + url: 'https://example.com/item-one', + date_published: '2024-01-01T00:00:00Z', + }, + { + content_text: '56 points by canpan 1 hour ago | hide | 18 comments', + date_published: '2024-01-02T00:00:00Z', + }, + { + content_text: '2. Item Two ( example.com )', + url: 'https://example.com/item-two', + date_published: '2024-01-03T00:00:00Z', + }, ], }), } as Response); }); - it('renders the simplified result actions and preview', async () => { + it('renders the success state actions and richer preview cards', async () => { render(); + expect(screen.getByText('Your feed is ready')).toBeInTheDocument(); expect(screen.getByText('Test Feed')).toBeInTheDocument(); expect(screen.getByRole('button', { name: 'Copy feed URL' })).toBeInTheDocument(); expect(screen.getByRole('link', { name: 'Open feed' })).toBeInTheDocument(); - expect(screen.getByRole('link', { name: 'JSON Feed' })).toHaveAttribute( + expect(screen.getByRole('link', { name: 'Open JSON Feed' })).toHaveAttribute( 'href', 'https://example.com/feed.json' ); await waitFor(() => { expect(screen.getByText('Item One')).toBeInTheDocument(); + expect(screen.getByText('First preview item with markup.')).toBeInTheDocument(); + expect(screen.getAllByText('Open original')).toHaveLength(2); expect(screen.getByText(/points by canpan/i)).toBeInTheDocument(); expect(screen.getByText('Item Two')).toBeInTheDocument(); + expect(screen.getByText('Latest items from this feed')).toBeInTheDocument(); }); expect(window.fetch).toHaveBeenCalledWith('https://example.com/feed.xml', { headers: { Accept: 'application/feed+json' }, @@ -59,6 +75,7 @@ describe('ResultDisplay', () => { await waitFor(() => { expect(screen.getByText('Preview unavailable right now.')).toBeInTheDocument(); + expect(screen.getByText('Latest items from this feed')).toBeInTheDocument(); }); }); diff --git a/frontend/src/__tests__/mocks/server.ts b/frontend/src/__tests__/mocks/server.ts index 60098f03..00359bc6 100644 --- a/frontend/src/__tests__/mocks/server.ts +++ b/frontend/src/__tests__/mocks/server.ts @@ -16,6 +16,7 @@ export const server = setupServer( enabled: true, access_token_required: true, }, + featured_feeds: [], }, }, }); @@ -26,14 +27,14 @@ export const server = setupServer( data: { strategies: [ { - id: 'ssrf_filter', - name: 'ssrf_filter', - display_name: 'Standard (recommended)', + id: 'faraday', + name: 'faraday', + display_name: 'Default', }, { id: 'browserless', name: 'browserless', - display_name: 'JavaScript pages', + display_name: 'JavaScript pages (recommended)', }, ], }, @@ -64,7 +65,7 @@ export function buildFeedResponse(overrides: FeedResponseOverrides = {}) { id: overrides.id ?? 'feed-123', name: overrides.name ?? 'Example Feed', url: overrides.url ?? 'https://example.com/articles', - strategy: overrides.strategy ?? 'ssrf_filter', + strategy: overrides.strategy ?? 'faraday', feed_token: overrides.feed_token ?? 'example-token', public_url: overrides.public_url ?? '/api/v1/feeds/example-token', json_public_url: overrides.json_public_url ?? '/api/v1/feeds/example-token.json', diff --git a/frontend/src/__tests__/useFeedConversion.contract.test.ts b/frontend/src/__tests__/useFeedConversion.contract.test.ts index 3c46604d..a7f51a17 100644 --- a/frontend/src/__tests__/useFeedConversion.contract.test.ts +++ b/frontend/src/__tests__/useFeedConversion.contract.test.ts @@ -13,7 +13,7 @@ describe('useFeedConversion contract', () => { const body = (await request.json()) as { url: string; strategy: string }; receivedAuthorization = request.headers.get('authorization'); - expect(body).toEqual({ url: 'https://example.com/articles', strategy: 'ssrf_filter' }); + expect(body).toEqual({ url: 'https://example.com/articles', strategy: 'faraday' }); return HttpResponse.json( buildFeedResponse({ @@ -30,7 +30,7 @@ describe('useFeedConversion contract', () => { const { result } = renderHook(() => useFeedConversion()); await act(async () => { - await result.current.convertFeed('https://example.com/articles', 'ssrf_filter', 'test-token-123'); + await result.current.convertFeed('https://example.com/articles', 'faraday', 'test-token-123'); }); expect(receivedAuthorization).toBe('Bearer test-token-123'); @@ -54,7 +54,7 @@ describe('useFeedConversion contract', () => { await act(async () => { await expect( - result.current.convertFeed('https://example.com/articles', 'ssrf_filter', 'token') + result.current.convertFeed('https://example.com/articles', 'faraday', 'token') ).rejects.toThrow('URL parameter is required'); }); @@ -76,7 +76,7 @@ describe('useFeedConversion contract', () => { await act(async () => { await expect( - result.current.convertFeed('https://example.com/articles', 'ssrf_filter', 'token') + result.current.convertFeed('https://example.com/articles', 'faraday', 'token') ).rejects.toThrow('Invalid response format from feed creation API'); }); diff --git a/frontend/src/__tests__/useFeedConversion.test.ts b/frontend/src/__tests__/useFeedConversion.test.ts index c2583985..9fe72176 100644 --- a/frontend/src/__tests__/useFeedConversion.test.ts +++ b/frontend/src/__tests__/useFeedConversion.test.ts @@ -27,7 +27,7 @@ describe('useFeedConversion', () => { id: 'test-id', name: 'Test Feed', url: 'https://example.com', - strategy: 'ssrf_filter', + strategy: 'faraday', feed_token: 'test-token', public_url: 'https://example.com/feed.xml', json_public_url: 'https://example.com/feed.json', @@ -51,7 +51,7 @@ describe('useFeedConversion', () => { const { result } = renderHook(() => useFeedConversion()); await act(async () => { - await result.current.convertFeed('https://example.com', 'ssrf_filter', 'testtoken'); + await result.current.convertFeed('https://example.com', 'faraday', 'testtoken'); }); expect(result.current.isConverting).toBe(false); @@ -77,9 +77,9 @@ describe('useFeedConversion', () => { const { result } = renderHook(() => useFeedConversion()); await act(async () => { - await expect( - result.current.convertFeed('https://example.com', 'ssrf_filter', 'testtoken') - ).rejects.toThrow('Bad Request'); + await expect(result.current.convertFeed('https://example.com', 'faraday', 'testtoken')).rejects.toThrow( + 'Bad Request' + ); }); expect(result.current.isConverting).toBe(false); @@ -93,9 +93,9 @@ describe('useFeedConversion', () => { const { result } = renderHook(() => useFeedConversion()); await act(async () => { - await expect( - result.current.convertFeed('https://example.com', 'ssrf_filter', 'testtoken') - ).rejects.toThrow('Network error'); + await expect(result.current.convertFeed('https://example.com', 'faraday', 'testtoken')).rejects.toThrow( + 'Network error' + ); }); expect(result.current.isConverting).toBe(false); diff --git a/frontend/src/api/contracts.ts b/frontend/src/api/contracts.ts index 57d0a54b..14867b84 100644 --- a/frontend/src/api/contracts.ts +++ b/frontend/src/api/contracts.ts @@ -10,5 +10,10 @@ export interface ApiMetadataRecord { enabled: boolean; access_token_required: boolean; }; + featured_feeds?: Array<{ + path: string; + title: string; + description: string; + }>; }; } diff --git a/frontend/src/components/App.tsx b/frontend/src/components/App.tsx index 02ccdd75..8fa7d018 100644 --- a/frontend/src/components/App.tsx +++ b/frontend/src/components/App.tsx @@ -1,4 +1,4 @@ -import { useEffect, useState } from 'preact/hooks'; +import { useEffect, useRef, useState } from 'preact/hooks'; import { ResultDisplay } from './ResultDisplay'; import { CreateFeedPanel, UtilityStrip, type Strategy } from './AppPanels'; import { useAccessToken } from '../hooks/useAccessToken'; @@ -8,6 +8,8 @@ import { useStrategies } from '../hooks/useStrategies'; const EMPTY_FEED_ERRORS = { url: '', form: '' }; const DEFAULT_FEED_CREATION = { enabled: true, access_token_required: true }; +const preferredStrategy = (strategies: { id: string }[]) => + strategies.find((strategy) => strategy.id === 'browserless')?.id ?? strategies[0]?.id; function BrandLockup() { return ( @@ -42,25 +44,29 @@ export function App() { } = useFeedConversion(); const { strategies, isLoading: strategiesLoading, error: strategiesError } = useStrategies(); - const [feedFormData, setFeedFormData] = useState({ url: '', strategy: 'ssrf_filter' }); + const [feedFormData, setFeedFormData] = useState({ url: '', strategy: '' }); const [feedFieldErrors, setFeedFieldErrors] = useState(EMPTY_FEED_ERRORS); const [showTokenPrompt, setShowTokenPrompt] = useState(false); const [tokenDraft, setTokenDraft] = useState(''); const [tokenError, setTokenError] = useState(''); const [focusCreateComposerKey, setFocusCreateComposerKey] = useState(0); + const autoSubmitUrlRef = useRef(null); + const hasAutoSubmittedRef = useRef(false); + const selectedStrategy = feedFormData.strategy || preferredStrategy(strategies) || ''; useEffect(() => { if (typeof window === 'undefined') return; - if (feedFormData.url) return; const urlParam = new URLSearchParams(window.location.search).get('url'); if (!urlParam) return; + autoSubmitUrlRef.current = urlParam; + if (feedFormData.url) return; setFeedFormData((prev) => ({ ...prev, url: urlParam })); }, [feedFormData.url]); useEffect(() => { - const nextStrategy = strategies[0]?.id; + const nextStrategy = preferredStrategy(strategies); if (!nextStrategy) return; const hasCurrentStrategy = strategies.some((strategy) => strategy.id === feedFormData.strategy); @@ -68,6 +74,8 @@ export function App() { }, [strategies, feedFormData.strategy]); const feedCreation = metadata?.instance.feed_creation ?? DEFAULT_FEED_CREATION; + const featuredFeeds = metadata?.instance.featured_feeds ?? []; + const submitDisabled = isConverting || strategiesLoading || !feedCreation.enabled || showTokenPrompt; const setFeedField = (key: 'url' | 'strategy', value: string) => { setFeedFormData((prev) => ({ ...prev, [key]: value })); @@ -80,7 +88,7 @@ export function App() { }; const strategyHint = (strategy: Strategy) => { - if (strategy.id === 'ssrf_filter') return 'Start here for most pages.'; + if (strategy.id === 'faraday') return 'Start here for most pages.'; if (strategy.id === 'browserless') return 'Use this if the page loads content with JavaScript.'; return strategy.name; }; @@ -96,11 +104,18 @@ export function App() { }; const attemptFeedCreation = async (accessToken: string) => { + const strategy = selectedStrategy; + if (!feedFormData.url.trim()) { setFeedFieldErrors({ ...EMPTY_FEED_ERRORS, url: 'Source URL is required.' }); return false; } + if (!strategy) { + setFeedFieldErrors({ ...EMPTY_FEED_ERRORS, form: 'Strategy is required' }); + return false; + } + if (!feedCreation.enabled) { setFeedFieldErrors({ ...EMPTY_FEED_ERRORS, @@ -117,7 +132,7 @@ export function App() { } try { - await convertFeed(feedFormData.url, feedFormData.strategy, accessToken); + await convertFeed(feedFormData.url, strategy, accessToken); setShowTokenPrompt(false); setTokenError(''); return true; @@ -166,12 +181,23 @@ export function App() { setFocusCreateComposerKey((current) => current + 1); }; + useEffect(() => { + const autoSubmitUrl = autoSubmitUrlRef.current; + if (!autoSubmitUrl || hasAutoSubmittedRef.current) return; + if (strategiesLoading || metadataLoading || tokenLoading) return; + if (feedFormData.url !== autoSubmitUrl || !selectedStrategy) return; + + hasAutoSubmittedRef.current = true; + setFeedFieldErrors(EMPTY_FEED_ERRORS); + void attemptFeedCreation(token ?? ''); + }, [feedFormData.url, metadataLoading, selectedStrategy, strategiesLoading, token, tokenLoading]); + if (metadataLoading || tokenLoading) { return (
-
-