diff --git a/.github/prompts/docs-self-healing-router.md b/.github/prompts/docs-self-healing-router.md index bb8b29e4af..406743171b 100644 --- a/.github/prompts/docs-self-healing-router.md +++ b/.github/prompts/docs-self-healing-router.md @@ -72,6 +72,13 @@ You run on Haiku for cost efficiency. Do NOT draft content or create PRs. - `"micro"` — ALL targets are micro-edits (`add_link`, `add_mention`, `add_tip`). Haiku can handle these. - `"full"` — at least one target is `create_page`, `update_section`, `add_section`, or `create_category`. Requires Sonnet. +Classify from the `action:` values you just wrote, and from nothing else. This +is a mechanical rule, not a judgement call: how small or obvious the change +feels is irrelevant, and a single `add_section` among ten `add_link` targets +still makes the whole PR `full`. The workflow re-checks this and overwrites a +`micro` that carries a Sonnet-only action, so getting it wrong either loses the +PR or produces a draft you were not equipped to write. + **`targets_yaml`** is the full Router YAML output (as a string), only when `decision` is `has_targets`. Include `doc_type`, `template`, `guide`, `confidence`, and the full `targets` block. ## Step 5 — Execute micro-edits (if all targets are micro) @@ -87,6 +94,16 @@ Then create the branch and PR: ```bash cd $DOC_REPO + +# Always branch from a clean origin/main. Resetting at the END of the previous +# iteration was not enough: on 2026-09-03 you opened PR #3439, then ran +# `git checkout -b` for the next PR while still on that branch, so PR #3440 +# carried #3439's commit too. Reset FIRST and each branch is independent +# whatever the previous iteration did. +git checkout main +git clean -fd +git reset --hard origin/main + BRANCH_NAME="/" git checkout -b "$BRANCH_NAME" git add . @@ -151,6 +168,7 @@ Update `/tmp/router-results.json` to include a `doc_pr` field for micro PRs you - **Do NOT read any agent prompts except `router.md`** - **For micro-edits only:** you may read and modify documentation files and create branches/PRs - **For full complexity:** do NOT modify files or create PRs — leave that for Sonnet +- **Never draft a section.** A micro-edit is a link, a mention, or a tip. If the change needs a new section, a rewritten section, a new page, or a new category, it is `full` and you stop at the routing decision. - **ONLY read diffs, the Router prompt, sidebars.js, llms.txt, and write the result file** (plus doc files for micro-edits) - **Max 5 PRs per run.** Log extras to stdout for the next run. - **NEVER run any write operation on strapi/strapi** diff --git a/.github/workflows/docs-self-healing.yml b/.github/workflows/docs-self-healing.yml index 08c7a182a8..4479bb4cf3 100644 --- a/.github/workflows/docs-self-healing.yml +++ b/.github/workflows/docs-self-healing.yml @@ -485,6 +485,54 @@ jobs: exit 0 fi + # ── Correct the Router's own classification before trusting it ── + # + # `micro` vs `full` is a mechanical rule: micro means every target is + # `add_link`, `add_mention` or `add_tip`, anything else is Sonnet's. + # Haiku has now got it wrong twice, in opposite directions: + # + # 2026-06-20 (run 27857268241) — stamped `micro` on `update_section` + # targets and then did nothing, because its prose reasoning had + # correctly concluded they were Sonnet's job. FULL=0 gated Sonnet + # out and both PRs were lost silently. + # 2026-09-03 (run 33705096766) — stamped `micro` on `add_section` and + # `update_section` targets and drafted whole sections itself, which + # its own micro-edit procedure does not even cover. Sonnet was + # never invoked. + # + # The rule needs no judgement, so the workflow applies it instead of + # asking the model to. Two outcomes: + # + # - No doc PR yet: rewrite to `full` so Sonnet picks it up. That is + # the 2026-06-20 rescue. + # - Doc PR already open: leave the record alone, because rerouting + # would duplicate the PR, and mark it `overreach` so the reporting + # says a Haiku draft needs closer review. That is 2026-09-03. + FULL_ONLY='action:[[:space:]]*(create_page|update_section|add_section|create_category)' + + jq --arg re "$FULL_ONLY" ' + .prs |= map( + if .decision == "has_targets" + and .complexity == "micro" + and ((.targets_yaml // "") | test($re)) + then + if (.doc_pr // "") == "" + then . + {complexity: "full", router_complexity: "micro"} + else . + {overreach: true, router_complexity: "micro"} + end + else . end + )' "$RESULTS_FILE" > "$RESULTS_FILE.tmp" && mv "$RESULTS_FILE.tmp" "$RESULTS_FILE" + + REROUTED=$(jq '[.prs[] | select(.router_complexity == "micro" and .complexity == "full")] | length' "$RESULTS_FILE") + OVERREACH=$(jq '[.prs[] | select(.overreach == true)] | length' "$RESULTS_FILE") + + if [ "$REROUTED" -gt 0 ]; then + echo "Rerouted $REROUTED misclassified micro target(s) to Sonnet" + fi + if [ "$OVERREACH" -gt 0 ]; then + echo "$OVERREACH doc PR(s) drafted by Haiku beyond its micro scope" + fi + # Count PRs by decision and complexity MICRO=$(jq '[.prs[] | select(.decision == "has_targets" and .complexity == "micro")] | length' "$RESULTS_FILE") FULL=$(jq '[.prs[] | select(.decision == "has_targets" and .complexity == "full")] | length' "$RESULTS_FILE") @@ -548,75 +596,95 @@ jobs: # ── The dead zone ── # - # A PR marked `micro` is handled by the Router itself; a PR marked - # `full` is handed to Sonnet. Nothing checks that the Router actually - # did the work it claimed, so a PR can fall between the two stages and - # the run still ends green and silent. - # - # Observed on 2026-06-20 (run 27857268241): Haiku stamped `micro` on two - # PRs whose targets were `update_section` — which its own prompt defines - # as `full` — and then did not execute them, because its prose reasoning - # had correctly concluded they were Sonnet's job. FULL=0 gated Sonnet - # out, so neither stage acted and both PRs were lost without a trace. + # Whatever the classification stage produced, the run must not end + # green and silent on a target nobody handled. Three distinct + # situations, three distinct messages: conflating them is what made + # the 2026-09-03 Slack post announce "2 doc PR(s) created" and + # "claimed but not delivered" about the very same two PRs. # - # Detected here rather than prevented: the Router keeps deciding, but a - # decision it did not honour is now reported instead of swallowed. - - # 1. Claimed micro, no PR produced. + # dropped — still `micro` after the correction above and no doc PR + # came out. Genuinely nobody's work, no auto-rescue. + # rerouted — misclassified, rewritten to `full` above, so Sonnet + # handles it in this same run. Informational. + # overreach — misclassified, but Haiku had already opened the PR. + # Delivered, just by the wrong model. Needs review. DROPPED=$(jq -r '[.prs[] | select(.decision == "has_targets" and .complexity == "micro" and (.doc_pr // "") == "")] | length' "$RESULTS_FILE") - # 2. Claimed micro, but the targets contain an action the prompt - # reserves for Sonnet. This is the misclassification itself, caught - # even when the PR was never dropped. - MISCLASSIFIED=$(jq -r '[.prs[] - | select(.decision == "has_targets" and .complexity == "micro") - | select((.targets_yaml // "") | test("action:[[:space:]]*(create_page|update_section|add_section|create_category)")) - ] | length' "$RESULTS_FILE") - echo "dead_zone_dropped=$DROPPED" >> $GITHUB_OUTPUT - echo "dead_zone_misclassified=$MISCLASSIFIED" >> $GITHUB_OUTPUT + echo "dead_zone_rerouted=$REROUTED" >> $GITHUB_OUTPUT + echo "dead_zone_overreach=$OVERREACH" >> $GITHUB_OUTPUT - if [ "$DROPPED" -gt 0 ] || [ "$MISCLASSIFIED" -gt 0 ]; then - echo "::warning::Router dead zone — $DROPPED micro target(s) produced no PR, $MISCLASSIFIED classified micro despite a full-only action" + if [ "$DROPPED" -gt 0 ]; then + echo "::warning::Router dead zone — $DROPPED micro target(s) produced no PR" { echo "### ⚠️ Router dead zone" echo "" - echo "A PR classified \`micro\` is the Router's own job, and one" - echo "classified \`full\` goes to Sonnet. The PRs below were claimed" - echo "but not delivered, so no stage handled them." + echo "These PRs were classified \`micro\`, which makes them the" + echo "Router's own job, and no doc PR came out. No stage handled them." echo "" } >> $GITHUB_STEP_SUMMARY - if [ "$DROPPED" -gt 0 ]; then - echo "**Claimed \`micro\`, no doc PR produced ($DROPPED):**" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - jq -r '.prs[] | select(.decision == "has_targets" and .complexity == "micro" and (.doc_pr // "") == "") - | "- [strapi/strapi#\(.number) — \(.title)](https://github.com/strapi/strapi/pull/\(.number))"' \ - "$RESULTS_FILE" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - fi + jq -r '.prs[] | select(.decision == "has_targets" and .complexity == "micro" and (.doc_pr // "") == "") + | "- [strapi/strapi#\(.number) — \(.title)](https://github.com/strapi/strapi/pull/\(.number))"' \ + "$RESULTS_FILE" >> $GITHUB_STEP_SUMMARY - if [ "$MISCLASSIFIED" -gt 0 ]; then - echo "**Classified \`micro\` despite a full-only action ($MISCLASSIFIED):**" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - jq -r '.prs[] - | select(.decision == "has_targets" and .complexity == "micro") - | select((.targets_yaml // "") | test("action:[[:space:]]*(create_page|update_section|add_section|create_category)")) - | "- [strapi/strapi#\(.number) — \(.title)](https://github.com/strapi/strapi/pull/\(.number)) — actions: " + - ([(.targets_yaml // "") | scan("action:[[:space:]]*([a-z_]+)") | .[0]] | unique | join(", "))' \ - "$RESULTS_FILE" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - fi + { + echo "" + echo "They are **not** added to the ignore list, so a later run can still pick them up, but only while they remain inside the 24-hour window." + echo "" + } >> $GITHUB_STEP_SUMMARY + fi + + if [ "$REROUTED" -gt 0 ]; then + echo "::notice::Router misclassification corrected — $REROUTED micro target(s) rerouted to Sonnet" - echo "These PRs are **not** added to the ignore list, so a later run can still pick them up — but only while they remain inside the 24-hour window." >> $GITHUB_STEP_SUMMARY + { + echo "### 🔁 Misclassification corrected" + echo "" + echo "The Router classified these \`micro\` despite a target action" + echo "reserved for Sonnet. The workflow rewrote them to \`full\`, so" + echo "Sonnet drafted them in this run." + echo "" + } >> $GITHUB_STEP_SUMMARY + + jq -r '.prs[] | select(.router_complexity == "micro" and .complexity == "full") + | "- [strapi/strapi#\(.number) — \(.title)](https://github.com/strapi/strapi/pull/\(.number)) — actions: " + + ([(.targets_yaml // "") | scan("action:[[:space:]]*([a-z_]+)") | .[0]] | unique | join(", "))' \ + "$RESULTS_FILE" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY fi - # Pass router results to Sonnet (only full-complexity PRs) + if [ "$OVERREACH" -gt 0 ]; then + echo "::warning::$OVERREACH doc PR(s) drafted by Haiku beyond its micro scope — review with extra care" + + { + echo "### ⚠️ Drafted beyond the micro scope" + echo "" + echo "The Router classified these \`micro\` despite a target action" + echo "reserved for Sonnet, then drafted them itself. The doc PRs" + echo "exist, but Haiku wrote section-level prose its micro-edit" + echo "procedure does not cover. **Review them more closely than a" + echo "normal micro-edit PR.**" + echo "" + } >> $GITHUB_STEP_SUMMARY + + jq -r '.prs[] | select(.overreach == true) + | "- [strapi/strapi#\(.number) — \(.title)](https://github.com/strapi/strapi/pull/\(.number)) → [doc PR](\(.doc_pr)) — actions: " + + ([(.targets_yaml // "") | scan("action:[[:space:]]*([a-z_]+)") | .[0]] | unique | join(", "))' \ + "$RESULTS_FILE" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + fi + + # Pass router results to Sonnet. The comment here used to claim "only + # full-complexity PRs" while handing over the whole file, and the + # Drafter prompt filters on `decision` alone — so a run with both micro + # and full targets would have had Sonnet redraft the micro PRs Haiku + # had just opened. Filter here rather than trusting the prompt, and + # exclude anything that already has a doc PR. { echo "router_results<> $GITHUB_OUTPUT @@ -828,20 +896,44 @@ jobs: # created other PRs. Left unreported it is invisible: green run, no # error, no PR. See the comment in "Check Router results". DROPPED="${{ steps.check-router.outputs.dead_zone_dropped }}" - MISCLASSIFIED="${{ steps.check-router.outputs.dead_zone_misclassified }}" + REROUTED="${{ steps.check-router.outputs.dead_zone_rerouted }}" + OVERREACH="${{ steps.check-router.outputs.dead_zone_overreach }}" DROPPED="${DROPPED:-0}" - MISCLASSIFIED="${MISCLASSIFIED:-0}" + REROUTED="${REROUTED:-0}" + OVERREACH="${OVERREACH:-0}" DEAD_ZONE="" - if [ "$DROPPED" -gt 0 ] || [ "$MISCLASSIFIED" -gt 0 ]; then - DEAD_ZONE_LIST=$(jq -r '.prs[] - | select(.decision == "has_targets" and .complexity == "micro") - | select((.doc_pr // "") == "" or ((.targets_yaml // "") | test("action:[[:space:]]*(create_page|update_section|add_section|create_category)"))) + + # Claimed micro, produced nothing: no stage handled it. + if [ "$DROPPED" -gt 0 ] && [ -f "$ROUTER_FILE" ]; then + LIST=$(jq -r '.prs[] | select(.decision == "has_targets" and .complexity == "micro" and (.doc_pr // "") == "") | "• "' \ "$ROUTER_FILE" 2>/dev/null || true) + if [ -n "$LIST" ]; then + DEAD_ZONE="${DEAD_ZONE}\n\n:pepe_alarm: *Router dead zone — claimed but not delivered:*\n${LIST}\nNeither Haiku nor Sonnet handled these. They stay out of the ignore list, so a rerun within 24h can still catch them." + fi + fi - if [ -n "$DEAD_ZONE_LIST" ]; then - DEAD_ZONE="\n\n:pepe_alarm: *Router dead zone — claimed but not delivered:*\n${DEAD_ZONE_LIST}\nNeither Haiku nor Sonnet handled these. They stay out of the ignore list, so a rerun within 24h can still catch them." + # Misclassified and rescued: Sonnet handled them and their doc PRs are + # already in the created list above. Reported so the misclassification + # stays visible instead of passing for a normal run. + if [ "$REROUTED" -gt 0 ] && [ -f "$ROUTER_FILE" ]; then + LIST=$(jq -r '.prs[] | select(.router_complexity == "micro" and .complexity == "full") + | "• "' \ + "$ROUTER_FILE" 2>/dev/null || true) + if [ -n "$LIST" ]; then + DEAD_ZONE="${DEAD_ZONE}\n\n:arrows_counterclockwise: *Misclassified as micro, rerouted to Sonnet:*\n${LIST}\nThe Router called these micro despite a Sonnet-only action. The workflow corrected it, so Sonnet drafted them in this run." + fi + fi + + # Misclassified but already delivered by Haiku: the doc PR exists, the + # model that wrote it was the wrong one. + if [ "$OVERREACH" -gt 0 ] && [ -f "$ROUTER_FILE" ]; then + LIST=$(jq -r '.prs[] | select(.overreach == true) + | "• <\(.doc_pr)|strapi/strapi#\(.number) — \(.title)>"' \ + "$ROUTER_FILE" 2>/dev/null || true) + if [ -n "$LIST" ]; then + DEAD_ZONE="${DEAD_ZONE}\n\n:warning: *Drafted by Haiku beyond its micro scope — review with extra care:*\n${LIST}\nThese doc PRs exist, but Haiku wrote section-level prose its micro-edit procedure does not cover." fi fi @@ -864,7 +956,7 @@ jobs: # A dropped target is not "found nothing" — the Router found targets # and failed to act on them. Saying otherwise is how 2026-06-20 went # unnoticed. - if [ "$DROPPED" -gt 0 ] || [ "$MISCLASSIFIED" -gt 0 ]; then + if [ "$DROPPED" -gt 0 ]; then MAIN=":pepe_alarm: *Docs Self-Healing* ($DATE)\nRouter (Haiku) found targets but produced no doc PR.\n<${RUN_URL}|View run>" else MAIN=":frog_shrug: *Docs Self-Healing* ($DATE)\nRouter (Haiku) found no documentation targets. Sonnet was not invoked.\n<${RUN_URL}|View run>"