docs: changelog and blog post for v0.10.0

hindsight-dev/hindsight_dev/generate_changelog.py

@@ -12,9 +12,11 @@
 import re
 import subprocess
 import sys
+from concurrent.futures import ThreadPoolExecutor
 from dataclasses import dataclass
 from pathlib import Path
 
+from hindsight_api.engine.token_encoding import count_tokens
 from openai import OpenAI
 from pydantic import BaseModel
 from rich.console import Console
@@ -624,16 +626,60 @@ def render_migrations_section(migrations: list[Migration]) -> list[str]:
     return lines
 
 
-def analyze_commits_with_llm(
-    client: OpenAI,
-    model: str,
+# A release's commits are summarized in token-sized batches rather than in one call.
+# One call over a whole release makes the model compress: 0.10.0's 220 commits came
+# back as 37 entries against 65 for the 145-commit release before it, with whole
+# user-facing fixes unlisted. A batch is small enough that every commit in it can be
+# considered on its own, and batches are independent, so they run concurrently.
+#
+# The budget is deliberately far below the model's input limit. Commit subjects are
+# short: a 245-commit release is ~8500 tokens in total, so any budget near the context
+# window is one batch and changes nothing. What is being bounded here is how much the
+# model is asked to hold at once before it starts summarizing away, not what fits.
+BATCH_TOKEN_BUDGET = 2000
+MAX_PARALLEL_BATCHES = 8
+# gpt-5.6-terra bills reasoning tokens against the response budget, so this has to
+# leave room for the reasoning that precedes the first entry, not just the entries.
+RESPONSE_TOKEN_BUDGET = 100000
+
+
+def _commit_payload(commit: Commit) -> dict[str, str]:
+    return {"commit_id": commit.hash, "message": commit.message}
+
+
+def batch_commits(commits: list[Commit], token_budget: int | None = None) -> list[list[Commit]]:
+    """Split commits into batches of at most ``token_budget`` tokens of commit text.
+
+    Batches follow the input order, so each one holds a contiguous run of the
+    release's history and the merged result stays in commit order. A single commit
+    over budget gets a batch to itself rather than being dropped or truncated.
+    """
+    # Read the module constant at call time rather than binding it as a default:
+    # a default is bound at import, so raising BATCH_TOKEN_BUDGET to compare batched
+    # against unbatched output silently measures two batched runs.
+    token_budget = BATCH_TOKEN_BUDGET if token_budget is None else token_budget
+    batches: list[list[Commit]] = []
+    current: list[Commit] = []
+    current_tokens = 0
+    for commit in commits:
+        tokens = count_tokens(json.dumps(_commit_payload(commit)))
+        if current and current_tokens + tokens > token_budget:
+            batches.append(current)
+            current, current_tokens = [], 0
+        current.append(commit)
+        current_tokens += tokens
+    if current:
+        batches.append(current)
+    return batches
+
+
+def _build_analysis_prompt(
     version: str,
     commits: list[Commit],
     file_diff: str,
-    integration: str | None = None,
-) -> list[ChangelogEntry]:
-    """Use LLM to analyze commits and return structured changelog entries."""
-    commits_json = json.dumps([{"commit_id": c.hash, "message": c.message} for c in commits], indent=2)
+    integration: str | None,
+) -> str:
+    commits_json = json.dumps([_commit_payload(c) for c in commits], indent=2)
 
     subject = f"the {integration} integration for Hindsight" if integration else f"release {version} of Hindsight"
 
@@ -647,35 +693,124 @@ def analyze_commits_with_llm(
         "its own package) — integrations are versioned and changelogged separately"
     )
 
-    prompt = f"""Analyze the following git commits for {subject} (an AI memory system).
+    return f"""Analyze the following git commits for {subject} (an AI memory system).
 
 For each meaningful change, create a changelog entry with:
 - category: one of "feature", "improvement", "bugfix", "breaking", "other"
 - summary: brief one-line description of the change (user-facing, not technical)
 - commit_id: the commit hash from the input
 
 Rules:
-- Group related commits into a single entry if they're part of the same change
+- Group related commits into a single entry ONLY when they are literally parts of one
+  change (a fix and its follow-up, the same feature landed across two commits). Do not
+  merge distinct fixes into one summary because they touch the same area
+- Be complete rather than selective: every commit that changes what a user, operator or
+  API caller can observe gets its own entry. A bug fix in retain, recall, reflect,
+  consolidation, embeddings, the CLI, the control plane, a provider integration, the
+  Docker images or the Helm chart is user-facing even when its title reads as internal
 - Skip trivial changes (typo fixes, formatting, internal refactoring)
 - Skip repository-only changes: README updates, CI/GitHub Actions, release scripts, changelog updates, version bumps{skip_integrations_rule}
 - Focus on user-facing changes that affect the product functionality
 - Use the exact commit_id from the input (pick the most relevant one if grouping)
 - If no meaningful changes remain after filtering, return an empty list
 
+This is one batch of the release's commits, so judge each commit on its own; do not
+assume a change is absent because its other half is not in this batch.
+
 Commits:
 {commits_json}
 
 Files changed summary:
 {file_diff[:4000]}"""
 
+
+def _parse_entries(client: OpenAI, model: str, prompt: str) -> list[ChangelogEntry]:
     response = client.beta.chat.completions.parse(
         model=model,
         messages=[{"role": "user", "content": prompt}],
         response_format=ChangelogResponse,
-        max_completion_tokens=16000,
+        max_completion_tokens=RESPONSE_TOKEN_BUDGET,
     )
+    parsed = response.choices[0].message.parsed
+    return parsed.entries if parsed else []
+
+
+def deduplicate_entries(
+    client: OpenAI,
+    model: str,
+    entries: list[ChangelogEntry],
+    commit_order: list[str],
+) -> list[ChangelogEntry]:
+    """Merge entries that describe the same change across batch boundaries.
+
+    Batches are contiguous, but a feature and its follow-up fix can still straddle
+    two of them, and two batches can describe one rollout twice. A deterministic
+    pass drops repeated commit_ids first — that needs no model — and the LLM pass
+    only has to catch the same change described in two different ways.
+
+    The result is re-ordered by the release's own commit order and filtered to
+    commit_ids that were actually in the input, so a dropped or invented commit_id
+    cannot reorder the changelog or point a reader at a commit that isn't there.
+    """
+    seen: set[str] = set()
+    unique: list[ChangelogEntry] = []
+    for entry in entries:
+        if entry.commit_id in seen:
+            continue
+        seen.add(entry.commit_id)
+        unique.append(entry)
+
+    if len(unique) > 1:
+        prompt = f"""These changelog entries were produced independently from separate batches of one
+release's commits, so the same change may appear more than once, worded differently.
+
+Return the entries to keep:
+- Drop an entry only when another entry describes the same underlying change. When two
+  entries describe one change, keep the clearer summary and its commit_id
+- Keep everything else exactly as it is: same summary text, same category, same commit_id
+- Do not merge distinct changes, do not reword what you keep, and do not invent entries
+
+Entries:
+{json.dumps([e.model_dump() for e in unique], indent=2)}"""
+        deduped = _parse_entries(client, model, prompt)
+        by_id = {e.commit_id: e for e in unique}
+        # Keep the model's text but never its idea of which commits exist.
+        unique = [e for e in deduped if e.commit_id in by_id] or unique
+
+    position = {commit_id: i for i, commit_id in enumerate(commit_order)}
+    return sorted(unique, key=lambda e: position.get(e.commit_id, len(position)))
+
+
+def analyze_commits_with_llm(
+    client: OpenAI,
+    model: str,
+    version: str,
+    commits: list[Commit],
+    file_diff: str,
+    integration: str | None = None,
+) -> list[ChangelogEntry]:
+    """Use LLM to analyze commits and return structured changelog entries."""
+    batches = batch_commits(commits)
+
+    if len(batches) == 1:
+        return _parse_entries(client, model, _build_analysis_prompt(version, batches[0], file_diff, integration))
+
+    console.print(f"[blue]Summarizing {len(commits)} commits in {len(batches)} batches...[/blue]")
+    with ThreadPoolExecutor(max_workers=MAX_PARALLEL_BATCHES) as pool:
+        results = list(
+            pool.map(
+                lambda batch: _parse_entries(
+                    client, model, _build_analysis_prompt(version, batch, file_diff, integration)
+                ),
+                batches,
+            )
+        )
 
-    return response.choices[0].message.parsed.entries
+    entries = [entry for batch_entries in results for entry in batch_entries]
+    console.print(f"[blue]{len(entries)} entries before deduplication[/blue]")
+    deduped = deduplicate_entries(client, model, entries, [c.hash for c in commits])
+    console.print(f"[blue]{len(deduped)} entries after deduplication[/blue]")
+    return deduped
 
 
 def build_changelog_markdown(
@@ -773,7 +908,7 @@ def write_changelog(path: Path, header: str, new_entry: str, existing_releases:
 
 def generate_changelog_entry(
     version: str,
-    llm_model: str = "gpt-5.2",
+    llm_model: str = "gpt-5.6-terra",
 ) -> None:
     """Generate changelog entry for a specific version."""
     api_key = os.environ.get("OPENAI_API_KEY")
@@ -881,7 +1016,7 @@ def generate_changelog_entry(
 def generate_integration_changelog_entry(
     integration: str,
     version: str,
-    llm_model: str = "gpt-5.2",
+    llm_model: str = "gpt-5.6-terra",
 ) -> None:
     """Generate changelog entry for a specific integration version."""
     if integration not in VALID_INTEGRATIONS:
@@ -1004,8 +1139,8 @@ def main():
     )
     parser.add_argument(
         "--model",
-        default="gpt-5.2",
-        help="OpenAI model to use (default: gpt-5.2)",
+        default="gpt-5.6-terra",
+        help="OpenAI model to use (default: gpt-5.6-terra)",
     )
     parser.add_argument(
         "--integration",

hindsight-dev/tests/test_generate_changelog_batching.py

@@ -0,0 +1,135 @@
+"""Tests for token-batched, parallel commit summarization and its dedup pass.
+
+The LLM calls themselves are stubbed: what these pin is the batching arithmetic
+and the guarantees the dedup pass makes about the model's output (order, and that
+a commit_id the model invented or dropped can't corrupt the changelog).
+"""
+
+import json
+
+from hindsight_api.engine.token_encoding import count_tokens
+
+from hindsight_dev.generate_changelog import (
+    BATCH_TOKEN_BUDGET,
+    ChangelogEntry,
+    Commit,
+    analyze_commits_with_llm,
+    batch_commits,
+    deduplicate_entries,
+)
+
+
+def _max_commit_tokens(batch: list[Commit]) -> int:
+    """A batch may exceed the budget by its last commit, which is added before the check."""
+    return max(count_tokens(json.dumps({"commit_id": c.hash, "message": c.message})) for c in batch)
+
+
+def _commits(n: int, message: str = "fix(api): something user-facing happened here") -> list[Commit]:
+    return [Commit(hash=f"{i:09x}", message=f"{message} (#{i})") for i in range(n)]
+
+
+class _StubClient:
+    """Returns one entry per commit named in the prompt, and records the calls."""
+
+    def __init__(self, dedup_result: list[ChangelogEntry] | None = None):
+        self.prompts: list[str] = []
+        self._dedup_result = dedup_result
+        parse = self._parse
+        self.beta = type(
+            "_Beta",
+            (),
+            {"chat": type("_Chat", (), {"completions": type("_C", (), {"parse": staticmethod(parse)})()})()},
+        )()
+
+    def _parse(self, *, model, messages, response_format, max_completion_tokens):
+        prompt = messages[0]["content"]
+        self.prompts.append(prompt)
+        if prompt.startswith("These changelog entries were produced independently"):
+            entries = self._dedup_result if self._dedup_result is not None else []
+        else:
+            payload = json.loads(prompt.split("Commits:\n", 1)[1].split("\n\nFiles changed", 1)[0])
+            entries = [
+                ChangelogEntry(category="bugfix", summary=c["message"], commit_id=c["commit_id"]) for c in payload
+            ]
+        parsed = response_format(entries=entries)
+        return type("_R", (), {"choices": [type("_C", (), {"message": type("_M", (), {"parsed": parsed})()})()]})()
+
+
+def test_batches_stay_within_the_token_budget_and_preserve_order():
+    commits = _commits(200)
+    batches = batch_commits(commits)
+
+    assert len(batches) > 1
+    assert [c for batch in batches for c in batch] == commits
+    for batch in batches:
+        assert sum(count_tokens(json.dumps({"commit_id": c.hash, "message": c.message})) for c in batch) <= (
+            BATCH_TOKEN_BUDGET + _max_commit_tokens(batch)
+        )
+
+
+def test_the_budget_is_read_at_call_time(monkeypatch):
+    """A default bound at import would make raising the constant a silent no-op."""
+    commits = _commits(200)
+    assert len(batch_commits(commits)) > 1
+
+    monkeypatch.setattr("hindsight_dev.generate_changelog.BATCH_TOKEN_BUDGET", 10**9)
+    assert batch_commits(commits) == [commits]
+
+
+def test_a_single_commit_over_budget_gets_its_own_batch():
+    huge = Commit(hash="deadbeef1", message="x " * BATCH_TOKEN_BUDGET)
+    batches = batch_commits([huge, *_commits(2)])
+
+    assert batches[0] == [huge]
+    assert len(batches) > 1
+
+
+def test_a_small_release_is_one_call_with_no_dedup_pass():
+    client = _StubClient()
+    entries = analyze_commits_with_llm(client, "m", "0.1.0", _commits(3), file_diff="")
+
+    assert len(client.prompts) == 1
+    assert [e.commit_id for e in entries] == [c.hash for c in _commits(3)]
+
+
+def test_a_large_release_fans_out_and_deduplicates():
+    commits = _commits(200)
+    client = _StubClient(dedup_result=[])  # model drops everything -> fall back to the union
+    entries = analyze_commits_with_llm(client, "m", "0.1.0", commits, file_diff="")
+
+    analysis_prompts = [p for p in client.prompts if p.startswith("Analyze the following")]
+    assert len(analysis_prompts) == len(batch_commits(commits))
+    assert sum(p.startswith("These changelog entries") for p in client.prompts) == 1
+    # Every commit survives, in release order, despite the empty dedup response.
+    assert [e.commit_id for e in entries] == [c.hash for c in commits]
+
+
+def test_dedup_keeps_release_order_and_rejects_unknown_commit_ids():
+    commits = _commits(3)
+    entries = [ChangelogEntry(category="bugfix", summary=c.message, commit_id=c.hash) for c in reversed(commits)]
+    kept = [
+        ChangelogEntry(category="bugfix", summary="merged", commit_id=commits[2].hash),
+        ChangelogEntry(category="bugfix", summary="hallucinated", commit_id="ffffffff0"),
+        ChangelogEntry(category="bugfix", summary="kept", commit_id=commits[0].hash),
+    ]
+    client = _StubClient(dedup_result=kept)
+
+    result = deduplicate_entries(client, "m", entries, [c.hash for c in commits])
+
+    assert [e.commit_id for e in result] == [commits[0].hash, commits[2].hash]
+
+
+def test_dedup_drops_a_repeated_commit_id_before_asking_the_model():
+    commits = _commits(2)
+    duplicated = [
+        ChangelogEntry(category="bugfix", summary="first wording", commit_id=commits[0].hash),
+        ChangelogEntry(category="bugfix", summary="second wording", commit_id=commits[0].hash),
+        ChangelogEntry(category="bugfix", summary="other", commit_id=commits[1].hash),
+    ]
+    client = _StubClient(dedup_result=[])
+
+    result = deduplicate_entries(client, "m", duplicated, [c.hash for c in commits])
+
+    assert [e.summary for e in result] == ["first wording", "other"]
+    sent = json.loads(client.prompts[0].split("Entries:\n", 1)[1])
+    assert len(sent) == 2