diff --git a/hindsight-api-slim/hindsight_api/api/http.py b/hindsight-api-slim/hindsight_api/api/http.py index 95c7979748..57e4d7e246 100644 --- a/hindsight-api-slim/hindsight_api/api/http.py +++ b/hindsight-api-slim/hindsight_api/api/http.py @@ -7323,6 +7323,14 @@ async def api_list_operations( limit: int = Query(default=20, ge=1, le=100, description="Maximum number of operations to return"), offset: int = Query(default=0, ge=0, description="Number of operations to skip"), exclude_parents: bool = Query(default=False, description="Exclude parent batch operations from results"), + active_only: bool = Query( + default=False, + description=( + "Return only operations that are not yet terminal (status pending or processing). " + "The reported total counts the same filtered set, so one limit=1 request yields the " + "exact active backlog." + ), + ), request_context: RequestContext = Depends(get_request_context), ): """List async operations for a memory bank with optional filtering and pagination.""" @@ -7334,6 +7342,7 @@ async def api_list_operations( limit=limit, offset=offset, exclude_parents=exclude_parents, + active_only=active_only, request_context=request_context, ) return OperationsListResponse( diff --git a/hindsight-api-slim/hindsight_api/engine/memory_engine.py b/hindsight-api-slim/hindsight_api/engine/memory_engine.py index 845f255c0f..0e84a92dd3 100644 --- a/hindsight-api-slim/hindsight_api/engine/memory_engine.py +++ b/hindsight-api-slim/hindsight_api/engine/memory_engine.py @@ -19316,6 +19316,7 @@ async def list_operations( limit: int = 20, offset: int = 0, exclude_parents: bool = False, + active_only: bool = False, request_context: "RequestContext", ) -> dict[str, Any]: """List async operations for a bank with optional filtering and pagination. @@ -19327,6 +19328,9 @@ async def list_operations( limit: Maximum number of operations to return (default 20) offset: Number of operations to skip (default 0) exclude_parents: If True, exclude parent batch operations (is_parent=True in result_metadata) + active_only: If True, return only operations that are not yet terminal (status pending + or processing). Narrows the returned `total` too, so one `limit=1` request reports + the exact backlog depth. request_context: Request context for authentication Returns: @@ -19359,6 +19363,9 @@ async def list_operations( if exclude_parents: where_conditions.append("NOT (result_metadata::jsonb @> '{\"is_parent\": true}'::jsonb)") + if active_only: + where_conditions.append("status IN ('pending', 'processing')") + where_clause = " AND ".join(where_conditions) # Get total count (with filter) diff --git a/hindsight-api-slim/hindsight_api/mcp_tools.py b/hindsight-api-slim/hindsight_api/mcp_tools.py index 0cbffc367c..b67e513873 100644 --- a/hindsight-api-slim/hindsight_api/mcp_tools.py +++ b/hindsight-api-slim/hindsight_api/mcp_tools.py @@ -3927,11 +3927,12 @@ async def delete_document( def _register_list_operations(mcp: FastMCP, memory: MemoryEngine, config: MCPToolsConfig) -> None: """Register the list_operations tool.""" - async def _run(target_bank: str, status: str | None, limit: int) -> Any: + async def _run(target_bank: str, status: str | None, limit: int, active_only: bool) -> Any: result = await memory.list_operations( target_bank, status=status, limit=limit, + active_only=active_only, request_context=_get_request_context(config), ) return result @@ -3942,6 +3943,7 @@ async def _run(target_bank: str, status: str | None, limit: int) -> Any: async def list_operations( status: str | None = None, limit: int = 20, + active_only: bool = False, bank_id: str | None = None, ) -> str: """ @@ -3952,6 +3954,8 @@ async def list_operations( Args: status: Filter by status: 'pending', 'running', 'completed', 'failed', 'cancelled' limit: Maximum number of results (default: 20) + active_only: Only operations still pending or processing. The response 'total' + counts the same set, so it reports the whole backlog even at a small 'limit'. bank_id: Optional bank (defaults to session bank). Use for cross-bank operations. """ return await _run_tool( @@ -3959,7 +3963,7 @@ async def list_operations( bank_id=bank_id, as_json=True, action="listing operations", - run=lambda target_bank: _run(target_bank, status, limit), + run=lambda target_bank: _run(target_bank, status, limit, active_only), ) else: @@ -3968,6 +3972,7 @@ async def list_operations( async def list_operations( status: str | None = None, limit: int = 20, + active_only: bool = False, ) -> dict: """ List async operations for this memory bank. @@ -3977,13 +3982,15 @@ async def list_operations( Args: status: Filter by status: 'pending', 'running', 'completed', 'failed', 'cancelled' limit: Maximum number of results (default: 20) + active_only: Only operations still pending or processing. The response 'total' + counts the same set, so it reports the whole backlog even at a small 'limit'. """ return await _run_tool( config, bank_id=None, as_json=False, action="listing operations", - run=lambda target_bank: _run(target_bank, status, limit), + run=lambda target_bank: _run(target_bank, status, limit, active_only), ) diff --git a/hindsight-api-slim/tests/test_operation_status.py b/hindsight-api-slim/tests/test_operation_status.py index ff75a7881b..dd7b8e9f9a 100644 --- a/hindsight-api-slim/tests/test_operation_status.py +++ b/hindsight-api-slim/tests/test_operation_status.py @@ -43,17 +43,25 @@ async def _ensure_bank(pool, bank_id: str) -> None: ) -async def _insert_operation(pool, bank_id: str, status: str) -> str: - """Insert a test operation with the given status and return its ID.""" +async def _insert_operation( + pool, bank_id: str, status: str, operation_type: str = "retain", result_metadata: str = "{}" +) -> str: + """Insert a test operation with the given status and return its ID. + + ``result_metadata`` is the column a batch parent is marked in (``is_parent``), which is + what exclude_parents filters on. + """ op_id = uuid.uuid4() await pool.execute( """ - INSERT INTO async_operations (operation_id, bank_id, operation_type, status, task_payload) - VALUES ($1, $2, 'retain', $3, '{"test": true}'::jsonb) + INSERT INTO async_operations (operation_id, bank_id, status, operation_type, task_payload, result_metadata) + VALUES ($1, $2, $3, $4, '{"test": true}'::jsonb, $5::jsonb) """, op_id, bank_id, status, + operation_type, + result_metadata, ) return str(op_id) @@ -118,6 +126,63 @@ async def test_list_operations_filter_by_pending_excludes_processing(api_client, assert ops[0]["status"] == "pending" +@pytest.mark.asyncio +async def test_list_operations_active_only_totals_every_non_terminal_row(api_client, memory, test_bank_id): + """`total` counts every non-terminal row, not the one-row page it returns. + + A leaked terminal row would keep a client waiting on work that already finished. + """ + pool = memory._pool + await _ensure_bank(pool, test_bank_id) + + for status in ("pending", "pending", "pending", "processing", "processing"): + await _insert_operation(pool, test_bank_id, status) + for status in ("completed", "completed", "failed", "cancelled"): + await _insert_operation(pool, test_bank_id, status) + + response = await api_client.get( + f"/v1/default/banks/{test_bank_id}/operations", + params={"active_only": "true", "limit": 1}, + ) + assert response.status_code == 200 + body = response.json() + assert body["total"] == 5 + assert len(body["operations"]) == 1 + assert body["operations"][0]["status"] in ("pending", "processing") + + +@pytest.mark.asyncio +async def test_list_operations_active_only_conjoins_with_the_other_filters(api_client, memory, test_bank_id): + """active_only narrows the WHERE clause the other filters share; it never replaces one. + + Winning over status, type or exclude_parents would report a backlog nobody asked about. + """ + pool = memory._pool + await _ensure_bank(pool, test_bank_id) + + await _insert_operation(pool, test_bank_id, "pending", "batch_retain", '{"is_parent": true}') + pending_retain = await _insert_operation(pool, test_bank_id, "pending") + processing_retain = await _insert_operation(pool, test_bank_id, "processing") + await _insert_operation(pool, test_bank_id, "completed") + pending_consolidation = await _insert_operation(pool, test_bank_id, "pending", "consolidation") + await _insert_operation(pool, test_bank_id, "failed", "consolidation") + + url = f"/v1/default/banks/{test_bank_id}/operations" + narrowed = await api_client.get( + url, + params={"active_only": "true", "status": "pending", "type": "retain", "exclude_parents": "true"}, + ) + assert narrowed.status_code == 200 + body = narrowed.json() + assert body["total"] == 1 + assert [op["id"] for op in body["operations"]] == [pending_retain] + + active_leaves = await api_client.get(url, params={"active_only": "true", "exclude_parents": "true"}) + body = active_leaves.json() + assert body["total"] == 3 + assert {op["id"] for op in body["operations"]} == {pending_retain, processing_retain, pending_consolidation} + + @pytest.mark.asyncio async def test_get_operation_returns_processing_status(api_client, memory, test_bank_id): """GET /operations/{id} should return 'processing' status.""" diff --git a/hindsight-cli/src/api.rs b/hindsight-cli/src/api.rs index 818281ec0c..31512d486e 100644 --- a/hindsight-cli/src/api.rs +++ b/hindsight-cli/src/api.rs @@ -392,7 +392,7 @@ impl ApiClient { loop { let response = self .client - .list_operations(agent_id, None, None, None, None, None, None) + .list_operations(agent_id, None, None, None, None, None, None, None) .humanized() .await?; let ops = response.into_inner(); @@ -532,7 +532,7 @@ impl ApiClient { self.runtime.block_on(async { let response = self .client - .list_operations(agent_id, None, None, None, None, None, None) + .list_operations(agent_id, None, None, None, None, None, None, None) .humanized() .await?; let value = response.into_inner(); diff --git a/hindsight-clients/go/api/openapi.yaml b/hindsight-clients/go/api/openapi.yaml index de47683fab..be90cf4a4b 100644 --- a/hindsight-clients/go/api/openapi.yaml +++ b/hindsight-clients/go/api/openapi.yaml @@ -2820,6 +2820,21 @@ paths: title: Exclude Parents type: boolean style: form + - description: "Return only operations that are not yet terminal (status pending\ + \ or processing). The reported total counts the same filtered set, so one\ + \ limit=1 request yields the exact active backlog." + explode: true + in: query + name: active_only + required: false + schema: + default: false + description: "Return only operations that are not yet terminal (status pending\ + \ or processing). The reported total counts the same filtered set, so\ + \ one limit=1 request yields the exact active backlog." + title: Active Only + type: boolean + style: form - explode: false in: header name: authorization diff --git a/hindsight-clients/go/api_operations.go b/hindsight-clients/go/api_operations.go index cb291b03e9..b94d1cd031 100644 --- a/hindsight-clients/go/api_operations.go +++ b/hindsight-clients/go/api_operations.go @@ -423,6 +423,7 @@ type ApiListOperationsRequest struct { limit *int32 offset *int32 excludeParents *bool + activeOnly *bool authorization *string } @@ -456,6 +457,12 @@ func (r ApiListOperationsRequest) ExcludeParents(excludeParents bool) ApiListOpe return r } +// Return only operations that are not yet terminal (status pending or processing). The reported total counts the same filtered set, so one limit=1 request yields the exact active backlog. +func (r ApiListOperationsRequest) ActiveOnly(activeOnly bool) ApiListOperationsRequest { + r.activeOnly = &activeOnly + return r +} + func (r ApiListOperationsRequest) Authorization(authorization string) ApiListOperationsRequest { r.authorization = &authorization return r @@ -528,6 +535,12 @@ func (a *OperationsAPIService) ListOperationsExecute(r ApiListOperationsRequest) var defaultValue bool = false r.excludeParents = &defaultValue } + if r.activeOnly != nil { + parameterAddToHeaderOrQuery(localVarQueryParams, "active_only", r.activeOnly, "form", "") + } else { + var defaultValue bool = false + r.activeOnly = &defaultValue + } // to determine the Content-Type header localVarHTTPContentTypes := []string{} diff --git a/hindsight-clients/python/hindsight_client_api/api/operations_api.py b/hindsight-clients/python/hindsight_client_api/api/operations_api.py index 9b1e081439..58e9c47be2 100644 --- a/hindsight-clients/python/hindsight_client_api/api/operations_api.py +++ b/hindsight-clients/python/hindsight_client_api/api/operations_api.py @@ -948,6 +948,7 @@ async def list_operations( limit: Annotated[Optional[Annotated[int, Field(le=100, strict=True, ge=1)]], Field(description="Maximum number of operations to return")] = None, offset: Annotated[Optional[Annotated[int, Field(strict=True, ge=0)]], Field(description="Number of operations to skip")] = None, exclude_parents: Annotated[Optional[StrictBool], Field(description="Exclude parent batch operations from results")] = None, + active_only: Annotated[Optional[StrictBool], Field(description="Return only operations that are not yet terminal (status pending or processing). The reported total counts the same filtered set, so one limit=1 request yields the exact active backlog.")] = None, authorization: Optional[StrictStr] = None, _request_timeout: Union[ None, @@ -978,6 +979,8 @@ async def list_operations( :type offset: int :param exclude_parents: Exclude parent batch operations from results :type exclude_parents: bool + :param active_only: Return only operations that are not yet terminal (status pending or processing). The reported total counts the same filtered set, so one limit=1 request yields the exact active backlog. + :type active_only: bool :param authorization: :type authorization: str :param _request_timeout: timeout setting for this request. If one @@ -1009,6 +1012,7 @@ async def list_operations( limit=limit, offset=offset, exclude_parents=exclude_parents, + active_only=active_only, authorization=authorization, _request_auth=_request_auth, _content_type=_content_type, @@ -1041,6 +1045,7 @@ async def list_operations_with_http_info( limit: Annotated[Optional[Annotated[int, Field(le=100, strict=True, ge=1)]], Field(description="Maximum number of operations to return")] = None, offset: Annotated[Optional[Annotated[int, Field(strict=True, ge=0)]], Field(description="Number of operations to skip")] = None, exclude_parents: Annotated[Optional[StrictBool], Field(description="Exclude parent batch operations from results")] = None, + active_only: Annotated[Optional[StrictBool], Field(description="Return only operations that are not yet terminal (status pending or processing). The reported total counts the same filtered set, so one limit=1 request yields the exact active backlog.")] = None, authorization: Optional[StrictStr] = None, _request_timeout: Union[ None, @@ -1071,6 +1076,8 @@ async def list_operations_with_http_info( :type offset: int :param exclude_parents: Exclude parent batch operations from results :type exclude_parents: bool + :param active_only: Return only operations that are not yet terminal (status pending or processing). The reported total counts the same filtered set, so one limit=1 request yields the exact active backlog. + :type active_only: bool :param authorization: :type authorization: str :param _request_timeout: timeout setting for this request. If one @@ -1102,6 +1109,7 @@ async def list_operations_with_http_info( limit=limit, offset=offset, exclude_parents=exclude_parents, + active_only=active_only, authorization=authorization, _request_auth=_request_auth, _content_type=_content_type, @@ -1134,6 +1142,7 @@ async def list_operations_without_preload_content( limit: Annotated[Optional[Annotated[int, Field(le=100, strict=True, ge=1)]], Field(description="Maximum number of operations to return")] = None, offset: Annotated[Optional[Annotated[int, Field(strict=True, ge=0)]], Field(description="Number of operations to skip")] = None, exclude_parents: Annotated[Optional[StrictBool], Field(description="Exclude parent batch operations from results")] = None, + active_only: Annotated[Optional[StrictBool], Field(description="Return only operations that are not yet terminal (status pending or processing). The reported total counts the same filtered set, so one limit=1 request yields the exact active backlog.")] = None, authorization: Optional[StrictStr] = None, _request_timeout: Union[ None, @@ -1164,6 +1173,8 @@ async def list_operations_without_preload_content( :type offset: int :param exclude_parents: Exclude parent batch operations from results :type exclude_parents: bool + :param active_only: Return only operations that are not yet terminal (status pending or processing). The reported total counts the same filtered set, so one limit=1 request yields the exact active backlog. + :type active_only: bool :param authorization: :type authorization: str :param _request_timeout: timeout setting for this request. If one @@ -1195,6 +1206,7 @@ async def list_operations_without_preload_content( limit=limit, offset=offset, exclude_parents=exclude_parents, + active_only=active_only, authorization=authorization, _request_auth=_request_auth, _content_type=_content_type, @@ -1222,6 +1234,7 @@ def _list_operations_serialize( limit, offset, exclude_parents, + active_only, authorization, _request_auth, _content_type, @@ -1267,6 +1280,10 @@ def _list_operations_serialize( _query_params.append(('exclude_parents', exclude_parents)) + if active_only is not None: + + _query_params.append(('active_only', active_only)) + # process the header parameters if authorization is not None: _header_params['authorization'] = authorization diff --git a/hindsight-clients/typescript/generated/types.gen.ts b/hindsight-clients/typescript/generated/types.gen.ts index 5fbed49ca3..cd77ae9b74 100644 --- a/hindsight-clients/typescript/generated/types.gen.ts +++ b/hindsight-clients/typescript/generated/types.gen.ts @@ -8262,6 +8262,12 @@ export type ListOperationsData = { * Exclude parent batch operations from results */ exclude_parents?: boolean; + /** + * Active Only + * + * Return only operations that are not yet terminal (status pending or processing). The reported total counts the same filtered set, so one limit=1 request yields the exact active backlog. + */ + active_only?: boolean; }; url: "/v1/default/banks/{bank_id}/operations"; }; diff --git a/hindsight-docs/docs/developer/api/operations.mdx b/hindsight-docs/docs/developer/api/operations.mdx index d584f1aa3e..18660a2ebd 100644 --- a/hindsight-docs/docs/developer/api/operations.mdx +++ b/hindsight-docs/docs/developer/api/operations.mdx @@ -118,6 +118,9 @@ Query parameters: | `limit` | 1–100, default 20. | | `offset` | Pagination offset. | | `exclude_parents` | Exclude parent batch operations from results (large `retain_batch` calls create one parent + N children). | +| `active_only` | Only operations that have not reached a terminal state (`pending` or `processing`). | + +`total` counts the whole filtered set, not the returned page, so `?active_only=true&limit=1` is the cheapest exact answer to "how much work is this bank still doing?". Counting rows in a page instead saturates at `limit`, and asking once per non-terminal status takes two counts at two different instants — an operation that moves `pending` → `processing` between them is missing from both, and the pair sums to zero while the bank is still working. diff --git a/hindsight-docs/docs/developer/mcp-server.md b/hindsight-docs/docs/developer/mcp-server.md index ed929c9ac3..d9228bb4e6 100644 --- a/hindsight-docs/docs/developer/mcp-server.md +++ b/hindsight-docs/docs/developer/mcp-server.md @@ -564,6 +564,7 @@ List async operations (retain processing, mental model refresh, etc.) with optio |-----------|------|----------|-------------| | `status` | string | No | Filter by status: `pending`, `running`, `completed`, `failed`, `cancelled` | | `limit` | integer | No | Maximum number of results (default: 100) | +| `active_only` | boolean | No | Only operations that have not reached a terminal state (`pending` or `processing`). The reported `total` counts the same set, so it reports the whole active backlog even when `limit` returns fewer rows | --- diff --git a/hindsight-docs/static/openapi.json b/hindsight-docs/static/openapi.json index 9b2b45e0c7..5be6aec2ec 100644 --- a/hindsight-docs/static/openapi.json +++ b/hindsight-docs/static/openapi.json @@ -4058,6 +4058,18 @@ }, "description": "Exclude parent batch operations from results" }, + { + "name": "active_only", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "description": "Return only operations that are not yet terminal (status pending or processing). The reported total counts the same filtered set, so one limit=1 request yields the exact active backlog.", + "default": false, + "title": "Active Only" + }, + "description": "Return only operations that are not yet terminal (status pending or processing). The reported total counts the same filtered set, so one limit=1 request yields the exact active backlog." + }, { "name": "authorization", "in": "header", diff --git a/hindsight-integrations/coding-agents/src/core/hindsight.test.ts b/hindsight-integrations/coding-agents/src/core/hindsight.test.ts index 2f9dd3b16c..00000cdc99 100644 --- a/hindsight-integrations/coding-agents/src/core/hindsight.test.ts +++ b/hindsight-integrations/coding-agents/src/core/hindsight.test.ts @@ -1,6 +1,3 @@ -import { readdirSync, readFileSync } from "node:fs"; -import { join } from "node:path"; -import { fileURLToPath } from "node:url"; import { afterEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_MAX_PARALLEL_RETAINS, @@ -456,65 +453,46 @@ describe("HindsightClient credential refresh", () => { }); }); -/** - * The other half of the same invariant: forwarding the config is worthless if a write path skips - * the client method that SENDS it. One bank must consolidate into exactly one observation scope - * (#3564), and `retain()` is the only place that puts `observation_scopes` on the wire — so a - * second `/memories` POST anywhere would quietly consolidate under the server's `combined` - * default, splitting the repo's beliefs per tag combination again. No unit test would fail: the - * new path writes perfectly good memories. Hence a check over the whole source tree. - */ -describe("every memory write goes through the one call site that scopes it", () => { - const SRC = fileURLToPath(new URL("..", import.meta.url)); - - function sourceFiles(dir: string, prefix = ""): string[] { - return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { - const rel = prefix ? `${prefix}/${entry.name}` : entry.name; - if (entry.isDirectory()) - return entry.name === "e2e" ? [] : sourceFiles(join(dir, entry.name), rel); - return entry.name.endsWith(".ts") && !entry.name.includes(".test.") ? [rel] : []; - }); - } - - it("has no module addressing the memories endpoint except the client", () => { - const writers = sourceFiles(SRC).filter((rel) => - readFileSync(join(SRC, rel), "utf8").includes('"/memories') +describe("HindsightClient.activeOperations", () => { + it("distinguishes unavailable or invalid counts from a confirmed empty backlog", async () => { + vi.stubGlobal( + "fetch", + vi + .fn() + .mockResolvedValueOnce(jsonResponse(404, { detail: "unavailable" })) + .mockResolvedValueOnce(jsonResponse(200, {})) + .mockResolvedValueOnce(jsonResponse(200, { total: 0 })) ); - expect(writers).toEqual(["core/hindsight.ts"]); - }); - - it("keeps that call site inside retain(), with the scoping on the item it posts", () => { - const src = readFileSync(join(SRC, "core/hindsight.ts"), "utf8"); - expect(src.match(/bankUrl\("\/memories"\)/g)).toHaveLength(1); - // Everything between retain()'s signature and the POST is the body it builds; the scoping - // has to be set in there, not left to whatever the server defaults to. - const body = src.slice(src.indexOf("async retain("), src.indexOf('bankUrl("/memories")')); - // The scoping may be derived per document (see `per_source`), but it must still be set on the - // item here and still come from the configured value — not from a server default. - expect(body).toMatch(/observation_scopes: .*this\.observationScopes/); - }); -}); - -describe("every client-building entrypoint forwards observationScopes", () => { - const SRC = fileURLToPath(new URL("..", import.meta.url)); - - function sourceFiles(dir: string, prefix = ""): string[] { - return readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { - const rel = prefix ? `${prefix}/${entry.name}` : entry.name; - if (entry.isDirectory()) - return entry.name === "e2e" ? [] : sourceFiles(join(dir, entry.name), rel); - return entry.name.endsWith(".ts") && !entry.name.includes(".test.") ? [rel] : []; + const client = new HindsightClient({ apiUrl: "http://x", bank: "b" }); + await expect(client.activeOperations()).rejects.toThrow(); + await expect(client.activeOperations()).rejects.toThrow(); + expect(await client.activeOperations()).toBe(0); + }); + + /** The fixture is the adversary: `active_only` applies the server's own predicate, `status` + * filters, `total` counts the FILTERED set while only `limit` rows come back, and the 374 + * in-flight ops sit in whichever non-terminal status a single-status caller did NOT ask about. + * So a page count reads 1, an unfiltered total 1000, and the per-status pair it replaced 0. */ + it("reads the whole non-terminal backlog from one server-side count", async () => { + const fetchMock = vi.fn(async (url: string) => { + const q = new URL(url).searchParams; + const asked = q.get("status"); + const flight = asked === "pending" ? "processing" : "pending"; + const rows = Array.from({ length: 1000 }, (_, i) => (i < 374 ? flight : "completed")) + .filter((s) => q.get("active_only") !== "true" || s === "pending" || s === "processing") + .filter((s) => !asked || s === asked); + return jsonResponse(200, { + total: rows.length, + operations: rows.slice(0, Number(q.get("limit") ?? 20)).map((status) => ({ status })), + }); }); - } + vi.stubGlobal("fetch", fetchMock); - it("has no module that builds a client without passing cfg.observationScopes", () => { - const dropped = sourceFiles(SRC).filter((rel) => { - const src = readFileSync(join(SRC, rel), "utf8"); - // `makeClient({` is the hook/session-start seam: the ClientOpts are built there even though - // the constructor call itself is the injected default further up the file. - const buildsClient = src.includes("new HindsightClient({") || src.includes("makeClient({"); - return buildsClient && !src.includes("observationScopes:"); - }); - expect(dropped).toEqual([]); + const client = new HindsightClient({ apiUrl: "http://x", bank: "b" }); + expect(await client.activeOperations()).toBe(374); + expect(fetchMock).toHaveBeenCalledTimes(1); + const q = new URL(String(fetchMock.mock.calls[0][0])).searchParams; + expect(q.get("active_only")).toBe("true"); + expect(q.get("limit")).toBe("1"); }); }); diff --git a/hindsight-integrations/coding-agents/src/core/hindsight.ts b/hindsight-integrations/coding-agents/src/core/hindsight.ts index 99dfbbe781..08c3fd90bc 100644 --- a/hindsight-integrations/coding-agents/src/core/hindsight.ts +++ b/hindsight-integrations/coding-agents/src/core/hindsight.ts @@ -26,6 +26,9 @@ export interface KnowledgeNode { /** The page's EFFECTIVE refresh policy, on servers new enough to report it (#3572). Absent * everywhere else, which `seedPages()` reads as "unknown, leave it alone". */ trigger?: { tags_match?: string }; + /** Pages only: an in-scope memory was written since the page last read them, so the server + * already knows the document is behind its corpus. Absent on folders and on older servers. */ + is_stale?: boolean | null; children?: KnowledgeNode[]; } @@ -439,20 +442,15 @@ export class HindsightClient { await this.req("DELETE", this.bankUrl(`/documents/${encodeURIComponent(documentId)}`)); } - /** Count of operations still ACTIVE on this bank — the list includes terminal ops (completed/ - * failed/cancelled), so filter by status. Powers syncStatus's "extractions drained" check. */ + /** Count all pending/processing operations from the server's filtered total, not one page. + * An unavailable count rejects rather than inventing a confirmed zero. */ async activeOperations(): Promise { - const r = await this.req("GET", this.bankUrl("/operations")); - try { - const j = (await r.json()) as { - operations?: { status?: string }[]; - items?: { status?: string }[]; - }; - const ops = j.operations ?? j.items ?? []; - return ops.filter((o) => !TERMINAL.has((o?.status || "").toLowerCase())).length; - } catch { - return 0; - } + const r = await this.req("GET", this.bankUrl("/operations?active_only=true&limit=1")); + if (!r.ok) throw new Error(`Operation count unavailable: HTTP ${r.status}`); + const { total } = (await r.json()) as { total?: unknown }; + if (typeof total !== "number" || !Number.isSafeInteger(total) || total < 0) + throw new Error("Invalid operation count"); + return total; } /** @@ -557,7 +555,13 @@ export class HindsightClient { * here, from `searchKnowledgePages`, or from a `[[page:]]` link all resolve identically. */ async listPages(): Promise { - const items: { id: string; name: string; description?: string; folder?: string }[] = []; + const items: { + id: string; + name: string; + description?: string; + folder?: string; + is_stale?: boolean; + }[] = []; const walk = (nodes: KnowledgeNode[], folder?: string): void => { for (const n of nodes) { if (!n?.id || !n?.name) continue; @@ -567,6 +571,9 @@ export class HindsightClient { name: n.name, ...(n.description ? { description: n.description } : {}), ...(folder ? { folder } : {}), + // Absent rather than false where the server said nothing: "unknown" must not render + // as "current" in the roster this feeds. + ...(typeof n.is_stale === "boolean" ? { is_stale: n.is_stale } : {}), }); } if (n.children?.length) walk(n.children, n.kind === "folder" ? n.name : folder); diff --git a/hindsight-integrations/coding-agents/src/core/knowledge-injection.test.ts b/hindsight-integrations/coding-agents/src/core/knowledge-injection.test.ts index ddef75c50b..d5b98ea5f3 100644 --- a/hindsight-integrations/coding-agents/src/core/knowledge-injection.test.ts +++ b/hindsight-integrations/coding-agents/src/core/knowledge-injection.test.ts @@ -1,4 +1,5 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; +import { HindsightClient } from "./hindsight"; import { parsePageList, buildKnowledgePreamble, buildRosterRefresh } from "./knowledge-injection"; describe("parsePageList", () => { @@ -93,3 +94,42 @@ describe("buildRosterRefresh", () => { expect(out).not.toContain("Current Hindsight knowledge pages"); }); }); + +describe("freshness from the server tree", () => { + it("marks stale pages in both injected rosters and removes the mark after refresh", async () => { + let stale = true; + const fetchSpy = vi.spyOn(globalThis, "fetch").mockImplementation(async () => + Response.json({ + roots: [ + { id: "p1", kind: "page", name: "Component map", is_stale: stale }, + { id: "p2", kind: "page", name: "Core concepts", is_stale: false }, + { id: "p3", kind: "page", name: "Key decisions" }, + { id: "p4", kind: "page", name: "Conventions", is_stale: "yes" }, + { + id: "folder", + kind: "folder", + name: "Initiatives", + children: [{ id: "p5", kind: "page", name: "Retry backoff", is_stale: stale }], + }, + ], + }) + ); + const client = new HindsightClient({ apiUrl: "http://server", bank: "repo" }); + try { + const pages = parsePageList(await client.listPages()); + for (const render of [buildKnowledgePreamble, buildRosterRefresh]) { + const output = render(pages); + expect(output).toContain("- Component map (p1) — STALE"); + expect(output).toContain("- Retry backoff (p5) — STALE"); + expect(output).not.toMatch(/(?:Core concepts|Key decisions|Conventions).*STALE/); + } + stale = false; + const refreshed = parsePageList(await client.listPages()); + for (const render of [buildKnowledgePreamble, buildRosterRefresh]) { + expect(render(refreshed)).not.toContain("STALE"); + } + } finally { + fetchSpy.mockRestore(); + } + }); +}); diff --git a/hindsight-integrations/coding-agents/src/core/knowledge-injection.ts b/hindsight-integrations/coding-agents/src/core/knowledge-injection.ts index ac18180fe5..07dd0b1108 100644 --- a/hindsight-integrations/coding-agents/src/core/knowledge-injection.ts +++ b/hindsight-integrations/coding-agents/src/core/knowledge-injection.ts @@ -1,6 +1,9 @@ export interface PageRef { id: string; title: string; + /** The server's own staleness verdict: in-scope memories written since the page was rebuilt. + * Undefined means the server did not say — rendered as nothing, never as "current". */ + stale?: boolean; } /** Defensive parse of HindsightClient.listPages() ({items:[{id,name}]}, flattened from the @@ -13,13 +16,27 @@ export function parsePageList(raw: unknown): PageRef[] { for (const it of items) { const id = (it as { id?: unknown })?.id; const name = (it as { name?: unknown })?.name; - if (typeof id === "string" && typeof name === "string") out.push({ id, title: name }); + if (typeof id === "string" && typeof name === "string") { + const stale = (it as { is_stale?: unknown })?.is_stale; + out.push({ id, title: name, ...(typeof stale === "boolean" ? { stale } : {}) }); + } } return out; } +/** The staleness legend, emitted only when the roster actually flags something — a standing + * caveat on every page in every session would be read as boilerplate and stop meaning anything. */ +const STALE_LEGEND = + "Pages marked STALE have had memories written in their scope since they were last rebuilt: the " + + "server already knows the page is behind this repository. Read them for orientation, and verify " + + "any specific claim — counts, inventories, file lists — against the code before relying on it."; + function roster(pages: PageRef[]): string { - return pages.map((p) => `- ${p.title} (${p.id})`).join("\n"); + return pages.map((p) => `- ${p.title} (${p.id})${p.stale ? " — STALE" : ""}`).join("\n"); +} + +function rosterWithLegend(pages: PageRef[]): string { + return pages.some((p) => p.stale) ? `${roster(pages)}\n${STALE_LEGEND}` : roster(pages); } /** @@ -71,7 +88,7 @@ function toolGuide(opts?: ToolGuideOpts): string { /** SessionStart: teach the whole tool suite + when to use each, and list what pages exist. Empty-state aware. */ export function buildKnowledgePreamble(pages: PageRef[], opts?: ToolGuideOpts): string { const body = pages.length - ? `Knowledge pages currently in this repository:\n${roster(pages)}` + ? `Knowledge pages currently in this repository:\n${rosterWithLegend(pages)}` : "No knowledge pages yet — Hindsight is still learning this repo; they'll appear as it processes."; return ( "\n" + @@ -95,7 +112,7 @@ export function buildKnowledgePreamble(pages: PageRef[], opts?: ToolGuideOpts): */ export function buildRosterRefresh(pages: PageRef[], opts?: ToolGuideOpts): string { const rosterBlock = pages.length - ? `Current Hindsight knowledge pages (may have changed):\n${roster(pages)}\n` + ? `Current Hindsight knowledge pages (may have changed):\n${rosterWithLegend(pages)}\n` : ""; return ( "\n" + diff --git a/skills/hindsight-docs/references/developer/api/operations.md b/skills/hindsight-docs/references/developer/api/operations.md index 9a0549c6c3..212a3b4f5d 100644 --- a/skills/hindsight-docs/references/developer/api/operations.md +++ b/skills/hindsight-docs/references/developer/api/operations.md @@ -106,6 +106,9 @@ Query parameters: | `limit` | 1–100, default 20. | | `offset` | Pagination offset. | | `exclude_parents` | Exclude parent batch operations from results (large `retain_batch` calls create one parent + N children). | +| `active_only` | Only operations that have not reached a terminal state (`pending` or `processing`). | + +`total` counts the whole filtered set, not the returned page, so `?active_only=true&limit=1` is the cheapest exact answer to "how much work is this bank still doing?". Counting rows in a page instead saturates at `limit`, and asking once per non-terminal status takes two counts at two different instants — an operation that moves `pending` → `processing` between them is missing from both, and the pair sums to zero while the bank is still working. ### Python diff --git a/skills/hindsight-docs/references/developer/mcp-server.md b/skills/hindsight-docs/references/developer/mcp-server.md index ed929c9ac3..d9228bb4e6 100644 --- a/skills/hindsight-docs/references/developer/mcp-server.md +++ b/skills/hindsight-docs/references/developer/mcp-server.md @@ -564,6 +564,7 @@ List async operations (retain processing, mental model refresh, etc.) with optio |-----------|------|----------|-------------| | `status` | string | No | Filter by status: `pending`, `running`, `completed`, `failed`, `cancelled` | | `limit` | integer | No | Maximum number of results (default: 100) | +| `active_only` | boolean | No | Only operations that have not reached a terminal state (`pending` or `processing`). The reported `total` counts the same set, so it reports the whole active backlog even when `limit` returns fewer rows | --- diff --git a/skills/hindsight-docs/references/openapi.json b/skills/hindsight-docs/references/openapi.json index 9b2b45e0c7..5be6aec2ec 100644 --- a/skills/hindsight-docs/references/openapi.json +++ b/skills/hindsight-docs/references/openapi.json @@ -4058,6 +4058,18 @@ }, "description": "Exclude parent batch operations from results" }, + { + "name": "active_only", + "in": "query", + "required": false, + "schema": { + "type": "boolean", + "description": "Return only operations that are not yet terminal (status pending or processing). The reported total counts the same filtered set, so one limit=1 request yields the exact active backlog.", + "default": false, + "title": "Active Only" + }, + "description": "Return only operations that are not yet terminal (status pending or processing). The reported total counts the same filtered set, so one limit=1 request yields the exact active backlog." + }, { "name": "authorization", "in": "header",