feat(HNT-2108): add article extraction handler with Corpus API client

packages/crawl-common/package.json

@@ -16,13 +16,15 @@
     "lint": "eslint src/"
   },
   "devDependencies": {
+    "@types/node": "^24.12.0",
     "eslint": "^10.0.0",
     "tsup": "^8.0.0",
     "typescript": "^5.8.0",
     "vitest": "^4.1.2"
   },
   "dependencies": {
     "is-network-error": "^1.3.1",
+    "jose": "^6.2.2",
     "p-retry": "^8.0.0"
   }
 }

packages/crawl-common/src/corpus-api/client.integration.ts

@@ -0,0 +1,179 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import {
+  initCorpusApiClient,
+  updateApprovedCorpusItem,
+  CorpusApiError,
+  RETRY_MAX_TIMEOUT_MS,
+  TOKEN_REFRESH_WINDOW_MS,
+} from './client.js';
+import {
+  CLIENT_OPTS,
+  UPDATE_APPROVED_CORPUS_ITEM_INPUT,
+  UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY,
+  mockResponse,
+} from './test-helpers.js';
+
+/** Extract the Bearer token from a fetch mock call. */
+function extractToken(
+  fetchMock: ReturnType<typeof vi.fn>,
+  callIndex: number,
+): string {
+  const headers = fetchMock.mock.calls[callIndex][1].headers as Record<
+    string,
+    string
+  >;
+  return headers.authorization.replace('Bearer ', '');
+}
+
+/**
+ * Integration tests for retry and JWT refresh behavior.
+ * Uses fake timers to avoid real retry delays and to
+ * control token expiry.
+ */
+describe('Corpus API integration', () => {
+  const fetchMock = vi.fn<typeof fetch>();
+
+  afterEach(() => {
+    fetchMock.mockReset();
+    vi.useRealTimers();
+    vi.unstubAllGlobals();
+  });
+
+  describe('retry', () => {
+    beforeEach(async () => {
+      vi.stubGlobal('fetch', fetchMock);
+      await initCorpusApiClient(CLIENT_OPTS);
+      // Prime the token cache with real timers so jose's
+      // async Web Crypto signing completes. Retry tests
+      // then use the cached token under fake timers.
+      fetchMock.mockResolvedValueOnce(
+        mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+      );
+      await updateApprovedCorpusItem(UPDATE_APPROVED_CORPUS_ITEM_INPUT);
+      fetchMock.mockReset();
+      vi.useFakeTimers();
+    });
+
+    it('retries on 5xx and succeeds', async () => {
+      fetchMock
+        .mockResolvedValueOnce(mockResponse({ error: 'server' }, 503))
+        .mockResolvedValueOnce(
+          mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+        );
+
+      const promise = updateApprovedCorpusItem(
+        UPDATE_APPROVED_CORPUS_ITEM_INPUT,
+      );
+      await vi.advanceTimersByTimeAsync(RETRY_MAX_TIMEOUT_MS);
+      const result = await promise;
+
+      expect(result.externalId).toBe(
+        UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY.data.updateApprovedCorpusItem
+          .externalId,
+      );
+      expect(fetchMock).toHaveBeenCalledTimes(2);
+    });
+
+    it('retries on network error and succeeds', async () => {
+      fetchMock
+        .mockRejectedValueOnce(new TypeError('fetch failed'))
+        .mockResolvedValueOnce(
+          mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+        );
+
+      const promise = updateApprovedCorpusItem(
+        UPDATE_APPROVED_CORPUS_ITEM_INPUT,
+      );
+      await vi.advanceTimersByTimeAsync(RETRY_MAX_TIMEOUT_MS);
+      const result = await promise;
+
+      expect(result.externalId).toBe(
+        UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY.data.updateApprovedCorpusItem
+          .externalId,
+      );
+      expect(fetchMock).toHaveBeenCalledTimes(2);
+    });
+
+    it('stops after max retries', async () => {
+      fetchMock.mockResolvedValue(mockResponse({ error: 'down' }, 500));
+
+      const promise = updateApprovedCorpusItem(
+        UPDATE_APPROVED_CORPUS_ITEM_INPUT,
+      ).catch((e) => e);
+      await vi.advanceTimersByTimeAsync(RETRY_MAX_TIMEOUT_MS * 5);
+      const err = await promise;
+
+      expect(err).toBeInstanceOf(CorpusApiError);
+      // 1 initial + 4 retries = 5 total.
+      expect(fetchMock).toHaveBeenCalledTimes(5);
+    });
+  });
+
+  describe('token refresh', () => {
+    beforeEach(async () => {
+      vi.stubGlobal('fetch', fetchMock);
+      await initCorpusApiClient(CLIENT_OPTS);
+      vi.useFakeTimers();
+    });
+
+    it('issues a new token after the refresh window', async () => {
+      // First call primes the token cache.
+      fetchMock.mockResolvedValueOnce(
+        mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+      );
+      const promise1 = updateApprovedCorpusItem(
+        UPDATE_APPROVED_CORPUS_ITEM_INPUT,
+      );
+      await vi.advanceTimersByTimeAsync(0);
+      await promise1;
+
+      const token1 = extractToken(fetchMock, 0);
+
+      // Advance past the refresh window.
+      await vi.advanceTimersByTimeAsync(TOKEN_REFRESH_WINDOW_MS + 1_000);
+
+      // Second call should issue a new token.
+      fetchMock.mockResolvedValueOnce(
+        mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+      );
+      const promise2 = updateApprovedCorpusItem(
+        UPDATE_APPROVED_CORPUS_ITEM_INPUT,
+      );
+      await vi.advanceTimersByTimeAsync(0);
+      await promise2;
+
+      const token2 = extractToken(fetchMock, 1);
+      expect(token2).not.toBe(token1);
+    });
+
+    it('reuses token within the refresh window', async () => {
+      fetchMock
+        .mockResolvedValueOnce(
+          mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+        )
+        .mockResolvedValueOnce(
+          mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+        );
+
+      // First call primes the cache.
+      const promise1 = updateApprovedCorpusItem(
+        UPDATE_APPROVED_CORPUS_ITEM_INPUT,
+      );
+      await vi.advanceTimersByTimeAsync(0);
+      await promise1;
+
+      // Advance to the halfway point of the refresh window.
+      await vi.advanceTimersByTimeAsync(TOKEN_REFRESH_WINDOW_MS / 2);
+
+      const promise2 = updateApprovedCorpusItem(
+        UPDATE_APPROVED_CORPUS_ITEM_INPUT,
+      );
+      await vi.advanceTimersByTimeAsync(0);
+      await promise2;
+
+      const token1 = extractToken(fetchMock, 0);
+      const token2 = extractToken(fetchMock, 1);
+      expect(token1).toBe(token2);
+    });
+  });
+});

packages/crawl-common/src/corpus-api/client.spec.ts

@@ -0,0 +1,167 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import {
+  initCorpusApiClient,
+  updateApprovedCorpusItem,
+  CorpusApiError,
+} from './client.js';
+import {
+  TEST_JWK,
+  CLIENT_OPTS,
+  UPDATE_APPROVED_CORPUS_ITEM_INPUT,
+  UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY,
+  mockResponse,
+} from './test-helpers.js';
+
+let fetchMock: ReturnType<typeof vi.fn>;
+
+describe('corpus-api client', () => {
+  beforeEach(async () => {
+    fetchMock = vi.fn();
+    vi.stubGlobal('fetch', fetchMock);
+    await initCorpusApiClient(CLIENT_OPTS);
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+    vi.restoreAllMocks();
+  });
+
+  describe('request', () => {
+    it('sends a GraphQL mutation with JWT auth', async () => {
+      fetchMock.mockResolvedValueOnce(
+        mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+      );
+
+      await updateApprovedCorpusItem(UPDATE_APPROVED_CORPUS_ITEM_INPUT);
+
+      expect(fetchMock).toHaveBeenCalledOnce();
+      const [url, init] = fetchMock.mock.calls[0] as [string, RequestInit];
+      expect(url).toBe(CLIENT_OPTS.endpoint);
+      expect(init.method).toBe('POST');
+
+      const headers = init.headers as Record<string, string>;
+      expect(headers['content-type']).toBe('application/json');
+      expect(headers.authorization).toMatch(/^Bearer eyJ/);
+      expect(headers['apollographql-client-name']).toBe('hnt-content');
+    });
+
+    it('sends the mutation variables', async () => {
+      fetchMock.mockResolvedValueOnce(
+        mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+      );
+
+      await updateApprovedCorpusItem(UPDATE_APPROVED_CORPUS_ITEM_INPUT);
+
+      const body = JSON.parse(fetchMock.mock.calls[0][1].body as string);
+      expect(body.variables.data.externalId).toBe(
+        UPDATE_APPROVED_CORPUS_ITEM_INPUT.externalId,
+      );
+      expect(body.variables.data.title).toBe(
+        UPDATE_APPROVED_CORPUS_ITEM_INPUT.title,
+      );
+      expect(body.query).toContain('updateApprovedCorpusItem');
+    });
+  });
+
+  describe('response', () => {
+    it('returns the mutation result on success', async () => {
+      fetchMock.mockResolvedValueOnce(
+        mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+      );
+
+      const result = await updateApprovedCorpusItem(
+        UPDATE_APPROVED_CORPUS_ITEM_INPUT,
+      );
+
+      const expected =
+        UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY.data.updateApprovedCorpusItem;
+      expect(result.externalId).toBe(expected.externalId);
+      expect(result.title).toBe(expected.title);
+    });
+
+    it.each([
+      {
+        scenario: 'GraphQL errors',
+        body: { errors: [{ message: 'Item not found' }] },
+        status: 200,
+      },
+      { scenario: 'null data', body: { data: null }, status: 200 },
+      { scenario: '4xx', body: { error: 'bad request' }, status: 400 },
+    ])(
+      'throws CorpusApiError on $scenario without retrying',
+      async ({ body, status }) => {
+        fetchMock.mockResolvedValueOnce(mockResponse(body, status));
+
+        await expect(
+          updateApprovedCorpusItem(UPDATE_APPROVED_CORPUS_ITEM_INPUT),
+        ).rejects.toThrow(CorpusApiError);
+        expect(fetchMock).toHaveBeenCalledOnce();
+      },
+    );
+  });
+
+  describe('jwt', () => {
+    it('caches the JWT token across calls', async () => {
+      fetchMock
+        .mockResolvedValueOnce(
+          mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+        )
+        .mockResolvedValueOnce(
+          mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+        );
+
+      await updateApprovedCorpusItem(UPDATE_APPROVED_CORPUS_ITEM_INPUT);
+      await updateApprovedCorpusItem(UPDATE_APPROVED_CORPUS_ITEM_INPUT);
+
+      // Both calls should use the same token.
+      const token1 = (
+        fetchMock.mock.calls[0][1].headers as Record<string, string>
+      ).authorization;
+      const token2 = (
+        fetchMock.mock.calls[1][1].headers as Record<string, string>
+      ).authorization;
+      expect(token1).toBe(token2);
+    });
+
+    it('handles the {"keys": [...]} wrapper format', async () => {
+      const wrapped = JSON.stringify({
+        keys: [JSON.parse(TEST_JWK)],
+      });
+      await initCorpusApiClient({
+        ...CLIENT_OPTS,
+        jwkJson: wrapped,
+      });
+      fetchMock.mockResolvedValueOnce(
+        mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+      );
+
+      const result = await updateApprovedCorpusItem(
+        UPDATE_APPROVED_CORPUS_ITEM_INPUT,
+      );
+
+      expect(result.externalId).toBe(
+        UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY.data.updateApprovedCorpusItem
+          .externalId,
+      );
+    });
+
+    it('includes kid in the JWT header', async () => {
+      fetchMock.mockResolvedValueOnce(
+        mockResponse(UPDATE_APPROVED_CORPUS_ITEM_SUCCESS_BODY),
+      );
+
+      await updateApprovedCorpusItem(UPDATE_APPROVED_CORPUS_ITEM_INPUT);
+
+      const authHeader = (
+        fetchMock.mock.calls[0][1].headers as Record<string, string>
+      ).authorization;
+      const token = authHeader.replace('Bearer ', '');
+      // Decode the JWT header (first segment, base64url).
+      const header = JSON.parse(
+        Buffer.from(token.split('.')[0], 'base64url').toString(),
+      );
+      expect(header.alg).toBe('RS256');
+      expect(header.kid).toBe('test-kid');
+    });
+  });
+});