feat: add server-side pagination to API endpoints

cypress/support/commands.js

@@ -134,18 +134,17 @@ Cypress.Commands.add('getTestRepoId', () => {
         `GET ${url} returned status ${res.status}: ${JSON.stringify(res.body).slice(0, 500)}`,
       );
     }
-    if (!Array.isArray(res.body)) {
+    const repos = res.body.data ?? res.body;
+    if (!Array.isArray(repos)) {
       throw new Error(
-        `GET ${url} returned non-array (${typeof res.body}): ${JSON.stringify(res.body).slice(0, 500)}`,
+        `GET ${url} returned unexpected shape: ${JSON.stringify(res.body).slice(0, 500)}`,
       );
     }
     const gitServerTarget = Cypress.env('GIT_SERVER_TARGET') || 'git-server:8443';
-    const repo = res.body.find(
-      (r) => r.url === `https://${gitServerTarget}/test-owner/test-repo.git`,
-    );
+    const repo = repos.find((r) => r.url === `https://${gitServerTarget}/test-owner/test-repo.git`);
     if (!repo) {
       throw new Error(
-        `test-owner/test-repo not found in database. Repos: ${res.body.map((r) => r.url).join(', ')}`,
+        `test-owner/test-repo not found in database. Repos: ${repos.map((r) => r.url).join(', ')}`,
       );
     }
     return cy.wrap(repo._id);
@@ -159,8 +158,9 @@ Cypress.Commands.add('cleanupTestRepos', () => {
       url: `${getApiBaseUrl()}/api/v1/repo`,
       failOnStatusCode: false,
     }).then((res) => {
-      if (res.status !== 200 || !Array.isArray(res.body)) return;
-      const testRepos = res.body.filter((r) => r.project === 'cypress-test');
+      const repos = res.body.data ?? res.body;
+      if (res.status !== 200 || !Array.isArray(repos)) return;
+      const testRepos = repos.filter((r) => r.project === 'cypress-test');
       testRepos.forEach((repo) => {
         cy.request({
           method: 'DELETE',

packages/git-proxy-cli/index.ts

@@ -106,12 +106,12 @@ async function getGitPushes(filters: Partial<PushQuery>) {
 
   try {
     const cookies = JSON.parse(fs.readFileSync(GIT_PROXY_COOKIE_FILE, 'utf8'));
-    const { data } = await axios.get<Action[]>(`${baseUrl}/api/v1/push/`, {
+    const response = await axios.get<{ data: Action[]; total: number }>(`${baseUrl}/api/v1/push/`, {
       headers: { Cookie: cookies },
       params: filters,
     });
 
-    const records = data.map((push: Action) => {
+    const records = response.data.data.map((push: Action) => {
       const {
         id,
         repo,

src/db/file/helper.ts

@@ -15,8 +15,40 @@
  */
 
 import { existsSync, mkdirSync } from 'fs';
+import Datastore from '@seald-io/nedb';
+import { PaginatedResult } from '../types';
 
 export const getSessionStore = (): undefined => undefined;
 export const initializeFolders = () => {
   if (!existsSync('./.data/db')) mkdirSync('./.data/db', { recursive: true });
 };
+
+export const paginatedFind = <T>(
+  db: Datastore,
+  filter: Record<string, unknown>,
+  sort: Record<string, 1 | -1>,
+  skip: number,
+  limit: number,
+): Promise<PaginatedResult<T>> => {
+  const countPromise = new Promise<number>((resolve, reject) => {
+    db.count(filter as any, (err: Error | null, count: number) => {
+      /* istanbul ignore if */
+      if (err) reject(err);
+      else resolve(count);
+    });
+  });
+
+  const dataPromise = new Promise<T[]>((resolve, reject) => {
+    db.find(filter as any)
+      .sort(sort)
+      .skip(skip)
+      .limit(limit)
+      .exec((err: Error | null, docs: any[]) => {
+        /* istanbul ignore if */
+        if (err) reject(err);
+        else resolve(docs);
+      });
+  });
+
+  return Promise.all([dataPromise, countPromise]).then(([data, total]) => ({ data, total }));
+};

src/db/file/pushes.ts

@@ -17,8 +17,9 @@
 import _ from 'lodash';
 import Datastore from '@seald-io/nedb';
 import { Action } from '../../proxy/actions/Action';
-import { toClass } from '../helper';
-import { PushQuery } from '../types';
+import { toClass, buildSearchFilter, buildSort } from '../helper';
+import { paginatedFind } from './helper';
+import { PaginatedResult, PaginationOptions, PushQuery } from '../types';
 import { CompletedAttestation, Rejection } from '../../proxy/processors/types';
 import { handleErrorAndLog } from '../../utils/errors';
 
@@ -49,25 +50,27 @@ const defaultPushQuery: Partial<PushQuery> = {
   type: 'push',
 };
 
-export const getPushes = (query: Partial<PushQuery>): Promise<Action[]> => {
+export const getPushes = (
+  query: Partial<PushQuery>,
+  pagination?: PaginationOptions,
+): Promise<PaginatedResult<Action>> => {
   if (!query) query = defaultPushQuery;
-  return new Promise((resolve, reject) => {
-    db.find(query)
-      .sort({ timestamp: -1 })
-      .exec((err, docs) => {
-        // ignore for code coverage as neDB rarely returns errors even for an invalid query
-        /* istanbul ignore if */
-        if (err) {
-          reject(err);
-        } else {
-          resolve(
-            _.chain(docs)
-              .map((x) => toClass(x, Action.prototype))
-              .value(),
-          );
-        }
-      });
-  });
+
+  const baseQuery = buildSearchFilter(
+    { ...query },
+    ['repo', 'branch', 'commitTo', 'user'],
+    pagination?.search,
+  );
+  const sort = buildSort(pagination, 'timestamp', -1);
+  const skip = pagination?.skip ?? 0;
+  const limit = pagination?.limit ?? 0;
+
+  return paginatedFind<Action>(db, baseQuery, sort, skip, limit).then(({ data, total }) => ({
+    data: _.chain(data)
+      .map((x) => toClass(x, Action.prototype))
+      .value(),
+    total,
+  }));
 };
 
 export const getPush = async (id: string): Promise<Action | null> => {
@@ -157,5 +160,5 @@ export const cancel = async (id: string): Promise<{ message: string }> => {
   action.canceled = true;
   action.rejected = false;
   await writeAudit(action);
-  return { message: `cancel ${id}` };
+  return { message: `canceled ${id}` };
 };

src/db/file/repo.ts

@@ -17,8 +17,9 @@
 import Datastore from '@seald-io/nedb';
 import _ from 'lodash';
 
-import { Repo, RepoQuery } from '../types';
-import { toClass } from '../helper';
+import { PaginatedResult, PaginationOptions, Repo, RepoQuery } from '../types';
+import { toClass, buildSearchFilter, buildSort } from '../helper';
+import { paginatedFind } from './helper';
 import { handleErrorAndLog } from '../../utils/errors';
 
 const COMPACTION_INTERVAL = 1000 * 60 * 60 * 24; // once per day
@@ -42,25 +43,25 @@ try {
 db.ensureIndex({ fieldName: 'name', unique: false });
 db.setAutocompactionInterval(COMPACTION_INTERVAL);
 
-export const getRepos = async (query: Partial<RepoQuery> = {}): Promise<Repo[]> => {
+export const getRepos = async (
+  query: Partial<RepoQuery> = {},
+  pagination?: PaginationOptions,
+): Promise<PaginatedResult<Repo>> => {
   if (query?.name) {
     query.name = query.name.toLowerCase();
   }
-  return new Promise<Repo[]>((resolve, reject) => {
-    db.find(query, (err: Error, docs: Repo[]) => {
-      // ignore for code coverage as neDB rarely returns errors even for an invalid query
-      /* istanbul ignore if */
-      if (err) {
-        reject(err);
-      } else {
-        resolve(
-          _.chain(docs)
-            .map((x) => toClass(x, Repo.prototype))
-            .value(),
-        );
-      }
-    });
-  });
+
+  const baseQuery = buildSearchFilter({ ...query }, ['name', 'project', 'url'], pagination?.search);
+  const sort = buildSort(pagination, 'name', 1);
+  const skip = pagination?.skip ?? 0;
+  const limit = pagination?.limit ?? 0;
+
+  return paginatedFind<Repo>(db, baseQuery, sort, skip, limit).then(({ data, total }) => ({
+    data: _.chain(data)
+      .map((x) => toClass(x, Repo.prototype))
+      .value(),
+    total,
+  }));
 };
 
 export const getRepo = async (name: string): Promise<Repo | null> => {

src/db/file/users.ts

@@ -17,8 +17,10 @@
 import fs from 'fs';
 import Datastore from '@seald-io/nedb';
 
-import { User, UserQuery } from '../types';
+import { PaginatedResult, PaginationOptions, User, UserQuery } from '../types';
 import { handleErrorAndLog } from '../../utils/errors';
+import { buildSearchFilter, buildSort } from '../helper';
+import { paginatedFind } from './helper';
 
 const COMPACTION_INTERVAL = 1000 * 60 * 60 * 24; // once per day
 
@@ -180,22 +182,25 @@ export const updateUser = (user: Partial<User>): Promise<void> => {
   });
 };
 
-export const getUsers = (query: Partial<UserQuery> = {}): Promise<User[]> => {
+export const getUsers = (
+  query: Partial<UserQuery> = {},
+  pagination?: PaginationOptions,
+): Promise<PaginatedResult<User>> => {
   if (query.username) {
     query.username = query.username.toLowerCase();
   }
   if (query.email) {
     query.email = query.email.toLowerCase();
   }
-  return new Promise<User[]>((resolve, reject) => {
-    db.find(query, (err: Error, docs: User[]) => {
-      // ignore for code coverage as neDB rarely returns errors even for an invalid query
-      /* istanbul ignore if */
-      if (err) {
-        reject(err);
-      } else {
-        resolve(docs);
-      }
-    });
-  });
+
+  const baseQuery = buildSearchFilter(
+    { ...query },
+    ['username', 'displayName', 'email', 'gitAccount'],
+    pagination?.search,
+  );
+  const sort = buildSort(pagination, 'username', 1);
+  const skip = pagination?.skip ?? 0;
+  const limit = pagination?.limit ?? 0;
+
+  return paginatedFind<User>(db, baseQuery, sort, skip, limit);
 };

src/db/helper.ts

@@ -14,6 +14,31 @@
  * limitations under the License.
  */
 
+import { PaginationOptions } from './types';
+
+export const escapeRegex = (s: string): string => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+
+export const buildSearchFilter = (
+  baseQuery: Record<string, unknown>,
+  searchFields: string[],
+  search?: string,
+): Record<string, unknown> => {
+  if (!search) return baseQuery;
+  const regex = new RegExp(escapeRegex(search), 'i');
+  return { ...baseQuery, $or: searchFields.map((f) => ({ [f]: regex })) };
+};
+
+export const buildSort = (
+  pagination: PaginationOptions | undefined,
+  defaultField: string,
+  defaultDir: 1 | -1,
+): Record<string, 1 | -1> => {
+  const field = pagination?.sortBy ?? defaultField;
+  const dir =
+    pagination?.sortOrder === 'asc' ? 1 : pagination?.sortOrder === 'desc' ? -1 : defaultDir;
+  return { [field]: dir };
+};
+
 export const toClass = function <T, U>(obj: T, proto: U): U {
   const out = JSON.parse(JSON.stringify(obj));
   out.__proto__ = proto;

src/db/index.ts

@@ -15,7 +15,16 @@
  */
 
 import { AuthorisedRepo } from '../config/generated/config';
-import { PushQuery, Repo, RepoQuery, Sink, User, UserQuery } from './types';
+import {
+  PaginatedResult,
+  PaginationOptions,
+  PushQuery,
+  Repo,
+  RepoQuery,
+  Sink,
+  User,
+  UserQuery,
+} from './types';
 import * as bcrypt from 'bcryptjs';
 import * as config from '../config';
 import * as mongo from './mongo';
@@ -177,7 +186,10 @@ export const canUserCancelPush = async (id: string, user: string) => {
 };
 
 export const getSessionStore = (): MongoDBStore | undefined => start().getSessionStore();
-export const getPushes = (query: Partial<PushQuery>): Promise<Action[]> => start().getPushes(query);
+export const getPushes = (
+  query: Partial<PushQuery>,
+  pagination?: PaginationOptions,
+): Promise<PaginatedResult<Action>> => start().getPushes(query, pagination);
 export const writeAudit = (action: Action): Promise<void> => start().writeAudit(action);
 export const getPush = (id: string): Promise<Action | null> => start().getPush(id);
 export const deletePush = (id: string): Promise<void> => start().deletePush(id);
@@ -188,7 +200,10 @@ export const authorise = (
 export const cancel = (id: string): Promise<{ message: string }> => start().cancel(id);
 export const reject = (id: string, rejection: Rejection): Promise<{ message: string }> =>
   start().reject(id, rejection);
-export const getRepos = (query?: Partial<RepoQuery>): Promise<Repo[]> => start().getRepos(query);
+export const getRepos = (
+  query?: Partial<RepoQuery>,
+  pagination?: PaginationOptions,
+): Promise<PaginatedResult<Repo>> => start().getRepos(query, pagination);
 export const getRepo = (name: string): Promise<Repo | null> => start().getRepo(name);
 export const getRepoByUrl = (url: string): Promise<Repo | null> => start().getRepoByUrl(url);
 export const getRepoById = (_id: string): Promise<Repo | null> => start().getRepoById(_id);
@@ -206,7 +221,10 @@ export const findUserByEmail = (email: string): Promise<User | null> =>
   start().findUserByEmail(email);
 export const findUserByOIDC = (oidcId: string): Promise<User | null> =>
   start().findUserByOIDC(oidcId);
-export const getUsers = (query?: Partial<UserQuery>): Promise<User[]> => start().getUsers(query);
+export const getUsers = (
+  query?: Partial<UserQuery>,
+  pagination?: PaginationOptions,
+): Promise<PaginatedResult<User>> => start().getUsers(query, pagination);
 export const deleteUser = (username: string): Promise<void> => start().deleteUser(username);
 
 export const updateUser = (user: Partial<User>): Promise<void> => start().updateUser(user);
@@ -218,7 +236,7 @@ export const updateUser = (user: Partial<User>): Promise<void> => start().update
  */
 
 export const getAllProxiedHosts = async (): Promise<string[]> => {
-  const repos = await getRepos();
+  const { data: repos } = await getRepos();
   const origins = new Set<string>();
   repos.forEach((repo) => {
     const parsedUrl = processGitUrl(repo.url);
@@ -229,4 +247,4 @@ export const getAllProxiedHosts = async (): Promise<string[]> => {
   return Array.from(origins);
 };
 
-export type { PushQuery, Repo, Sink, User } from './types';
+export type { PaginatedResult, PaginationOptions, PushQuery, Repo, Sink, User } from './types';

src/db/mongo/helper.ts

@@ -14,7 +14,8 @@
  * limitations under the License.
  */
 
-import { MongoClient, Db, Collection, Filter, Document, FindOptions } from 'mongodb';
+import { MongoClient, Db, Collection, Filter, Document, FindOptions, Sort } from 'mongodb';
+import { PaginatedResult } from '../types';
 import { getDatabase } from '../../config';
 import MongoDBStore from 'connect-mongo';
 import { fromNodeProviderChain } from '@aws-sdk/credential-providers';
@@ -56,6 +57,22 @@ export const connect = async (collectionName: string): Promise<Collection> => {
   return _db.collection(collectionName);
 };
 
+export const paginatedFind = async <T>(
+  collection: Collection,
+  filter: Filter<Document>,
+  sort: Sort,
+  skip: number,
+  limit: number,
+  projection?: Document,
+): Promise<PaginatedResult<T>> => {
+  const total = await collection.countDocuments(filter);
+  const cursor = collection.find(filter, projection ? { projection } : undefined).sort(sort);
+  if (skip) cursor.skip(skip);
+  if (limit) cursor.limit(limit);
+  const data = (await cursor.toArray()) as T[];
+  return { data, total };
+};
+
 export const findDocuments = async <T>(
   collectionName: string,
   filter: Filter<Document> = {},