feat: support form versioning

package.json

@@ -47,10 +47,12 @@
     "tsx": "^4.19.1",
     "typescript": "^5.6.2",
     "vite": "^8.0.16",
-    "vite-tsconfig-paths": "^5.0.1",
     "vitest": "^4.0.0",
     "vitest-mock-express": "^2.2.0",
-    "vitest-mock-extended": "^4.0.0"
+    "vitest-mock-extended": "^4.0.0",
+    "esbuild": "^0.28.1",
+    "form-data": "^4.0.6"
   },
+  "//": "esbuild and form-data are not direct dependencies of forms-api; however, we had to add them to enforce the use of versions that are not flagged as vulnerable by our security scanning system (we should be able to remove them in the future)",
   "packageManager": "pnpm@11.8.0+sha512.c1f5e7c4cb241c8f174b743851d82f42b802324afc8b0f116b96adb15aa06664948dde36960a3ba1079ba5b4b29dd0140135b94b5b5f5263592249d68e555f26"
 }

pnpm-workspace.yaml

@@ -1,3 +1,6 @@
+blockExoticSubdeps: true
+minimumReleaseAge: 10080 # 7 days
+
 allowBuilds:
   '@biomejs/biome': false
   esbuild: false

src/lib/formsClient/getFormTemplate.ts

@@ -4,16 +4,36 @@ import { logMessage } from "@lib/logging/logger.js";
 
 export async function getFormTemplate(
   formId: string,
+  version: number,
 ): Promise<FormTemplate | undefined> {
   try {
-    const template = await prisma.template.findUnique({
-      where: {
-        id: formId,
-      },
-      select: {
-        jsonConfig: true,
-      },
-    });
+    const template = await prisma.templateVersion
+      .findUnique({
+        where: {
+          templateId_versionNumber: {
+            templateId: formId,
+            versionNumber: version,
+          },
+        },
+        select: {
+          jsonConfig: true,
+        },
+      })
+      // Fallback query to be deleted once form versioning is fully released in Production
+      .then((result) => {
+        if (result !== null) {
+          return result;
+        }
+
+        return prisma.template.findUnique({
+          where: {
+            id: formId,
+          },
+          select: {
+            jsonConfig: true,
+          },
+        });
+      });
 
     if (template === null) {
       return undefined;

src/lib/vault/getFormSubmission.ts

@@ -16,7 +16,7 @@ export async function getFormSubmission(
           TableName: "Vault",
           Key: { FormID: formId, NAME_OR_CONF: `NAME#${submissionName}` },
           ProjectionExpression:
-            "CreatedAt,#statusCreatedAtKey,ConfirmationCode,FormSubmission,FormSubmissionHash,SubmissionAttachments",
+            "CreatedAt,#statusCreatedAtKey,ConfirmationCode,FormSubmission,FormSubmissionHash,SubmissionAttachments,Version",
           ExpressionAttributeNames: {
             "#statusCreatedAtKey": "Status#CreatedAt",
           },

src/lib/vault/getNewFormSubmissions.ts

@@ -1,4 +1,5 @@
 import { QueryCommand, type QueryCommandOutput } from "@aws-sdk/lib-dynamodb";
+import { ENVIRONMENT_MODE, EnvironmentMode } from "@config";
 import { AwsServicesConnector } from "@lib/integration/awsServicesConnector.js";
 import { logMessage } from "@lib/logging/logger.js";
 import { mapNewFormSubmissionFromDynamoDbResponse } from "@lib/vault/mappers/formSubmission.mapper.js";
@@ -17,12 +18,15 @@ export async function getNewFormSubmissions(
         await AwsServicesConnector.getInstance().dynamodbClient.send(
           new QueryCommand({
             TableName: "Vault",
-            IndexName: "StatusCreatedAt",
+            IndexName:
+              ENVIRONMENT_MODE !== EnvironmentMode.production // Condition to be deleted once StatusCreatedAt_v2 becomes available in Production
+                ? "StatusCreatedAt_v2"
+                : "StatusCreatedAt",
             ExclusiveStartKey: lastEvaluatedKey ?? undefined,
             Limit: limit - newFormSubmissions.length,
             KeyConditionExpression:
               "FormID = :formId AND begins_with(#statusCreatedAt, :status)",
-            ProjectionExpression: "#name,CreatedAt",
+            ProjectionExpression: `#name,CreatedAt${ENVIRONMENT_MODE !== EnvironmentMode.production ? ",Version" : ""}`, // Condition to be deleted once StatusCreatedAt_v2 becomes available in Production
             ExpressionAttributeNames: {
               "#statusCreatedAt": "Status#CreatedAt",
               "#name": "Name",

src/lib/vault/mappers/formSubmission.mapper.ts

@@ -11,20 +11,24 @@ export function mapNewFormSubmissionFromDynamoDbResponse(
   response: Record<string, unknown>,
 ): NewFormSubmission {
   try {
+    // response.Version could be undefined if API users are retrieving responses that have been created before we implemented the form versioning feature
     if (response.Name === undefined || response.CreatedAt === undefined) {
       throw new Error("Missing key properties in DynamoDB response");
     }
 
     if (
       typeof response.Name !== "string" ||
-      typeof response.CreatedAt !== "number"
+      typeof response.CreatedAt !== "number" ||
+      // response.Version could be undefined if API users are retrieving responses that have been created before we implemented the form versioning feature
+      (response.Version !== undefined && typeof response.Version !== "number")
     ) {
       throw new Error("Unexpected type in DynamoDB response");
     }
 
     return {
       name: response.Name,
       createdAt: response.CreatedAt,
+      version: response.Version ?? 1,
     };
   } catch (error) {
     logMessage.info(
@@ -47,6 +51,7 @@ export function mapFormSubmissionFromDynamoDbResponse(
       response.FormSubmission === undefined ||
       response.FormSubmissionHash === undefined
       // response.SubmissionAttachments could be undefined if API users are retrieving responses that have been created before we implemented the submission attachments feature
+      // response.Version could be undefined if API users are retrieving responses that have been created before we implemented the form versioning feature
     ) {
       throw new Error("Missing key properties in DynamoDB response");
     }
@@ -57,8 +62,11 @@ export function mapFormSubmissionFromDynamoDbResponse(
       typeof response.ConfirmationCode !== "string" ||
       typeof response.FormSubmission !== "string" ||
       typeof response.FormSubmissionHash !== "string" ||
+      // response.SubmissionAttachments could be undefined if API users are retrieving responses that have been created before we implemented the submission attachments feature
       (response.SubmissionAttachments !== undefined &&
-        typeof response.SubmissionAttachments !== "string")
+        typeof response.SubmissionAttachments !== "string") ||
+      // response.Version could be undefined if API users are retrieving responses that have been created before we implemented the form versioning feature
+      (response.Version !== undefined && typeof response.Version !== "number")
     ) {
       throw new Error("Unexpected type in DynamoDB response");
     }
@@ -74,6 +82,7 @@ export function mapFormSubmissionFromDynamoDbResponse(
             response.SubmissionAttachments,
           )
         : [],
+      version: response.Version ?? 1,
     };
   } catch (error) {
     logMessage.info(

src/lib/vault/types/formSubmission.types.ts

@@ -1,6 +1,7 @@
 export type NewFormSubmission = {
   name: string;
   createdAt: number;
+  version: number;
 };
 
 export const SubmissionStatus = {
@@ -40,4 +41,5 @@ export type FormSubmission = {
   answers: string;
   checksum: string;
   attachments: PartialAttachment[];
+  version: number;
 };

src/operations/retrieveNewSubmissions.v1.ts

@@ -48,9 +48,14 @@ async function v1(
 function buildResponse(newFormSubmissions: NewFormSubmission[]): {
   name: string;
   createdAt: number;
+  version: number;
 }[] {
   return newFormSubmissions.map((item) => {
-    return { name: item.name, createdAt: item.createdAt };
+    return {
+      name: item.name,
+      createdAt: item.createdAt,
+      version: item.version,
+    };
   });
 }
 

src/operations/retrieveSubmission.v1.ts

@@ -111,6 +111,7 @@ function buildJsonResponse(
         md5: attachment.md5,
       })),
     }),
+    version: formSubmission.version,
   });
 }
 

src/operations/retrieveTemplate.v1.ts

@@ -17,9 +17,17 @@ async function v1(
   const serviceUserId = retrieveRequestContextData(
     RequestContextualStoreKey.serviceUserId,
   );
+  const version = Number(request.query.version ?? 1);
 
   try {
-    const formTemplate = await getFormTemplate(formId);
+    if (Number.isNaN(version)) {
+      response
+        .status(400)
+        .json({ error: "URL parameter 'version' should be a number" });
+      return;
+    }
+
+    const formTemplate = await getFormTemplate(formId, version);
 
     if (formTemplate === undefined) {
       response.status(404).json({ error: "Form template does not exist" });
@@ -38,7 +46,7 @@ async function v1(
   } catch (error) {
     next(
       new Error(
-        `[operation] Internal error while retrieving template. Params: formId = ${formId}`,
+        `[operation] Internal error while retrieving template. Params: formId = ${formId} ; version = ${request.query.version}`,
         { cause: error },
       ),
     );

test/lib/encryption/encryptResponse.test.ts

@@ -32,9 +32,9 @@ describe("encryptFormSubmission should", () => {
     });
     const logMessageSpy = vi.spyOn(logMessage, "error");
 
-    expect(() =>
-      encryptResponse("serviceAccountPublicKey", "payload"),
-    ).toThrowError("custom error");
+    expect(() => encryptResponse("serviceAccountPublicKey", "payload")).toThrow(
+      "custom error",
+    );
 
     expect(logMessageSpy).toHaveBeenCalledWith(
       customError,

test/lib/formsClient/getFormTemplate.test.ts

@@ -1,4 +1,9 @@
-import { type PrismaClient, type Template, prisma } from "@gcforms/database";
+import {
+  type PrismaClient,
+  type Template,
+  type TemplateVersion,
+  prisma,
+} from "@gcforms/database";
 import { getFormTemplate } from "@lib/formsClient/getFormTemplate.js";
 import { logMessage } from "@lib/logging/logger.js";
 import { beforeEach, describe, expect, it, vi } from "vitest";
@@ -13,14 +18,43 @@ describe("getFormTemplate should", () => {
   });
 
   it("return an undefined form template if database was not able to find it", async () => {
-    prismaMock.template.findUnique.mockResolvedValueOnce(null);
+    prismaMock.templateVersion.findUnique.mockResolvedValueOnce(null);
 
-    const formTemplate = await getFormTemplate("clzamy5qv0000115huc4bh90m");
+    const formTemplate = await getFormTemplate("clzamy5qv0000115huc4bh90m", 1);
 
     expect(formTemplate).toBeUndefined();
   });
 
   it("return a form template if database was able to find it", async () => {
+    prismaMock.templateVersion.findUnique.mockResolvedValue({
+      jsonConfig: {
+        elements: [
+          {
+            id: 1,
+            type: "textField",
+          },
+        ],
+      },
+    } as unknown as TemplateVersion);
+
+    const formTemplate = await getFormTemplate("clzamy5qv0000115huc4bh90m", 1);
+
+    expect(formTemplate).toEqual({
+      jsonConfig: {
+        elements: [
+          {
+            id: 1,
+            type: "textField",
+          },
+        ],
+      },
+    });
+  });
+
+  // This test can be deleted once form versioning is implemented in Production and migrated old form template database entries to the new versioned schema
+  it("return a form template even if form versioning is not fully deployed (testing fallback Prisma query)", async () => {
+    prismaMock.templateVersion.findUnique.mockResolvedValue(null);
+
     prismaMock.template.findUnique.mockResolvedValue({
       jsonConfig: {
         elements: [
@@ -32,7 +66,7 @@ describe("getFormTemplate should", () => {
       },
     } as unknown as Template);
 
-    const formTemplate = await getFormTemplate("clzamy5qv0000115huc4bh90m");
+    const formTemplate = await getFormTemplate("clzamy5qv0000115huc4bh90m", 1);
 
     expect(formTemplate).toEqual({
       jsonConfig: {
@@ -48,12 +82,12 @@ describe("getFormTemplate should", () => {
 
   it("throw an error if database has an internal failure", async () => {
     const customError = new Error("custom error");
-    prismaMock.template.findUnique.mockRejectedValueOnce(customError);
+    prismaMock.templateVersion.findUnique.mockRejectedValueOnce(customError);
     const logMessageSpy = vi.spyOn(logMessage, "error");
 
     await expect(() =>
-      getFormTemplate("clzamy5qv0000115huc4bh90m"),
-    ).rejects.toThrowError(customError);
+      getFormTemplate("clzamy5qv0000115huc4bh90m", 1),
+    ).rejects.toThrow(customError);
 
     expect(logMessageSpy).toHaveBeenCalledWith(
       customError,

test/lib/formsClient/getPublicKey.test.ts

@@ -56,7 +56,7 @@ describe("getPublicKey should", () => {
     prismaMock.apiServiceAccount.findUnique.mockRejectedValueOnce(customError);
     const logMessageSpy = vi.spyOn(logMessage, "info");
 
-    await expect(() => getPublicKey("254354365464565461")).rejects.toThrowError(
+    await expect(() => getPublicKey("254354365464565461")).rejects.toThrow(
       customError,
     );
 

test/lib/idp/verifyAccessToken.test.ts

@@ -201,7 +201,7 @@ describe("verifyAccessToken should", () => {
           "RkS8hzu0MtwL+Qs2lK7KX9CLK7v6lxYpqs7ns5MwuOs=",
           "0000",
         ),
-      ).rejects.toThrowError(ZitadelConnectionError);
+      ).rejects.toThrow(ZitadelConnectionError);
 
       expect(logMessageSpy).toHaveBeenCalledWith(
         connectionError,

test/lib/integration/zitadelConnector.test.ts

@@ -43,9 +43,9 @@ describe("introspectAccessToken should", () => {
 
     const logMessageSpy = vi.spyOn(logMessage, "info");
 
-    await expect(() =>
-      introspectAccessToken("accessToken"),
-    ).rejects.toThrowError(ZitadelConnectionError);
+    await expect(() => introspectAccessToken("accessToken")).rejects.toThrow(
+      ZitadelConnectionError,
+    );
 
     expect(logMessageSpy).toHaveBeenCalledWith(
       connectionError,

test/lib/storage/requestContextualStore.test.ts

@@ -22,7 +22,7 @@ describe("requestContextualStore", () => {
     it("throw an error if the requestContextualStore is undefined", () => {
       expect(() =>
         saveRequestContextData(RequestContextualStoreKey.serviceUserId, "test"),
-      ).toThrowError("requestContextualStore is undefined");
+      ).toThrow("requestContextualStore is undefined");
     });
   });
 
@@ -40,7 +40,7 @@ describe("requestContextualStore", () => {
     it("throw an error if the requestContextualStore is undefined", () => {
       expect(() =>
         retrieveOptionalRequestContextData(RequestContextualStoreKey.clientIp),
-      ).toThrowError("requestContextualStore is undefined");
+      ).toThrow("requestContextualStore is undefined");
     });
   });
 
@@ -61,7 +61,7 @@ describe("requestContextualStore", () => {
 
         expect(() =>
           retrieveRequestContextData(RequestContextualStoreKey.clientIp),
-        ).toThrowError(
+        ).toThrow(
           "requestContextualStore does not have any set value for key: clientIp",
         );
       });
@@ -70,7 +70,7 @@ describe("requestContextualStore", () => {
     it("throw an error if the requestContextualStore is undefined", () => {
       expect(() =>
         retrieveRequestContextData(RequestContextualStoreKey.clientIp),
-      ).toThrowError("requestContextualStore is undefined");
+      ).toThrow("requestContextualStore is undefined");
     });
   });
 });

test/lib/support/notifySupportAboutFormSubmissionProblem.test.ts

@@ -67,7 +67,7 @@ Here is my problem<br/>
         "en",
         EnvironmentMode.production,
       ),
-    ).rejects.toThrowError(customError);
+    ).rejects.toThrow(customError);
 
     expect(logMessageSpy).toHaveBeenCalledWith(
       customError,