🌳 fix: Preserve Primary Files Until Secondary Cleanup Succeeds

api/server/controllers/__tests__/deleteUser.spec.js

@@ -213,7 +213,10 @@ function stubDeletionMocks() {
   mockDeleteAllSharedLinks.mockResolvedValue();
   mockDeleteAllSharedLinksWithCleanup.mockResolvedValue({ deletedCount: 0 });
   mockGetFiles.mockResolvedValue([]);
-  mockProcessDeleteRequest.mockResolvedValue({ deletedFileIds: [], failedFileIds: [] });
+  mockProcessDeleteRequest.mockImplementation(async ({ files }) => ({
+    deletedFileIds: files.map((file) => file.file_id),
+    failedFileIds: [],
+  }));
   mockDeleteFiles.mockResolvedValue();
   mockDeleteToolCalls.mockResolvedValue();
   mockDeleteUserAgents.mockResolvedValue();
@@ -265,7 +268,7 @@ describe('deleteUserController - 2FA enforcement', () => {
     const req = { user: { id: USER_ID, _id: USER_ID, email: 'a@b.com' } };
     const res = createRes();
     mockGetUserById.mockResolvedValue({ _id: USER_ID, twoFactorEnabled: false });
-    mockGetFiles.mockResolvedValue(files);
+    mockGetFiles.mockResolvedValueOnce(files);
 
     await deleteUserController(req, res);
 
@@ -276,6 +279,27 @@ describe('deleteUserController - 2FA enforcement', () => {
     expect(fileRequest.user).toBe(req.user);
   });
 
+  it.each(['reported', 'thrown'])('retains the account on %s failure', async (failure) => {
+    const req = { user: { id: USER_ID, _id: USER_ID }, body: {} };
+    const res = createRes();
+    mockGetUserById.mockResolvedValue({ _id: USER_ID, twoFactorEnabled: false });
+    mockGetFiles.mockResolvedValue([{ file_id: 'file-1', embedded: true }]);
+    if (failure === 'reported') {
+      mockProcessDeleteRequest.mockResolvedValueOnce({
+        deletedFileIds: [],
+        failedFileIds: ['file-1'],
+      });
+    } else {
+      mockProcessDeleteRequest.mockRejectedValueOnce(new Error('provider down'));
+    }
+    await deleteUserController(req, res);
+    expect(res.status).toHaveBeenCalledWith(500);
+    expect(mockDeleteFiles).not.toHaveBeenCalled();
+    expect(mockDeleteUserKey).not.toHaveBeenCalled();
+    expect(mockDeleteUserById).not.toHaveBeenCalled();
+    expect(mockCancelAgentTriggerUserDeletion).toHaveBeenCalled();
+  });
+
   it('proceeds with deletion when 2FA is not enabled', async () => {
     const req = { user: { id: USER_ID, _id: USER_ID, email: 'a@b.com' }, body: {} };
     const res = createRes();

api/server/controllers/agents/__tests__/openai.spec.js

@@ -175,6 +175,8 @@ jest.mock('@librechat/agents', () => ({
 }));
 
 jest.mock('@librechat/api', () => ({
+  createFileProvisionRecovery: () =>
+    jest.requireActual('@librechat/api').createFileProvisionRecovery(),
   getAgentErrorMetadata: (...args) =>
     jest.requireActual('@librechat/api').getAgentErrorMetadata(...args),
   isTemporaryTurn: (...args) => jest.requireActual('@librechat/api').isTemporaryTurn(...args),

api/server/controllers/agents/__tests__/responses.unit.spec.js

@@ -198,6 +198,8 @@ jest.mock('@librechat/agents', () => ({
 }));
 
 jest.mock('@librechat/api', () => ({
+  createFileProvisionRecovery: () =>
+    jest.requireActual('@librechat/api').createFileProvisionRecovery(),
   getAgentErrorMetadata: (...args) =>
     jest.requireActual('@librechat/api').getAgentErrorMetadata(...args),
   getConversationWriteContext: (...args) =>

api/server/services/Endpoints/agents/initialize.js

@@ -117,6 +117,7 @@ const {
 } = require('./backgroundCompletion');
 const { logViolation } = require('~/cache');
 const db = require('~/models');
+const { getFileStorage } = require('~/server/services/Files/operations');
 const { getAppConfig } = require('~/server/services/Config');
 const { getStrategyFunctions } = require('~/server/services/Files/strategies');
 const { encodeAndFormat } = require('~/server/services/Files/images/encode');
@@ -1991,6 +1992,7 @@ const initializeClientWithProvider = async ({
     filterFiles: filterFilesByAgentAccess,
     listPublications: db.listRunArtifacts,
     provisioning: {
+      fileStorage: getFileStorage(),
       provisionToCodeEnv,
       provisionToVectorDB,
       updateFile: db.updateFile,

api/server/services/Files/Azure/crud.js

@@ -5,6 +5,7 @@ const fetch = require('node-fetch');
 const { logger } = require('@librechat/data-schemas');
 const {
   deleteRagFile,
+  dispatchFileDeletion,
   createOwnedBlobFileDeleter,
   extractAzureBlobPath,
   assertRemoteFileURL,
@@ -122,13 +123,13 @@ async function getAzureURL({ fileName, basePath = defaultBasePath, userId, conta
  * @param {ServerRequest} params.req - The Express request object.
  * @param {MongoFile} params.file - The file object.
  */
-const deleteFileFromAzure = (req, file) =>
+const deleteFileFromAzure = (req, file, _client, onDelete) =>
   createOwnedBlobFileDeleter({
     extractPath: (filepath) => extractAzureBlobPath(filepath, AZURE_CONTAINER_NAME),
     deleteBlob: async (blobPath) => {
       const containerClient = await getAzureContainerClient(AZURE_CONTAINER_NAME);
       const blockBlobClient = containerClient.getBlockBlobClient(blobPath);
-      await blockBlobClient.delete();
+      await dispatchFileDeletion(onDelete, () => blockBlobClient.delete());
       logger.debug('[deleteFileFromAzure] Blob deleted successfully from Azure Blob Storage');
     },
     isMissing: (error) => error?.statusCode === 404,

api/server/services/Files/Azure/crud.spec.js

@@ -1,16 +1,20 @@
 const mockDownload = jest.fn();
-const mockGetBlockBlobClient = jest.fn(() => ({ download: mockDownload }));
+const mockDelete = jest.fn();
+const mockGetBlockBlobClient = jest.fn(() => ({ download: mockDownload, delete: mockDelete }));
 const mockGetAzureContainerClient = jest.fn(async () => ({
   url: 'https://account.blob.core.windows.net/files',
   getBlockBlobClient: mockGetBlockBlobClient,
 }));
 const mockGetSafeErrorMetadata = jest.fn(() => ({ type: 'Error', status: 403 }));
 
 jest.mock('@librechat/data-schemas', () => ({
-  logger: { error: jest.fn() },
+  logger: { error: jest.fn(), debug: jest.fn() },
 }));
 
 jest.mock('@librechat/api', () => ({
+  ...jest.requireActual('../../../../../packages/api/src/storage/ownedBlobDelete'),
+  dispatchFileDeletion: jest.requireActual('../../../../../packages/api/src/files/dispatch')
+    .dispatchFileDeletion,
   deleteRagFile: jest.fn(),
   assertRemoteFileURL: jest.fn((url) => url),
   getSafeErrorMetadata: (...args) => mockGetSafeErrorMetadata(...args),
@@ -21,7 +25,8 @@ jest.mock('@librechat/api', () => ({
 }));
 
 const { logger } = require('@librechat/data-schemas');
-const { getAzureFileStream } = require('./crud');
+const { getAzureFileStream, deleteFileFromAzure } = require('./crud');
+const { deleteRagFile } = require('@librechat/api');
 
 describe('getAzureFileStream', () => {
   it('downloads private blobs through the authenticated Azure client', async () => {
@@ -76,3 +81,51 @@ describe('getAzureFileStream', () => {
     expect(JSON.stringify(logger.error.mock.calls)).not.toContain(signedUrl);
   });
 });
+
+describe('deleteFileFromAzure', () => {
+  const req = { user: { id: 'owner' } };
+  const file = { filepath: 'https://account.blob.core.windows.net/files/uploads/owner/file.txt' };
+
+  beforeEach(() => {
+    mockDelete.mockReset().mockResolvedValue(undefined);
+    deleteRagFile.mockReset().mockResolvedValue(undefined);
+  });
+
+  it('does not mark a failed client preflight', async () => {
+    const onDelete = jest.fn();
+    const error = Object.assign(new Error('client timed out'), { code: 'ETIMEDOUT' });
+    mockGetAzureContainerClient.mockRejectedValueOnce(error);
+    await expect(deleteFileFromAzure(req, file, undefined, onDelete)).rejects.toBe(error);
+    expect(onDelete).not.toHaveBeenCalled();
+    expect(mockDelete).not.toHaveBeenCalled();
+  });
+
+  it('does not mark invalid ownership before dispatch', async () => {
+    const onDelete = jest.fn();
+    await expect(
+      deleteFileFromAzure({ user: { id: 'other' } }, file, undefined, onDelete),
+    ).rejects.toThrow('Invalid file path');
+    expect(onDelete).not.toHaveBeenCalled();
+    expect(mockDelete).not.toHaveBeenCalled();
+  });
+
+  it('marks a lost delete response without confirming deletion', async () => {
+    const onDelete = jest.fn();
+    const error = Object.assign(new Error('response lost'), { code: 'ETIMEDOUT' });
+    mockDelete.mockRejectedValueOnce(error);
+    await expect(deleteFileFromAzure(req, file, undefined, onDelete)).rejects.toBe(error);
+    expect(onDelete.mock.calls).toEqual([[]]);
+    expect(deleteRagFile).not.toHaveBeenCalled();
+  });
+
+  it('confirms deletion before a later RAG denial', async () => {
+    const onDelete = jest.fn();
+    const error = Object.assign(new Error('RAG denied'), { statusCode: 403 });
+    deleteRagFile.mockImplementationOnce(async () => {
+      expect(onDelete.mock.calls).toEqual([[], ['confirmed']]);
+      throw error;
+    });
+    await expect(deleteFileFromAzure(req, file, undefined, onDelete)).rejects.toBe(error);
+    expect(onDelete.mock.calls).toEqual([[], ['confirmed']]);
+  });
+});

api/server/services/Files/Code/__tests__/process-traversal.spec.js

@@ -22,6 +22,7 @@ jest.mock('@librechat/api', () => {
   const https = require('https');
   return {
     processCodeOutput: jest.requireActual('@librechat/api').processCodeOutput,
+    createFileProvisionRecovery: jest.requireActual('@librechat/api').createFileProvisionRecovery,
     createSandboxTextReader: jest.requireActual('@librechat/api').createSandboxTextReader,
     resolveDownloadPath: (file) => file.storageKey || file.filepath,
     logAxiosError: jest.fn(),
@@ -82,6 +83,12 @@ jest.mock('~/models', () => ({
   updateFile: jest.fn(),
   claimCodeFile: jest.fn().mockResolvedValue({ file_id: 'mock-uuid', usage: 0 }),
   commitCodeFile: jest.fn().mockResolvedValue(true),
+  acquireFileProvision: jest.fn().mockResolvedValue(true),
+  releaseFileProvision: jest.fn().mockResolvedValue(true),
+  commitFileProvision: jest.fn(async ({ output, releaseOnly }) => {
+    if (releaseOnly) return true;
+    return jest.requireMock('~/models').commitCodeFile(output.file, output.sourceDispatchedAt);
+  }),
 }));
 
 const mockSaveBuffer = jest.fn().mockResolvedValue('/uploads/user123/mock-uuid__output.csv');

api/server/services/Files/Code/crud.js

@@ -1,9 +1,11 @@
 const FormData = require('form-data');
 const { logger } = require('@librechat/data-schemas');
 const { getCodeBaseURL } = require('@librechat/agents');
-const { EModelEndpoint, getCodeEnvRefs } = require('librechat-data-provider');
+const { clearFileCodeEnvRefs } = require('~/models');
 const {
   logAxiosError,
+  createCodeFileDeleter,
+  dispatchFileStorageOperation,
   wrapCodeApiUploadError,
   appendCodeEnvFile,
   createAxiosInstance,
@@ -72,106 +74,18 @@ async function getCodeOutputDownloadStream(fileIdentifier, identity, req, route
   }
 }
 
-/**
- * A deployment that no longer serves a ref's profile rejects every retry the same
- * way, so failing the delete would leave a record the user can never remove and an
- * agent that aborts every code turn on it.
- */
-const isProfileMismatch = (error) =>
-  error.response?.status === 409 && error.response?.data?.error === 'execution_profile_mismatch';
-
-/**
- * Deletes a file from the Code Environment server.
- *
- * @param {ServerRequest} req - Current authenticated request, used to mint Code API auth.
- * @param {MongoFile} file - File metadata containing `metadata.codeEnvRef`.
- * @returns {Promise<void>}
- */
-async function deleteCodeEnvFile(req, file) {
-  const refs = getCodeEnvRefs(file?.metadata);
-  if (refs.length === 0) {
-    return;
-  }
-
-  for (const [executionRouteKey, ref] of refs) {
-    const executionProfile = ref.executionProfile ?? 'default';
-    const environments =
-      req.config?.endpoints?.[EModelEndpoint.agents]?.statefulCodeSessions?.environments;
-    const configuredEnvironment = environments?.find(
-      (environment) =>
-        createCodeExecutionRouteKey(executionProfile, environment) === executionRouteKey,
-    );
-    if (
-      executionProfile === 'stateful' &&
-      executionRouteKey !== executionProfile &&
-      !configuredEnvironment
-    ) {
-      logger.warn(
-        `[deleteCodeEnvFile] Skipping remote cleanup for unmapped historical route ${executionRouteKey}`,
-      );
-      continue;
-    }
-    let baseURL;
-    try {
-      baseURL = getCodeExecutionBaseUrl(executionProfile, configuredEnvironment);
-    } catch (error) {
-      if (
-        executionProfile === 'stateful' &&
-        executionRouteKey === executionProfile &&
-        !configuredEnvironment
-      ) {
-        logger.warn(
-          '[deleteCodeEnvFile] Skipping remote cleanup for retired legacy stateful route',
-        );
-        continue;
-      }
-      throw error;
-    }
-    const query = buildCodeEnvDownloadQuery({
-      kind: ref.kind,
-      id: ref.id,
-      ...(ref.kind === 'skill' ? { version: ref.version } : {}),
-    });
-    const bridgeWorkerId =
-      configuredEnvironment?.workerId ?? configuredEnvironment?.pairing?.workerId;
-    const authHeaders = await getCodeApiAuthHeaders(req, bridgeWorkerId);
-    /* codeapi has mounted DELETE at `/files/:session_id/:fileId` since its
-     * first release. The file-server's own `/sessions/:id/objects/:fileId`
-     * only gained DELETE in LibreChat-AI/code-interpreter#85, so trying it
-     * first cost a guaranteed 404 against every older deployment. */
-    try {
-      await axios({
-        method: 'delete',
-        url: `${baseURL}/files/${ref.storage_session_id}/${ref.file_id}${query}`,
-        headers: {
-          'User-Agent': 'LibreChat/1.0',
-          ...authHeaders,
-          ...codeExecutionHeaders({
-            executionProfile,
-            bridgeWorkerId,
-          }),
-        },
-        httpAgent: codeServerHttpAgent,
-        httpsAgent: codeServerHttpsAgent,
-        timeout: 15000,
-      });
-    } catch (error) {
-      const profileMismatch = isProfileMismatch(error);
-      if (error.response?.status !== 404 && !profileMismatch) {
-        throw error;
-      }
-      /* Already gone. Logged rather than swallowed: a 404 from a
-       * misconfigured base URL is indistinguishable from one for an absent
-       * object, and this branch drops the file's record either way. */
-      logAxiosError({
-        error,
-        message: profileMismatch
-          ? `Code environment no longer serves the ${executionProfile} profile; dropping the unreachable object: ${error.message}`
-          : `Code environment object already absent: ${error.message}`,
-      });
-    }
-  }
-}
+const deleteCodeEnvFile = createCodeFileDeleter({
+  httpClient: axios,
+  getAuthHeaders: getCodeApiAuthHeaders,
+  getBaseUrl: getCodeExecutionBaseUrl,
+  routeKey: createCodeExecutionRouteKey,
+  profileHeaders: codeExecutionHeaders,
+  downloadQuery: buildCodeEnvDownloadQuery,
+  clearFileCodeEnvRefs,
+  logger,
+  httpAgent: codeServerHttpAgent,
+  httpsAgent: codeServerHttpsAgent,
+});
 
 /**
  * Uploads a file to the Code Environment server.
@@ -211,6 +125,7 @@ async function uploadCodeEnvFile({
   executionProfile,
   bridgeWorkerId,
   signal,
+  onUploadStart,
 }) {
   try {
     const form = new FormData();
@@ -237,7 +152,9 @@ async function uploadCodeEnvFile({
       ...(signal ? { signal } : {}),
     };
 
-    const response = await axios.post(`${baseURL}/upload`, form, options);
+    const response = await dispatchFileStorageOperation(onUploadStart, () =>
+      axios.post(`${baseURL}/upload`, form, options),
+    );
 
     /** @type {{ message: string; storage_session_id: string; files: Array<{ fileId: string; filename: string }> }} */
     const result = response.data;