feat: implement permanent deletion of documents

apps/dashboard-api/src/controllers/project.controller.js

@@ -1368,13 +1368,13 @@ module.exports.insertData = async (req, res) => {
     delete incomingData.isDeleted;
     delete incomingData.deletedAt;
 
-    let docSize = 0;
+    let estimatedDocSize = 0;
     if (!project.resources.db.isExternal) {
-      docSize = Buffer.byteLength(JSON.stringify(incomingData));
+      estimatedDocSize = Buffer.byteLength(JSON.stringify(incomingData));
 
       const limit = project.databaseLimit || 20 * 1024 * 1024;
 
-      if ((project.databaseUsed || 0) + docSize > limit) {
+      if ((project.databaseUsed || 0) + estimatedDocSize > limit) {
         return res
           .status(403)
           .json({ error: "Database limit exceeded. Delete some data." });
@@ -1392,7 +1392,8 @@ module.exports.insertData = async (req, res) => {
     const result = await model.create(incomingData);
 
     if (!project.resources.db.isExternal) {
-      project.databaseUsed = (project.databaseUsed || 0) + docSize;
+      const actualDocSize = mongoose.mongo.BSON.calculateObjectSize(result.toObject());
+      project.databaseUsed = (project.databaseUsed || 0) + actualDocSize;
     }
     await project.save();
 
@@ -1443,30 +1444,55 @@ module.exports.deleteRow = async (req, res, next) => {
       project.resources.db.isExternal,
     );
 
-    const result = await Model.findOneAndUpdate(
-      { _id: id, isDeleted: { $ne: true } },
-      {
-        $set: {
-          isDeleted: true,
-          deletedAt: new Date()
-        }
-      },
-      { new: false }
-    ).lean();
+    const isPermanent = req.query.permanent === 'true';
+
+    let result;
+    if (isPermanent) {
+      result = await Model.findOneAndDelete({ _id: id }).lean();
+    } else {
+      result = await Model.findOneAndUpdate(
+        { _id: id, isDeleted: { $ne: true } },
+        {
+          $set: {
+            isDeleted: true,
+            deletedAt: new Date()
+          }
+        },
+        { new: false }
+      ).lean();
+    }
 
     if (!result) {
       return next(new AppError(404, "Document not found."));
     }
 
-    // We don't decrement databaseUsed here because the document still occupies space.
-    // It will be decremented during hard delete in the background worker.
-    try {
-      await enqueueCollectionCleanup(projectId, collectionName);
-    } catch (err) {
-      console.error("Failed to enqueue trash cleanup job", { projectId, collectionName, err });
+    if (isPermanent) {
+      if (!project.resources.db.isExternal) {
+        try {
+          const docSize = mongoose.mongo.BSON.calculateObjectSize(result);
+          await Project.updateOne(
+            { _id: project._id },
+            { $inc: { databaseUsed: -docSize } }
+          );
+        } catch (sizeErr) {
+          console.warn("Failed to calculate document size on permanent delete", sizeErr);
+        }
+      }
+    } else {
+      // We don't decrement databaseUsed here because the document still occupies space.
+      // It will be decremented during hard delete in the background worker.
+      try {
+        await enqueueCollectionCleanup(projectId, collectionName);
+      } catch (err) {
+        console.error("Failed to enqueue trash cleanup job", { projectId, collectionName, err });
+      }
     }
 
-    res.json({ success: true, data: { id: result._id }, message: "Document moved to trash" });
+    res.json({ 
+      success: true, 
+      data: { id: result._id }, 
+      message: isPermanent ? "Document permanently deleted" : "Document moved to trash" 
+    });
   } catch (err) {
     console.error("Delete Error:", err);
     next(new AppError(500, "Failed to delete document"));
@@ -1593,11 +1619,11 @@ module.exports.editRow = async (req, res) => {
     delete req.body.isDeleted;
     delete req.body.deletedAt;
 
-    const oldSize = Buffer.byteLength(JSON.stringify(docToEdit.toObject()));
+    const oldSize = mongoose.mongo.BSON.calculateObjectSize(docToEdit.toObject());
 
     docToEdit.set(req.body);
 
-    const newSize = Buffer.byteLength(JSON.stringify(docToEdit.toObject()));
+    const newSize = mongoose.mongo.BSON.calculateObjectSize(docToEdit.toObject());
     const sizeDiff = newSize - oldSize;
 
     if (!project.resources.db.isExternal) {

apps/web-dashboard/src/components/CollectionTable.jsx

@@ -226,21 +226,33 @@ export default function CollectionTable({ data, activeCollection, onDelete, onVi
                             </button>
                             {activeCollection?.name !== 'users' && !isViewer && (
                                 (record.isDeleted || recoveringIds.has(record._id)) ? (
-                                    <button
-                                        className={`btn-icon ${recoveringIds.has(record._id) ? 'loading' : ''}`}
-                                        onClick={() => onRecover(record._id)}
-                                        onPointerDown={e => e.stopPropagation()}
-                                        aria-label={`Restore record ${record._id}`}
-                                        disabled={recoveringIds.has(record._id)}
-                                        title={getDeletionTooltip(record.deletedAt, now)}
-                                        style={{ width: '22px', height: '22px', padding: 0 }}
-                                    >
-                                        {recoveringIds.has(record._id) ? (
-                                            <div className="spinner-small" style={{ width: '11px', height: '11px' }}></div>
-                                        ) : (
-                                            <RotateCcw size={12} color="var(--color-primary)" />
-                                        )}
-                                    </button>
+                                    <>
+                                        <button
+                                            className={`btn-icon ${recoveringIds.has(record._id) ? 'loading' : ''}`}
+                                            onClick={() => onRecover(record._id)}
+                                            onPointerDown={e => e.stopPropagation()}
+                                            aria-label={`Restore record ${record._id}`}
+                                            disabled={recoveringIds.has(record._id)}
+                                            title={getDeletionTooltip(record.deletedAt, now)}
+                                            style={{ width: '22px', height: '22px', padding: 0 }}
+                                        >
+                                            {recoveringIds.has(record._id) ? (
+                                                <div className="spinner-small" style={{ width: '11px', height: '11px' }}></div>
+                                            ) : (
+                                                <RotateCcw size={12} color="var(--color-primary)" />
+                                            )}
+                                        </button>
+                                        <button
+                                            className="btn-icon danger-hover"
+                                            onClick={() => onDelete(record._id, true)}
+                                            onPointerDown={e => e.stopPropagation()}
+                                            aria-label="Permanently Delete Record"
+                                            title="Permanently Delete"
+                                            style={{ width: '22px', height: '22px', padding: 0 }}
+                                        >
+                                            <Trash2 size={12} color="var(--color-danger)" />
+                                        </button>
+                                    </>
                                 ) : (
                                     <>
                                         <button
@@ -255,7 +267,7 @@ export default function CollectionTable({ data, activeCollection, onDelete, onVi
                                         </button>
                                         <button
                                             className="btn-icon danger-hover"
-                                            onClick={() => onDelete(record._id)}
+                                            onClick={() => onDelete(record._id, false)}
                                             onPointerDown={e => e.stopPropagation()}
                                             aria-label="Delete Record"
                                             title="Delete"

apps/web-dashboard/src/pages/Database.jsx

@@ -33,6 +33,7 @@ export default function Database() {
   const [isSidebarOpen, setIsSidebarOpen] = useState(false);
   const [showModal, setShowModal] = useState(false);
   const [selectedId, setSelectedId] = useState(null);
+  const [isPermanentDelete, setIsPermanentDelete] = useState(false);
   const [showDeleted, setShowDeleted] = useState(false);
   const [collectionToDelete, setCollectionToDelete] = useState(null);
   const [selectedRecord, setSelectedRecord] = useState(null);
@@ -160,11 +161,22 @@ export default function Database() {
     } catch { toast.error("Failed to save RLS"); return false; }
   };
 
-  const handleDeleteRecord = async (id) => {
+  const handleDeleteRecord = async (id, isPermanent) => {
     try {
-      await api.delete(`/api/projects/${projectId}/collections/${activeCollection.name}/data/${id}`);
-      setData(prev => prev.filter(item => item._id !== id));
-      toast.success("Document deleted");
+      await api.delete(`/api/projects/${projectId}/collections/${activeCollection.name}/data/${id}${isPermanent ? '?permanent=true' : ''}`);
+      if (isPermanent || !showDeleted) {
+        setData(prev => {
+          const newData = prev.filter(item => item._id !== id);
+          if (newData.length === 0 && queryParams.page > 1) {
+            setQueryParams(q => ({ ...q, page: q.page - 1 }));
+          }
+          return newData;
+        });
+        setTotalRecords(prev => Math.max(0, prev - 1));
+      } else {
+        setData(prev => prev.map(item => item._id === id ? { ...item, isDeleted: true, deletedAt: new Date().toISOString() } : item));
+      }
+      toast.success(isPermanent ? "Document permanently deleted" : "Document deleted");
     } catch { toast.error("Failed to delete document"); }
   };
 
@@ -371,7 +383,7 @@ export default function Database() {
                   <CollectionTable 
                     data={data} 
                     activeCollection={activeCollection} 
-                    onDelete={(id) => { setSelectedId(id); setShowModal(true); }} 
+                    onDelete={(id, isPermanent) => { setSelectedId(id); setIsPermanentDelete(isPermanent); setShowModal(true); }} 
                     onView={setSelectedRecord} 
                     onEdit={(rec) => { if (activeCollection?.name === 'users') return; setEditingRecord(rec); setIsAddModalOpen(true); }} 
                     onRecover={handleRecoverRecord}
@@ -446,7 +458,7 @@ export default function Database() {
       )}
 
       {/* Confirmation Modals */}
-      {showModal && <ConfirmationModal open={showModal} title="Delete Record" message="Confirm delete?" onConfirm={() => { handleDeleteRecord(selectedId); setShowModal(false); }} onCancel={() => setShowModal(false)} />}
+      {showModal && <ConfirmationModal open={showModal} title={isPermanentDelete ? "Permanently Delete Record" : "Delete Record"} message={isPermanentDelete ? "This action cannot be undone. Confirm permanent delete?" : "Confirm delete?"} onConfirm={() => { handleDeleteRecord(selectedId, isPermanentDelete); setShowModal(false); }} onCancel={() => setShowModal(false)} />}
       {collectionToDelete && <ConfirmationModal open={!!collectionToDelete} title="Delete Collection" message={`Delete ${collectionToDelete.name}?`} onConfirm={async () => { await api.delete(`/api/projects/${projectId}/collections/${collectionToDelete.name}`); setCollections(c => c.filter(x => x.name !== collectionToDelete.name)); setCollectionToDelete(null); }} onCancel={() => setCollectionToDelete(null)} />}
 
       {/* RLS Dialog */}

check_db2.js

@@ -0,0 +1,26 @@
+const mongoose = require('mongoose');
+require('dotenv').config();
+
+const MONGO_URL = process.env.MONGO_URL || 'mongodb://localhost:27017/urbackend-testV18';
+
+async function run() {
+  try {
+    await mongoose.connect(MONGO_URL);
+    const Project = require('./packages/common/src/models/Project.js');
+
+    const project = await Project.findOne({});
+    if (project && project.collections.length > 0) {
+      const col = project.collections[project.collections.length - 1];
+      console.log("Last collection:", col.name);
+      console.log("Model:", JSON.stringify(col.model, null, 2));
+    } else {
+      console.log("No projects");
+    }
+  } catch (err) {
+    console.error(err);
+    process.exitCode = 1;
+  } finally {
+    await mongoose.disconnect();
+  }
+}
+run();

scratch.js

@@ -0,0 +1,15 @@
+const mongoose = require('mongoose');
+const { Schema } = mongoose;
+
+const TestSchema = new Schema({
+  name: String,
+  isDeleted: Boolean
+});
+
+const TestModel = mongoose.model('Test', TestSchema);
+
+let q = TestModel.find();
+q = q.find({ isDeleted: { $ne: true } });
+q = q.and([{ name: 'foo' }]);
+
+console.log(JSON.stringify(q.getQuery(), null, 2));

sdks/urbackend-python/src/urbackend/db.py

@@ -419,13 +419,15 @@ def delete(
         collection: str,
         doc_id: str,
         token: Optional[str] = None,
+        permanent: bool = False,
     ) -> Dict[str, bool]:
         """Delete a document by its ``_id``.
 
         Args:
             collection: Collection name.
             doc_id: MongoDB ObjectId string of the document to delete.
             token: Bearer token for RLS ownership check.
+            permanent: True to permanently delete instead of soft-deleting.
 
         Returns:
             ``{"deleted": True}``
@@ -435,12 +437,14 @@ def delete(
 
         Example:
             >>> result = client.db.delete("posts", post_id,
-            ...                           token=client.auth.get_token())
+            ...                           token=client.auth.get_token(),
+            ...                           permanent=True)
             >>> assert result["deleted"] is True
         """
         # http.request raises on non-2xx, so reaching this point means the
         # delete succeeded.
+        query = "?permanent=true" if permanent else ""
         self._http.request(
-            "DELETE", f"/api/data/{collection}/{doc_id}", token=token
+            "DELETE", f"/api/data/{collection}/{doc_id}{query}", token=token
         )
         return {"deleted": True}

sdks/urbackend-sdk/src/modules/database.ts

@@ -371,17 +371,29 @@ export class DatabaseModule {
    *   console.log('User deleted');
    * }
    */
-  public async delete(collection: string, id: string, token?: string): Promise<{ deleted: boolean }> {
+  public async delete(collection: string, id: string, optionsOrToken?: { token?: string, permanent?: boolean } | string): Promise<{ deleted: boolean }> {
+    let token: string | undefined;
+    let permanent = false;
+
+    if (typeof optionsOrToken === 'string') {
+      token = optionsOrToken;
+    } else if (optionsOrToken) {
+      token = optionsOrToken.token;
+      permanent = !!optionsOrToken.permanent;
+    }
+
+    const qs = permanent ? '?permanent=true' : '';
+
     const result = await this.client.request<{ message?: string; id?: string } | null>(
       'DELETE',
-      `/api/data/${collection}/${id}`,
+      `/api/data/${collection}/${id}${qs}`,
       { token },
     );
 
     const deleted =
       typeof result === 'object' &&
       result !== null &&
-      (result.id === id || result.message === 'Document deleted');
+      (result.id === id || result.message === 'Document deleted' || result.message === 'Document permanently deleted');
 
     return { deleted };
   }

test_full.js

@@ -0,0 +1,37 @@
+const mongoose = require('mongoose');
+
+async function test() {
+  await mongoose.connect('mongodb://127.0.0.1:27017/test_isdeleted2');
+
+  const schemaDef = { name: String, isDeleted: { type: Boolean, default: false } };
+  const schema = new mongoose.Schema(schemaDef, { timestamps: true, strict: false });
+  const Model = mongoose.model('TestDoc2', schema);
+
+  await Model.deleteMany({});
+
+  await Model.create({ name: 'doc1', isDeleted: false });
+  await Model.create({ name: 'doc2', isDeleted: true });
+
+  const QueryEngine = require('./packages/common/src/utils/queryEngine.js');
+  const reqQuery = {};
+  const baseFilter = {}; // simulating req.rlsFilter
+
+  const features = new QueryEngine(Model.find(), reqQuery).filter();
+  if (Object.keys(baseFilter).length > 0) {
+      features.query = features.query.and([baseFilter]);
+  }
+
+  features.sort().limitFields(); // not populate for now
+  features.paginate();
+
+  const docs = await features.query.lean();
+  console.log("Returned docs:", docs);
+  console.log("Query was:", JSON.stringify(features.query.getQuery()));
+}
+
+test()
+  .catch(err => {
+    console.error(err);
+    process.exitCode = 1;
+  })
+  .finally(() => mongoose.disconnect());