fix(#120): MEDIUM: a shard backup copies a live WAL-mode SQLite with no quiesce step; resto

README.md

@@ -86,6 +86,13 @@ key: `ENCRYPTION_KEY`, persisted to `.encryption_key` next to the database
 - **Backups** must contain `webcalendar.db` *and* `.encryption_key` (or the
   `ENCRYPTION_KEY` value). A database restored without its key cannot be used. Restore the
   key and the service starts again.
+- **Consistent snapshot:** the shard backup copies the data directory file by file, which can
+  tear the live WAL-mode `webcalendar.db` / `-wal` / `-shm` while writes happen. The web app
+  therefore writes `webcalendar.db.backup` next to the database at boot and then every hour
+  (`VACUUM INTO`, checked with `PRAGMA integrity_check`, renamed into place). **To restore**,
+  stop all services, replace `webcalendar.db` with `webcalendar.db.backup`, delete
+  `webcalendar.db-wal` and `webcalendar.db-shm`, and start again. Changes made after the
+  snapshot (at most an hour before the backup) are lost.
 - **Rotation:** there is no re-encryption tool yet. To replace a key, stop all services,
   delete the rows of `linked_accounts`, `msgraph_accounts` and `encryption_key_check`,
   remove `.encryption_key`, set the new `ENCRYPTION_KEY`, start, and re-link the accounts.

agents.md

@@ -123,6 +123,8 @@ Radicale as the internal CalDAV server, a Python sync-worker for Google Calendar
   keep writes short-lived; schema changes must be additive and go into a new numbered entry in
   `server/utils/migrations.js` (never an ad-hoc `ALTER TABLE` in db.js). The server listens only after
   `Db.ready` (migrations + encryption-key check) resolved; a failed migration exits with status 1 (#343).
+  The server process (`server/utils/dbBackup.js`) writes the consistent snapshot `webcalendar.db.backup`
+  via `Db.createBackup` at boot and hourly, for the shard backup that copies the live files (#120).
 - Fail fast (#343): no empty or comment-only `catch {}`, no `.catch(() => {})` / `.catch(() => [])` /
   `.catch(noop)` (ESLint errors) and no `except Exception: pass`
   or blind `except Exception` (ruff S110/BLE001). Catch only the specific expected error, or let it reach

server/db.js

@@ -3005,40 +3005,35 @@ const Db = {
     return new Promise((resolve, reject) => {
       db.get(`PRAGMA wal_checkpoint(${normalizedMode})`, (err, row) => {
         if (err) return reject(err);
-        resolve({
-          busy: row ? (row.busy ?? 0) : 0,
-          log: row ? (row.log ?? 0) : 0,
-          checkpointed: row ? (row.checkpointed ?? 0) : 0
-        });
+        if (!row || typeof row.busy !== 'number') {
+          return reject(new Error(`PRAGMA wal_checkpoint(${normalizedMode}) returned no result; is the database in WAL mode?`));
+        }
+        resolve({ busy: row.busy, log: row.log, checkpointed: row.checkpointed });
       });
     });
   },
 
+  // Issue #120: a consistent copy of the live database for the shard backup, which copies the
+  // data directory file by file and would tear the live WAL triplet. VACUUM INTO reads within one
+  // transaction, so it includes commits still in the WAL and is not torn by concurrent writers.
+  // It writes a temp file, checks it, and renames it into place, so targetPath is always whole.
   async createBackup(targetPath) {
     if (!targetPath) {
       throw new Error('targetPath is required for database backup');
     }
-    const quiesceResult = await this.quiesceDatabase('TRUNCATE');
-    const dir = path.dirname(targetPath);
-    if (!fs.existsSync(dir)) {
-      fs.mkdirSync(dir, { recursive: true });
+    await ready;
+    const tmpPath = `${targetPath}.tmp`;
+    // VACUUM INTO refuses an existing file; a leftover from an interrupted run is never valid.
+    fs.rmSync(tmpPath, { force: true });
+    await new Promise((resolve, reject) => {
+      db.run('VACUUM INTO ?', [tmpPath], (err) => (err ? reject(err) : resolve()));
+    });
+    if (!(await this.verifyDatabaseIntegrity(tmpPath))) {
+      fs.rmSync(tmpPath, { force: true });
+      throw new Error(`Database snapshot ${tmpPath} failed PRAGMA integrity_check`);
     }
-
-    return new Promise((resolve, reject) => {
-      db.run(`VACUUM INTO ?`, [targetPath], (err) => {
-        if (!err) {
-          const stats = fs.statSync(targetPath);
-          return resolve({ backupPath: targetPath, size: stats.size, quiesceResult });
-        }
-        try {
-          fs.copyFileSync(dbPath, targetPath);
-          const stats = fs.statSync(targetPath);
-          resolve({ backupPath: targetPath, size: stats.size, quiesceResult, fallback: true });
-        } catch (copyErr) {
-          reject(copyErr);
-        }
-      });
-    });
+    fs.renameSync(tmpPath, targetPath);
+    return { backupPath: targetPath, size: fs.statSync(targetPath).size };
   },
 
   verifyDatabaseIntegrity(targetPath = dbPath) {
@@ -3093,6 +3088,7 @@ Db.migrations = migrations;
 Db.db = db;
 Db.getDbPath = () => dbPath;
 Db.dbPath = dbPath;
+Db.backupPath = `${dbPath}.backup`;
 
 module.exports = Db;
 module.exports.validateEncryptionKey = validateEncryptionKey;

server/index.js

@@ -109,6 +109,7 @@ const freebusyRouter = require('./routes/freebusy');
 const notificationsRouter = require('./routes/notifications');
 const webcalRouter = require('./routes/webcal');
 const { startWebcalSyncWorker } = require('./utils/webcalSubscription');
+const { startDbBackupWorker } = require('./utils/dbBackup');
 
 const app = express();
 const PORT = process.env.PORT || 3000;
@@ -663,6 +664,7 @@ if (listenWhenEmbedded) listen();
 Db.ready.then(() => {
   if (runAsMain) listen();
   if (runAsMain || listenWhenEmbedded) module.exports.webcalWorker = startWebcalSyncWorker();
+  if (runAsMain) module.exports.dbBackupWorker = startDbBackupWorker();
 }).catch((err) => {
   console.error(`[startup] ${err.message}`);
   process.exit(1);

server/test/issue120_consistent_backup_snapshot.test.js

@@ -0,0 +1,195 @@
+'use strict';
+
+/**
+ * Issue #120 — the shard backup (`rclone sync` of the app data directory at 03:00) copies the live
+ * WAL-mode `webcalendar.db`, `-wal` and `-shm` at three different instants, which restores torn.
+ *
+ * The app therefore keeps a consistent snapshot, `webcalendar.db.backup`, next to the live
+ * database: written with `VACUUM INTO` (a single read transaction, so commits that still live only
+ * in the WAL are included and concurrent writers cannot tear it), checked with
+ * `PRAGMA integrity_check`, and renamed into place so the backup never sees a half-written file.
+ * The server process writes it at boot and then hourly.
+ *
+ * Tests here restore the snapshot (open it as a database of its own) while another connection keeps
+ * committing inserts and deletes, and check integrity and row counts against the source.
+ */
+
+const os = require('node:os');
+const path = require('node:path');
+const fs = require('node:fs');
+
+const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'issue120-'));
+process.env.NODE_ENV = 'test';
+process.env.DATABASE_PATH = path.join(tmpDir, 'webcalendar.db');
+process.on('exit', () => fs.rmSync(tmpDir, { recursive: true, force: true }));
+
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const { spawn } = require('node:child_process');
+const sqlite3 = require('sqlite3');
+const Db = require('../db');
+const { startDbBackupWorker } = require('../utils/dbBackup');
+
+const PROBE_ROWS = 500;
+
+function open(file, mode) {
+  return new Promise((resolve, reject) => {
+    const conn = new sqlite3.Database(file, mode, (err) => (err ? reject(err) : resolve(conn)));
+  });
+}
+const exec = (conn, sql) => new Promise((resolve, reject) => conn.exec(sql, (err) => (err ? reject(err) : resolve())));
+const get = (conn, sql) => new Promise((resolve, reject) => conn.get(sql, (err, row) => (err ? reject(err) : resolve(row))));
+const close = (conn) => new Promise((resolve, reject) => conn.close((err) => (err ? reject(err) : resolve())));
+
+function payload(i) {
+  return `testuser-${i}-`.padEnd(20 + ((i * 37) % 400), 'x');
+}
+
+// Opens the file the way a restore would (a fresh process, no sidecars) and reports what it holds.
+async function restoreAndInspect(file) {
+  const conn = await open(file, sqlite3.OPEN_READONLY);
+  try {
+    const integrity = Object.values(await get(conn, 'PRAGMA integrity_check'))[0];
+    const { n } = await get(conn, 'SELECT count(*) AS n FROM backup_probe');
+    const { users } = await get(conn, 'SELECT count(*) AS users FROM users');
+    return { integrity, rows: n, users };
+  } finally {
+    await close(conn);
+  }
+}
+
+// A second connection that never checkpoints, so every commit stays in the WAL, and that keeps the
+// b-tree churning (variable-length inserts plus deletes) while the row count stays constant.
+async function startWriter(dbPath) {
+  const conn = await open(dbPath, sqlite3.OPEN_READWRITE);
+  await exec(conn, 'PRAGMA busy_timeout = 30000; PRAGMA wal_autocheckpoint = 0;');
+  await exec(conn, 'CREATE TABLE IF NOT EXISTS backup_probe (id INTEGER PRIMARY KEY, payload TEXT NOT NULL)');
+  let next = 0;
+  const insertBatch = (count) => {
+    const values = [];
+    for (let i = 0; i < count; i++, next++) values.push(`(${next}, '${payload(next)}')`);
+    return `INSERT INTO backup_probe (id, payload) VALUES ${values.join(',')};`;
+  };
+  await exec(conn, `BEGIN; ${insertBatch(PROBE_ROWS)} COMMIT;`);
+
+  let running = true;
+  let batches = 0;
+  const loop = (async () => {
+    while (running) {
+      await exec(conn, `BEGIN IMMEDIATE; ${insertBatch(50)}
+        DELETE FROM backup_probe WHERE id IN (SELECT id FROM backup_probe ORDER BY id LIMIT 50); COMMIT;`);
+      batches++;
+    }
+  })();
+  return {
+    conn,
+    batches: () => batches,
+    async stop() { running = false; await loop; },
+  };
+}
+
+test.describe('Issue #120 — consistent webcalendar.db.backup snapshot', () => {
+  const snapshotPath = `${process.env.DATABASE_PATH}.backup`;
+  let writer;
+
+  test.before(async () => {
+    await Db.ready;
+    await Db.upsertUser('testuser-id', 'testuser', 'testuser@example.com');
+    writer = await startWriter(process.env.DATABASE_PATH);
+  });
+
+  test.after(async () => {
+    await writer.stop();
+    await close(writer.conn);
+  });
+
+  test('the snapshot is written next to the live database', () => {
+    assert.equal(Db.backupPath, snapshotPath);
+  });
+
+  test('the backup worker writes a restorable snapshot holding every committed row while writes continue', async () => {
+    const walSize = fs.statSync(`${process.env.DATABASE_PATH}-wal`).size;
+    assert.ok(walSize > 0, 'precondition: the probe rows are committed but still only in the WAL');
+    const batchesBefore = writer.batches();
+
+    const worker = startDbBackupWorker({ intervalMs: 60 * 60 * 1000 });
+    try {
+      await worker.firstRun;
+    } finally {
+      worker.stop();
+    }
+    assert.ok(writer.batches() > batchesBefore, 'precondition: the writer committed while the snapshot was taken');
+
+    const restored = await restoreAndInspect(snapshotPath);
+    assert.equal(restored.integrity, 'ok');
+    assert.equal(restored.rows, PROBE_ROWS);
+    const live = await get(writer.conn, 'SELECT count(*) AS users FROM users');
+    assert.ok(live.users >= 1);
+    assert.equal(restored.users, live.users);
+    assert.equal(fs.existsSync(`${snapshotPath}.tmp`), false, 'no half-written temp file is left behind');
+  });
+
+  test('each run replaces the previous snapshot with the current state', async () => {
+    await Db.createBackup(snapshotPath);
+    await exec(writer.conn, `INSERT INTO backup_probe (id, payload) VALUES (-1, '${payload(1)}')`);
+    await Db.createBackup(snapshotPath);
+
+    const restored = await restoreAndInspect(snapshotPath);
+    assert.equal(restored.integrity, 'ok');
+    assert.equal(restored.rows, PROBE_ROWS + 1);
+  });
+
+  test('a stale temp file from an interrupted run does not block the next snapshot', async () => {
+    fs.writeFileSync(`${snapshotPath}.tmp`, 'half-written');
+    await Db.createBackup(snapshotPath);
+    assert.equal((await restoreAndInspect(snapshotPath)).integrity, 'ok');
+    assert.equal(fs.existsSync(`${snapshotPath}.tmp`), false);
+  });
+});
+
+test('the production server writes webcalendar.db.backup into its data directory at boot', async () => {
+  const dataDir = fs.mkdtempSync(path.join(tmpDir, 'prod-'));
+  const dbPath = path.join(dataDir, 'webcalendar.db');
+  const child = spawn(process.execPath, [path.join(__dirname, '..', 'index.js')], {
+    stdio: ['ignore', 'pipe', 'pipe'],
+    env: {
+      PATH: process.env.PATH,
+      HOME: process.env.HOME,
+      NODE_ENV: 'production',
+      AUTH_METHOD: 'env',
+      PORT: '0',
+      SSL_KEY_PATH: '',
+      SSL_CERT_PATH: '',
+      DATABASE_PATH: dbPath,
+      SESSION_SECRET: 'issue120-prod-session-secret-0123456789abcdef',
+      ENCRYPTION_KEY: '0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef',
+      SYSTEM_SYNC_SECRET: 'issue120-system-sync-secret-0123456789abcdef',
+      INTERNAL_WEBHOOK_SECRET: 'issue120-internal-webhook-secret-0123456789',
+      HTTP_HEADER_AUTH_SECRET: 'issue120-proxy-secret-0123456789abcdef0123',
+      BASE_URL: 'https://calendar.shard.example',
+      CALDAV_USERNAME: 'testuser',
+      CALDAV_EMAIL: 'testuser@example.com',
+    },
+  });
+  let output = '';
+  child.stdout.on('data', (c) => { output += c; });
+  child.stderr.on('data', (c) => { output += c; });
+  const exited = new Promise((resolve) => child.on('exit', resolve));
+  try {
+    const deadline = Date.now() + 15000;
+    while (!fs.existsSync(`${dbPath}.backup`) && child.exitCode === null && Date.now() < deadline) {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    }
+    assert.ok(fs.existsSync(`${dbPath}.backup`), `no snapshot written by the server; output:\n${output}`);
+    const conn = await open(`${dbPath}.backup`, sqlite3.OPEN_READONLY);
+    try {
+      assert.equal(Object.values(await get(conn, 'PRAGMA integrity_check'))[0], 'ok');
+      assert.ok((await get(conn, "SELECT count(*) AS n FROM sqlite_master WHERE name = 'users'")).n === 1);
+    } finally {
+      await close(conn);
+    }
+  } finally {
+    child.kill('SIGTERM');
+    await exited;
+  }
+});

server/utils/dbBackup.js

@@ -0,0 +1,27 @@
+'use strict';
+
+const Db = require('../db');
+
+const BACKUP_INTERVAL_MS = 60 * 60 * 1000;
+
+/**
+ * Issue #120: keeps Db.backupPath (`webcalendar.db.backup`) a consistent snapshot of the live
+ * database, so the nightly shard backup always contains a restorable copy. Writes one at start and
+ * then every intervalMs; hourly keeps it at most an hour behind whenever the shard backup runs.
+ */
+function startDbBackupWorker({ intervalMs = BACKUP_INTERVAL_MS } = {}) {
+  const run = async () => {
+    const { backupPath, size } = await Db.createBackup(Db.backupPath);
+    console.log(`[DbBackup] Wrote database snapshot ${backupPath} (${size} bytes)`);
+  };
+  const logFailure = (err) => console.error('[DbBackup] Writing the database snapshot failed:', err.message);
+
+  const firstRun = run();
+  firstRun.catch(logFailure);
+  const timer = setInterval(() => run().catch(logFailure), intervalMs);
+  if (timer.unref) timer.unref();
+
+  return { firstRun, stop: () => clearInterval(timer) };
+}
+
+module.exports = { startDbBackupWorker, BACKUP_INTERVAL_MS };