Skip to content

fix: stream SQL database dumps #206

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Treasure520520 wants to merge 2 commits into
base: main
Choose a base branch
from
Open

fix: stream SQL database dumps #206

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
22474ff
fix: stream SQL database dumps
Treasure520520 May 13, 2026
5fb75a9
fix: make SQL dump stream pull driven
Treasure520520 May 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 39 additions & 5 deletions src/export/dump.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,15 @@ describe('Database Dump Module', () => {
{ id: 1, name: 'Alice' },
{ id: 2, name: 'Bob' },
])
.mockResolvedValueOnce([])
.mockResolvedValueOnce([
{ sql: 'CREATE TABLE orders (id INTEGER, total REAL);' },
])
.mockResolvedValueOnce([
{ id: 1, total: 99.99 },
{ id: 2, total: 49.5 },
])
.mockResolvedValueOnce([])

const response = await dumpDatabaseRoute(mockDataSource, mockConfig)

Expand All @@ -71,13 +73,13 @@ describe('Database Dump Module', () => {
expect(dumpText).toContain(
'CREATE TABLE users (id INTEGER, name TEXT);'
)
expect(dumpText).toContain("INSERT INTO users VALUES (1, 'Alice');")
expect(dumpText).toContain("INSERT INTO users VALUES (2, 'Bob');")
expect(dumpText).toContain('INSERT INTO "users" VALUES (1, \'Alice\');')
expect(dumpText).toContain('INSERT INTO "users" VALUES (2, \'Bob\');')
expect(dumpText).toContain(
'CREATE TABLE orders (id INTEGER, total REAL);'
)
expect(dumpText).toContain('INSERT INTO orders VALUES (1, 99.99);')
expect(dumpText).toContain('INSERT INTO orders VALUES (2, 49.5);')
expect(dumpText).toContain('INSERT INTO "orders" VALUES (1, 99.99);')
expect(dumpText).toContain('INSERT INTO "orders" VALUES (2, 49.5);')
})

it('should handle empty databases (no tables)', async () => {
Expand Down Expand Up @@ -118,13 +120,45 @@ describe('Database Dump Module', () => {
{ sql: 'CREATE TABLE users (id INTEGER, bio TEXT);' },
])
.mockResolvedValueOnce([{ id: 1, bio: "Alice's adventure" }])
.mockResolvedValueOnce([])

const response = await dumpDatabaseRoute(mockDataSource, mockConfig)

expect(response).toBeInstanceOf(Response)
const dumpText = await response.text()
expect(dumpText).toContain(
"INSERT INTO users VALUES (1, 'Alice''s adventure');"
"INSERT INTO \"users\" VALUES (1, 'Alice''s adventure');"
)
})

it('should read table rows in bounded pages', async () => {
vi.mocked(executeOperation)
.mockResolvedValueOnce([{ name: 'events' }])
.mockResolvedValueOnce([
{ sql: 'CREATE TABLE events (id INTEGER, payload TEXT);' },
])
.mockResolvedValueOnce([{ id: 1, payload: 'first' }])
.mockResolvedValueOnce([{ id: 2, payload: 'second' }])
.mockResolvedValueOnce([])

const response = await dumpDatabaseRoute(mockDataSource, mockConfig)
const dumpText = await response.text()

expect(dumpText).toContain(
'INSERT INTO "events" VALUES (1, \'first\');'
)
expect(dumpText).toContain(
'INSERT INTO "events" VALUES (2, \'second\');'
)
expect(executeOperation).toHaveBeenCalledWith(
[
{
sql: 'SELECT * FROM "events" LIMIT ? OFFSET ?;',
params: [500, 500],
},
],
mockDataSource,
mockConfig
)
})

Expand Down
145 changes: 103 additions & 42 deletions src/export/dump.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,45 @@ import { StarbaseDBConfiguration } from '../handler'
import { DataSource } from '../types'
import { createResponse } from '../utils'

const DEFAULT_DUMP_BATCH_SIZE = 500

function quoteIdentifier(identifier: string): string {
return `"${identifier.replace(/"/g, '""')}"`
}

function toSqlHex(bytes: Uint8Array): string {
return Array.from(bytes)
.map((byte) => byte.toString(16).padStart(2, '0'))
.join('')
}

function formatSqlValue(value: unknown): string {
if (value === null || value === undefined) return 'NULL'
if (typeof value === 'number') {
return Number.isFinite(value) ? String(value) : 'NULL'
}
if (typeof value === 'bigint') return value.toString()
if (typeof value === 'boolean') return value ? '1' : '0'
if (value instanceof ArrayBuffer) {
return `X'${toSqlHex(new Uint8Array(value))}'`
}
if (ArrayBuffer.isView(value)) {
return `X'${toSqlHex(
new Uint8Array(value.buffer, value.byteOffset, value.byteLength)
)}'`
}

return `'${String(value).replace(/'/g, "''")}'`
}

function buildInsertStatement(
table: string,
row: Record<string, unknown>
): string {
const values = Object.values(row).map(formatSqlValue)
return `INSERT INTO ${quoteIdentifier(table)} VALUES (${values.join(', ')});\n`
}

export async function dumpDatabaseRoute(
dataSource: DataSource,
config: StarbaseDBConfiguration
Expand All @@ -16,54 +55,76 @@ export async function dumpDatabaseRoute(
)

const tables = tablesResult.map((row: any) => row.name)
let dumpContent = 'SQLite format 3\0' // SQLite file header

// Iterate through all tables
for (const table of tables) {
// Get table schema
const schemaResult = await executeOperation(
[
{
sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name='${table}';`,
},
],
dataSource,
config
)

if (schemaResult.length) {
const schema = schemaResult[0].sql
dumpContent += `\n-- Table: ${table}\n${schema};\n\n`
}

// Get table data
const dataResult = await executeOperation(
[{ sql: `SELECT * FROM ${table};` }],
dataSource,
config
)

for (const row of dataResult) {
const values = Object.values(row).map((value) =>
typeof value === 'string'
? `'${value.replace(/'/g, "''")}'`
: value
)
dumpContent += `INSERT INTO ${table} VALUES (${values.join(', ')});\n`
}

dumpContent += '\n'
}

// Create a Blob from the dump content
const blob = new Blob([dumpContent], { type: 'application/x-sqlite3' })
const encoder = new TextEncoder()
const stream = new ReadableStream({
async start(controller) {
try {
controller.enqueue(encoder.encode('SQLite format 3\0'))

for (const table of tables) {
const schemaResult = await executeOperation(
[
{
sql: `SELECT sql FROM sqlite_master WHERE type='table' AND name=?;`,
params: [table],
},
],
dataSource,
config
)

if (schemaResult.length) {
const schema = schemaResult[0].sql
controller.enqueue(
encoder.encode(
`\n-- Table: ${table}\n${schema};\n\n`
)
)
}

let offset = 0
while (true) {
const dataResult = await executeOperation(
[
{
sql: `SELECT * FROM ${quoteIdentifier(table)} LIMIT ? OFFSET ?;`,
params: [
DEFAULT_DUMP_BATCH_SIZE,
offset,
],
},
],
dataSource,
config
)

if (!dataResult.length) break

const chunk = dataResult
.map((row: Record<string, unknown>) =>
buildInsertStatement(table, row)
)
.join('')
controller.enqueue(encoder.encode(chunk))
offset += DEFAULT_DUMP_BATCH_SIZE
}

controller.enqueue(encoder.encode('\n'))
}

controller.close()
} catch (error) {
controller.error(error)
}
},
})

const headers = new Headers({
'Content-Type': 'application/x-sqlite3',
'Content-Disposition': 'attachment; filename="database_dump.sql"',
})

return new Response(blob, { headers })
return new Response(stream, { headers })
} catch (error: any) {
console.error('Database Dump Error:', error)
return createResponse(undefined, 'Failed to create database dump', 500)
Expand Down