Loading files
/**
* Read-only SQLite access for the engines that keep their conversation in a store instead of a
* transcript file (opencode, kilo, hermes, devin) and for the session-repair branch that looks them up.
*
* Two ways in, tried in this order:
*
* 1. `node:sqlite` — built into the Node the installers ship (22.23, see `runtime/current-node`),
* and into any Node ≥ 22.13 without a flag. In-process, parameterised, no JSON round-trip: the
* query that hung the `sqlite3` CLI for good on one machine (an opencode session with a ~7.5 MB
* row) finishes here in tens of milliseconds. The binding is SYNCHRONOUS — it blocks the event
* loop for the length of the query — so the busy timeout is kept short: a contended read returns
* `transient` and the caller polls again next tick, exactly as a locked CLI read did before.
* 2. The `sqlite3` CLI — for a daemon on a Node without the binding (`engines.node >= 20`). Same
* invocation the readers always used, plus a hard timeout: a CLI that never answers used to hold
* the caller (and, at boot, the whole daemon) forever.
*
* Neither present ⇒ `missing`, which the readers report once and stop on, as they did for a missing CLI.
*
* Every store this reads is WAL (measured: opencode, kilo, hermes), so a reader never blocks the
* engine's writer and is rarely blocked by it; `busy_timeout` only covers the checkpoint window.
*/
import { execFile } from 'node:child_process'
import { closeSync, existsSync, openSync, readSync, statSync } from 'node:fs'
import { pathToFileURL } from 'node:url'
import { promisify } from 'node:util'
const execFileAsync = promisify(execFile)
export type SqliteRow = Record<string, unknown>
export type SqliteParam = string | number | null
export interface SqliteReadOptions {
/** How long the built-in binding may wait on a lock. Short on purpose: it blocks the event loop. */
busyTimeoutMs?: number
/** Hard bound on the CLI fallback process. */
cliTimeoutMs?: number
/** stdout cap for the CLI fallback. */
maxBuffer?: number
}
export type SqliteReadResult =
| { ok: true; rows: SqliteRow[]; via: 'builtin' | 'cli' }
/** `missing`: no way to read SQLite on this machine. `transient`: locked, mid-write, unreadable — retry. */
| { ok: false; reason: 'missing' | 'transient'; error?: Error }
const DEFAULT_BUSY_TIMEOUT_MS = 250
const DEFAULT_CLI_TIMEOUT_MS = 5_000
const DEFAULT_MAX_BUFFER = 32 * 1024 * 1024
// ---- built-in binding ---------------------------------------------------------------------------
interface StatementLike { all(...params: SqliteParam[]): SqliteRow[] }
interface DatabaseLike {
prepare(sql: string): StatementLike
exec(sql: string): void
close(): void
}
interface DatabaseConstructor { new (path: string | URL, options: { readOnly: boolean }): DatabaseLike }
/**
* A WAL store its engine is not using right now: its header says WAL (file format versions 2) and no
* `-wal` file is beside it. Opening one read-only would CREATE `-wal` and `-shm` in the engine's
* folder — and a handle held open keeps the engine from tidying them away when it next closes the
* store. Opened immutable it creates nothing, and nothing can be changing it: a writer would have
* made the `-wal` first. A store in the older journal mode is never opened immutable, since a writer
* may be changing it in place; it gets the ordinary locked read.
*/
export function idleWalStore(dbPath: string): boolean {
if (existsSync(`${dbPath}-wal`)) return false
let fd: number | null = null
try {
fd = openSync(dbPath, 'r')
const header = Buffer.alloc(20)
return readSync(fd, header, 0, 20, 0) === 20 && header[18] === 2 && header[19] === 2
} catch {
return false
} finally {
if (fd !== null) closeSync(fd)
}
}
let builtin: DatabaseConstructor | null | undefined
/**
* `node:sqlite` when this Node has it, else null. Resolved through `process.getBuiltinModule` (Node
* ≥ 20.16 / 22.3) rather than `import()`: synchronous, and invisible to the bundler, which must not
* try to resolve a module that only some runtimes have.
*
* The binding announces itself with an ExperimentalWarning on first load. That line would land in
* `harness.log` at every daemon boot, so it is filtered — only that one warning, and every other
* listener is kept.
*/
export function builtinSqlite(): DatabaseConstructor | null {
if (builtin !== undefined) return builtin
builtin = null
const get = (process as { getBuiltinModule?: (id: string) => unknown }).getBuiltinModule
if (typeof get !== 'function') return builtin
const previous = process.listeners('warning')
process.removeAllListeners('warning')
process.on('warning', (warning) => {
if (warning.name === 'ExperimentalWarning' && /SQLite/i.test(warning.message)) return
for (const listener of previous) listener.call(process, warning)
})
try {
const mod = get.call(process, 'node:sqlite') as { DatabaseSync?: unknown } | undefined
if (mod && typeof mod.DatabaseSync === 'function') builtin = mod.DatabaseSync as DatabaseConstructor
} catch {
builtin = null
}
return builtin
}
interface Handle { db: DatabaseLike; dev: number; ino: number; immutable: boolean; stamp: string }
/** One open handle per store: the readers poll every second, and opening is the expensive part. */
const handles = new Map<string, Handle>()
/**
* The cached handle, or a fresh one. Keyed by path but checked by inode: an engine that rebuilds its
* store (a reinstall, a `rm` and restart) leaves a handle on the OLD file, which would keep answering
* from a database nobody writes to any more — a poller that never sees another row, with no error.
*/
function openHandle(Database: DatabaseConstructor, dbPath: string, busyTimeoutMs: number): DatabaseLike {
const { dev, ino, size, mtimeMs, ctimeMs } = statSync(dbPath)
const stamp = `${size}:${mtimeMs}:${ctimeMs}`
// An idle store is read immutable; once its engine opens it again (a `-wal` appears), it is read live.
const immutable = idleWalStore(dbPath)
const cached = handles.get(dbPath)
// A writer can open, checkpoint, and close between polls, leaving no WAL. An immutable handle
// never checks for writes itself, so reuse it only while the main file's fingerprint holds.
if (cached && cached.dev === dev && cached.ino === ino && cached.immutable === immutable
&& (!immutable || cached.stamp === stamp)) return cached.db
if (cached) dropHandle(dbPath)
const url = pathToFileURL(dbPath)
url.search = 'immutable=1'
const db = new Database(immutable ? url : dbPath, { readOnly: true })
// Not a write: the pragma is per-connection state, accepted on a read-only handle.
db.exec(`PRAGMA busy_timeout = ${Math.max(0, Math.trunc(busyTimeoutMs))}`)
handles.set(dbPath, { db, dev, ino, immutable, stamp })
return db
}
function dropHandle(dbPath: string): void {
const handle = handles.get(dbPath)
handles.delete(dbPath)
try { handle?.db.close() } catch { /* already gone */ }
}
/** Test seam, and for a store that was replaced on disk: forget every open handle. */
export function closeSqliteHandles(): void {
for (const path of [...handles.keys()]) dropHandle(path)
}
function readBuiltin(
Database: DatabaseConstructor,
dbPath: string,
sql: string,
params: SqliteParam[],
busyTimeoutMs: number,
): SqliteReadResult {
try {
const db = openHandle(Database, dbPath, busyTimeoutMs)
return { ok: true, rows: db.prepare(sql).all(...params), via: 'builtin' }
} catch (err) {
// Locked, mid-checkpoint, the file replaced underneath the handle, not a database at all: every one
// of these is "try again later" for a poller, and a handle that failed is not trusted again — the
// next call reopens from the path.
dropHandle(dbPath)
return { ok: false, reason: 'transient', error: err instanceof Error ? err : new Error(String(err)) }
}
}
// ---- CLI fallback -------------------------------------------------------------------------------
/**
* The CLI takes no bound parameters, so the values are inlined as SQL literals: strings with their
* quotes doubled (the only escaping SQLite needs), numbers as finite decimals, null as NULL. A NUL
* byte cannot be carried in a literal and is refused rather than truncated. A `?` inside a string
* literal of the SQL itself (`json_extract(data, '$.a?')`) is left alone: only bare ones bind.
*/
export function inlineSqlParams(sql: string, params: SqliteParam[]): string {
let index = 0
const out = sql.replace(/'(?:[^']|'')*'|\?/g, (match) => {
if (match !== '?') return match
if (index >= params.length) throw new Error('sqlite: more placeholders than parameters')
const value = params[index++]
if (value === null) return 'NULL'
if (typeof value === 'number') {
if (!Number.isFinite(value)) throw new Error('sqlite: non-finite number parameter')
return String(value)
}
if (value.includes('\0')) throw new Error('sqlite: NUL byte in string parameter')
return `'${value.replace(/'/g, "''")}'`
})
if (index !== params.length) throw new Error('sqlite: more parameters than placeholders')
return out
}
/** A path as a `file:` URI body: the three characters SQLite's URI parser would otherwise read as syntax. */
function uriPath(path: string): string {
return path.replace(/[%?#]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase().padStart(2, '0')}`)
}
async function readCli(
dbPath: string,
sql: string,
params: SqliteParam[],
timeoutMs: number,
maxBuffer: number,
): Promise<SqliteReadResult> {
let inlined: string
try {
inlined = inlineSqlParams(sql, params)
} catch (err) {
return { ok: false, reason: 'transient', error: err as Error }
}
let stdout: string
try {
// `.timeout` is the SILENT dot-command form — `PRAGMA busy_timeout=…` prints a row under -json and
// would corrupt the single-array parse below. `query_only` is silent and guards against writes.
// `mode=ro` so a path that does not exist yet is an error, not a freshly created empty store.
;({ stdout } = await execFileAsync(
'sqlite3',
['-json', '-cmd', '.timeout 3000', '-cmd', 'PRAGMA query_only=1', `file:${uriPath(dbPath)}?mode=ro${idleWalStore(dbPath) ? '&immutable=1' : ''}`, inlined],
{ maxBuffer, timeout: timeoutMs, killSignal: 'SIGKILL' },
))
} catch (err) {
const error = err as NodeJS.ErrnoException
if (error?.code === 'ENOENT') return { ok: false, reason: 'missing', error }
return { ok: false, reason: 'transient', error }
}
const trimmed = stdout.trim()
if (!trimmed) return { ok: true, rows: [], via: 'cli' }
try {
const rows = JSON.parse(trimmed) as unknown
return { ok: true, rows: Array.isArray(rows) ? rows as SqliteRow[] : [], via: 'cli' }
} catch (err) {
return { ok: false, reason: 'transient', error: err as Error }
}
}
// ---- entry point --------------------------------------------------------------------------------
/**
* Run one read-only statement against `dbPath` and return its rows. `?` placeholders are bound from
* `params` in order — build the SQL as a constant and pass the values here, never paste them in.
*/
export async function sqliteReadAll(
dbPath: string,
sql: string,
params: SqliteParam[] = [],
options: SqliteReadOptions = {},
): Promise<SqliteReadResult> {
const Database = builtinSqlite()
if (Database) {
return readBuiltin(Database, dbPath, sql, params, options.busyTimeoutMs ?? DEFAULT_BUSY_TIMEOUT_MS)
}
return readCli(
dbPath,
sql,
params,
options.cliTimeoutMs ?? DEFAULT_CLI_TIMEOUT_MS,
options.maxBuffer ?? DEFAULT_MAX_BUFFER,
)
}
/** Test seam: force the CLI path (or the built-in one) regardless of what this Node has. */
export function overrideBuiltinSqlite(value: DatabaseConstructor | null | undefined): void {
builtin = value
}
51eb5328aThis file is pinned to the selected release. Download the original to open it in your design tool.