Loading files
d5b711565This file is pinned to the selected release. Download the original to open it in your design tool.
/** The Models popover's local lifecycle. Grid remains the hardware, catalog,
* download and process authority. A click is a durable daemon operation, never a chat task. */
import { createHash, randomUUID } from 'node:crypto'
import { readFile, readdir, stat, mkdir, rename, writeFile, statfs } from 'node:fs/promises'
import { basename, dirname, join } from 'node:path'
import { GridFleetRpc, type GridFleetResult } from './gridFleetRpc.js'
import { gridCredentialsPath } from './gridCredentials.js'
import { forgetGridModels } from './gridModels.js'
import { binaryOnPath } from './binaryOnPath.js'
const GiB = 1024 ** 3
const obj = (v: unknown): Record<string, any> => v && typeof v === 'object' && !Array.isArray(v) ? v : {}
const rows = (v: unknown): Record<string, any>[] => Array.isArray(v) ? v.map(obj) : []
const str = (v: unknown): string => typeof v === 'string' ? v : ''
const num = (v: unknown): number | undefined => typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : undefined
const key = (v: string): string => createHash('sha256').update(v).digest('hex').slice(0, 24)
const cleanName = (v: string): string => basename(v).replace(/\.gguf$/i, '').replace(/-GGUF$/i, '')
const validArg = (v: string): boolean => !!v && !v.startsWith('-') && !/[\x00-\x1f]/.test(v)
export interface LocalModel {
id: string; name: string; state: 'available' | 'downloaded' | 'running'
sizeBytes?: number; quant?: string; recommended?: boolean; canStart: boolean; canStop: boolean
tokensPerSecond?: number; requests?: number; windowSeconds?: number
operation?: ModelOperation
}
export interface ModelOperation {
id: string; modelId: string; action: 'start' | 'stop'; phase: 'running' | 'done' | 'failed'
stage: 'checking' | 'downloading' | 'starting' | 'verifying' | 'stopping'
progress?: number; error?: string; updatedAt: string
}
export interface LocalModelsSnapshot {
models: LocalModel[]; memoryBytes?: number; hardware?: string; error?: string
observedAt: string; busy: boolean
}
interface Candidate {
id: string; name: string; pull: string; file: string; files: string[]; size: number; quant: string; context: number
}
interface Owned { file: string; selector: string; aliases: string[]; nodeId: string; name: string; live: boolean; context: number; siblings: number }
interface Receipt { spec: 1; grid: string; operation?: ModelOperation }
interface Options {
stateDir: string; processEnv?: NodeJS.ProcessEnv
run?: (args: string[], output?: (chunk: string) => void, timeout?: number) => Promise<GridFleetResult>
request?: typeof fetch
}
/** One concrete, machine-fitted version per model. Non-chat and unprobed offline
* catalog rows are never called compatible. Split GGUF files stay one model. */
export function compatibleModels(raw: unknown): Candidate[] {
const seen = new Set<string>()
return rows(obj(raw).models).flatMap(row => {
const fit = obj(row.fit)
if (row.runnable !== true || !['text-generation', 'image-text-to-text'].includes(row.task) || row.format !== 'GGUF') return []
const version = rows(row.versions).find(v => v.version === fit.version)
const pull = str(version?.pull_spec), size = num(version?.size_bytes)
const id = str(row.repo_id), split = pull.indexOf(':')
if (!id || seen.has(id) || !validArg(pull) || split < 1 || !size || !num(fit.ctx)) return []
const file = basename(pull.slice(split + 1))
if (!file.toLowerCase().endsWith('.gguf')) return []
const files = (Array.isArray(version?.urls) ? version.urls : []).flatMap((url: unknown) => {
try { return [basename(decodeURIComponent(new URL(str(url)).pathname))] } catch { return [] }
})
seen.add(id)
return [{ id, name: cleanName(id), pull, file, files: files.length ? files : [file], size,
quant: str(fit.version), context: Math.min(16384, Math.floor(fit.ctx)) }]
})
}
export class LocalModels {
private readonly processEnv: NodeJS.ProcessEnv
private readonly home: string
private readonly run: NonNullable<Options['run']>
private readonly request: typeof fetch
private candidates: Candidate[] = []
private device: Record<string, any> = {}
private catalogAt = 0
private catalogError: string | undefined
private catalogPending?: Promise<void>
private listPending?: Promise<LocalModelsSnapshot>
private listGrid?: string
private cached?: { at: number; grid: string; value: LocalModelsSnapshot }
private active?: { grid: string; operation: ModelOperation; done: Promise<void> }
private receipt?: Receipt
private receiptScope?: string
private knownByGrid = new Map<string, Candidate[]>()
private blockers = new Map<string, string>()
constructor(private readonly options: Options) {
this.processEnv = options.processEnv ?? process.env
this.home = dirname(gridCredentialsPath(this.processEnv))
const rpc = new GridFleetRpc(this.processEnv)
this.run = options.run ?? ((args, output, timeout = 30_000) =>
rpc.run('local-models', randomUUID(), { args, timeoutMs: timeout, thinking: false }, output, 4 * 1024 * 1024))
this.request = options.request ?? fetch
}
private async json(args: string[]): Promise<any> {
const result = await this.run(args)
if (!result.ok) throw new Error('Models could not be checked. Try again.')
try { return JSON.parse(result.stdout) } catch { throw new Error('Models could not be checked. Try again.') }
}
/** The same authenticated catalog Grid uses, paged until every compatible row
* is included. Credentials never enter RPC replies, receipts, arguments or logs. */
private async catalog(device: Record<string, any>): Promise<unknown> {
const top = (await readFile(gridCredentialsPath(this.processEnv), 'utf8')).split(/^\s*\[/m)[0]
const value = (name: string): string => {
const match = new RegExp(`^\\s*${name}\\s*=\\s*("(?:[^"\\\\]|\\\\.)*"|'[^']*')\\s*$`, 'm').exec(top)
if (!match) return ''
try { return match[1][0] === '"' ? JSON.parse(match[1]) : match[1].slice(1, -1) } catch { return '' }
}
const token = value('session_token')
if (!token) throw new Error('Sign in to find models for this computer.')
const base = value('api_url') || this.processEnv.GRID_CONTROL_PLANE_URL || 'https://api-grid.autonomous.ai'
const url = new URL(`${base.replace(/\/$/, '')}/v1/grid/catalog`)
if (url.protocol !== 'https:' && !(url.protocol === 'http:' && ['localhost', '127.0.0.1', '[::1]'].includes(url.hostname))) {
throw new Error('The model catalog address is unavailable.')
}
const all: any[] = []
for (let page = 1; page <= 100; page++) {
const response = await this.request(url, {
method: 'POST', headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify({ browse: true, page, page_size: 50, device: {
device_class: device.device_class, usable_bytes: device.usable_bytes, backend: device.backend,
} }), signal: AbortSignal.timeout(20_000), redirect: 'error',
})
if (!response.ok) throw new Error('Compatible models are unavailable. Try again.')
const body = obj(await response.json())
if (!Array.isArray(body.models)) throw new Error('Compatible models are unavailable. Try again.')
all.push(...body.models)
const totalPages = num(obj(body.pagination).total_pages) ?? 1
if (page >= totalPages || (num(body.runnable_total) !== undefined && all.filter(m => m.runnable === true).length >= body.runnable_total)) return { models: all }
if (!body.models.length || obj(body.pagination).page !== page) throw new Error('The model catalog is incomplete. Try again.')
}
throw new Error('The model catalog is incomplete. Try again.')
}
private async loadCatalog(force = false): Promise<void> {
if (!force && this.catalogAt && Date.now() - this.catalogAt < 5 * 60_000) return
return this.catalogPending ??= (async () => {
try {
this.device = obj(await this.json(['device-info', '--json']))
const catalog = obj(await this.catalog(this.device))
this.candidates = compatibleModels(catalog)
// Prefer an already downloaded fitting quant rather than downloading
// the catalog's default version of the same model again.
for (let i = 0; i < this.candidates.length; i++) {
const chosen = this.candidates[i]
if (await this.downloaded(chosen)) continue
const source = rows(catalog.models).find(m => m.repo_id === chosen.id)
for (const version of rows(source?.versions)) {
if ((num(version.size_bytes) ?? Infinity) > chosen.size) continue
const alternate = compatibleModels({ models: [{ ...source, fit: { ...source?.fit, version: version.version } }] })[0]
if (alternate && await this.downloaded(alternate)) { this.candidates[i] = alternate; break }
}
}
// A useful small download makes the first reply arrive sooner. Keep
// the catalog's relevance order within that group, and show all others.
const suggested = this.candidates.findIndex(c => c.size <= 8 * GiB && c.size >= GiB)
if (suggested > 0) this.candidates.unshift(...this.candidates.splice(suggested, 1))
this.catalogError = undefined
this.catalogAt = Date.now()
} catch (error) {
this.catalogError = error instanceof Error && /^(Sign in|The model catalog)/.test(error.message)
? error.message : 'Compatible models are unavailable. Try again.'
}
})().finally(() => { this.catalogPending = undefined })
}
private async downloaded(candidate: Candidate): Promise<boolean> {
const sizes = await Promise.all(candidate.files.map(file => stat(join(this.home, 'models', file)).then(s => s.isFile() ? s.size : 0).catch(() => 0)))
return sizes.every(size => size > 0) && sizes.reduce((sum, size) => sum + size, 0) === candidate.size
}
/** Grid's non-secret run records distinguish locally owned --serve processes
* from external endpoints. Stop delegates identity checks and teardown to Grid. */
private async owned(grid: string): Promise<Owned[]> {
this.blockers.delete(grid)
if (!validArg(grid)) return []
const grids = rows(await this.json(['--remote', 'ls', '--json']))
const gridId = str(grids.find(g => g.grid === grid)?.id)
if (!gridId || basename(gridId) !== gridId) return []
const folder = join(this.home, 'run', 'engines', gridId)
const names = await readdir(folder).catch(() => [])
const result: Owned[] = []
for (const name of names.filter(n => n.endsWith('.json'))) {
let record: Record<string, any>
try { record = obj(JSON.parse(await readFile(join(folder, name), 'utf8'))) } catch { continue }
const heartbeat = await stat(join(folder, name.replace(/\.json$/, '.heartbeat'))).catch(() => null)
const live = heartbeat !== null && Date.now() - heartbeat.mtimeMs < 90_000
const specs = rows(record.engines)
if (specs.length || record.media) this.blockers.set(grid, 'Open Model Manager to add a model alongside those already running.')
for (const spec of specs) {
if (spec.endpoint_url || spec.api_kind || !Array.isArray(spec.models) || spec.models.length !== 1) continue
const file = str(spec.models[0])
if (!validArg(file) || !file.toLowerCase().endsWith('.gguf')) continue
const advertised = Array.isArray(record.advertise_as) && specs.length === 1
? record.advertise_as.filter((v: unknown) => typeof v === 'string' && v.length > 0) : []
const aliases: string[] = advertised.length ? advertised : [file]
result.push({ file: basename(file), selector: file, aliases, nodeId: str(record.node_id), name: str(record.meta_name), live, context: Math.min(16384, num(record.ctx_size) || 8192), siblings: specs.length + (record.media ? 1 : 0) })
this.blockers.set(grid, `Stop ${cleanName(aliases[0])} first to start another local model.`)
}
}
return result
}
/** Keep models imported from an existing Grid setup after Stop removes its
* run record. Only completed local files that this computer already served
* become restartable; arbitrary downloaded GGUFs are not assumed compatible. */
private async known(grid: string, owned: Owned[]): Promise<Candidate[]> {
const path = join(this.options.stateDir, `${key(grid)}.known.json`)
let known = this.knownByGrid.get(grid)
if (!known) {
try {
known = rows(JSON.parse(await readFile(path, 'utf8'))).filter(c =>
c.id === `local:${c.file}` && basename(str(c.file)) === c.file && validArg(c.file) &&
c.pull === '' && typeof c.name === 'string' && c.name.length < 256 && num(c.size) && num(c.context) && c.context <= 16384 &&
Array.isArray(c.files) && c.files.length === 1 && c.files[0] === c.file) as Candidate[]
} catch { known = [] }
this.knownByGrid.set(grid, known)
}
let changed = false
for (const instance of owned) {
if (this.candidates.some(c => c.file === instance.file) || known.some(c => c.file === instance.file)) continue
const file = await stat(join(this.home, 'models', instance.file)).catch(() => null)
if (!file?.isFile() || !file.size) continue
known.push({ id: `local:${instance.file}`, name: cleanName(instance.aliases[0]),
file: instance.file, files: [instance.file], pull: '', size: file.size, quant: '', context: instance.context })
changed = true
}
if (changed) {
await mkdir(this.options.stateDir, { recursive: true, mode: 0o700 })
const temp = `${path}.${randomUUID()}.tmp`
await writeFile(temp, JSON.stringify(known), { mode: 0o600 }); await rename(temp, path)
}
const existing = await Promise.all(known.map(async candidate => await this.downloaded(candidate) ? candidate : null))
return existing.filter((c): c is Candidate => c !== null)
}
private receiptPath(grid: string): string { return join(this.options.stateDir, `${key(grid)}.json`) }
private async readReceipt(grid: string): Promise<void> {
if (this.receiptScope === grid) return
this.receiptScope = grid
this.receipt = undefined
try {
const value = JSON.parse(await readFile(this.receiptPath(grid), 'utf8')) as Receipt
if (value.spec === 1 && value.grid === grid) {
this.receipt = value
if (value.operation?.phase === 'running' && this.active?.operation.id !== value.operation.id) {
value.operation.phase = 'failed'
value.operation.error = 'Setup was interrupted. Start again to continue.'
}
}
} catch { /* no prior operation */ }
}
private async save(grid: string, operation: ModelOperation): Promise<void> {
const value: Receipt = { spec: 1, grid, operation: { ...operation, updatedAt: new Date().toISOString() } }
this.receipt = value; this.receiptScope = grid; this.cached = undefined
await mkdir(this.options.stateDir, { recursive: true, mode: 0o700 })
const file = this.receiptPath(grid), temp = `${file}.${randomUUID()}.tmp`
await writeFile(temp, JSON.stringify(value), { mode: 0o600 })
await rename(temp, file)
}
async list(grid: string | null, force = false): Promise<LocalModelsSnapshot> {
if (!grid) return { models: [], error: 'Sign in to find models for this computer.', observedAt: new Date().toISOString(), busy: false }
if (!force && this.cached?.grid === grid && Date.now() - this.cached.at < 2500) return this.cached.value
if (this.listPending) {
if (this.listGrid === grid) return this.listPending
await this.listPending; return this.list(grid, force)
}
this.listGrid = grid
this.listPending = this.readList(grid, force)
try { return await this.listPending } finally { this.listPending = undefined }
}
private async readList(grid: string, force: boolean): Promise<LocalModelsSnapshot> {
await Promise.all([this.loadCatalog(force), this.readReceipt(grid)])
let owned: Owned[] = [], nodes: Record<string, any>[] = [], inventoryError: string | undefined
try {
[owned, nodes] = await Promise.all([this.owned(grid), this.json(['--remote', 'engines', grid, '--json']).then(rows)])
} catch { inventoryError = 'Running models could not be checked. Try again.' }
const operation = this.active?.grid === grid ? this.active.operation : this.receipt?.grid === grid ? this.receipt.operation : undefined
const choices = [...this.candidates, ...(await this.known(grid, owned)).filter(k => !this.candidates.some(c => c.file === k.file))]
const servingNode = (instance?: Owned): Record<string, any> | undefined => {
const matches = instance?.live ? nodes.filter(n => n.online === true &&
(str(n.node_id || n.id) ? str(n.node_id || n.id) === instance.nodeId :
!!instance.name && str(n.name) === instance.name) &&
instance.aliases.every(alias => (Array.isArray(n.models) ? n.models : []).some((m: unknown) =>
str(typeof m === 'string' ? m : obj(m).model).toLowerCase() === alias.toLowerCase()))) : []
return matches.length === 1 ? matches[0] : undefined
}
const models = await Promise.all(choices.map(async (candidate, index): Promise<LocalModel> => {
const instance = owned.find(o => o.file === candidate.file)
const node = servingNode(instance)
const running = !!node
const available = await this.downloaded(candidate)
const answered = obj(node?.answered)
const perModel = rows(answered.by_model).find(a => instance?.aliases.some(alias => alias.toLowerCase() === str(a.model).toLowerCase()))
const single = Array.isArray(node?.models) && node.models.length === 1
return { id: candidate.id, name: candidate.name, state: running ? 'running' : available ? 'downloaded' : 'available',
sizeBytes: candidate.size, quant: candidate.quant, recommended: index === 0,
canStart: !inventoryError && (!this.catalogError || !candidate.pull) && !instance, canStop: !inventoryError && !!instance,
// Device memory is deliberately not presented as this model's memory.
tokensPerSecond: single && running ? num(node?.throughput_tok_s) : undefined,
requests: running ? num(perModel?.requests) : undefined,
windowSeconds: running ? num(answered.window_seconds) : undefined,
operation: operation?.modelId === candidate.id ? operation : undefined }
}))
for (const instance of owned.filter(o => !choices.some(c => c.file === o.file))) {
models.push({ id: `local:${instance.file}`, name: cleanName(instance.aliases[0]),
state: servingNode(instance) ? 'running' : 'available', canStart: false, canStop: !inventoryError,
operation: operation?.modelId === `local:${instance.file}` ? operation : undefined })
}
const value: LocalModelsSnapshot = { models, memoryBytes: num(obj(this.device.memory).total_gb) === undefined ? undefined : this.device.memory.total_gb * GiB,
hardware: str(obj(this.device.machine).model || obj(this.device.cpu).brand), error: inventoryError || this.catalogError,
observedAt: new Date().toISOString(), busy: !!this.active }
this.cached = { grid, at: Date.now(), value }
return value
}
/** A repeated click or lost RPC acknowledgement joins the same operation.
* A different model waits: starts/stops must not race on the machine's budget. */
async act(grid: string | null, modelId: unknown, action: 'start' | 'stop'): Promise<{ operation?: ModelOperation; error?: string }> {
if (!grid || typeof modelId !== 'string') return { error: 'The model is unavailable. Refresh and try again.' }
if (this.active) return this.active.grid === grid && this.active.operation.modelId === modelId && this.active.operation.action === action
? { operation: this.active.operation } : { error: 'Wait for the current model to finish starting or stopping.' }
const operation: ModelOperation = { id: randomUUID(), modelId, action, stage: action === 'start' ? 'checking' : 'stopping', phase: 'running', updatedAt: new Date().toISOString() }
// Reserve before any await; independent RPCs may arrive on separate connections.
const active = { grid, operation, done: Promise.resolve() }
this.active = active
try { await this.save(grid, operation) } catch { this.active = undefined; return { error: 'Setup could not be saved. Try again.' } }
active.done = this.perform(grid, operation).catch(async error => {
operation.phase = 'failed'
operation.error = error instanceof ModelError ? error.message : 'The model could not finish. Start again to retry.'
await this.save(grid, operation).catch(() => {})
}).finally(() => { this.active = undefined; this.cached = undefined; forgetGridModels() })
return { operation }
}
async settled(): Promise<void> { await this.active?.done }
private async perform(grid: string, operation: ModelOperation): Promise<void> {
const change = async (stage: ModelOperation['stage']) => {
operation.stage = stage; delete operation.progress; await this.save(grid, operation)
}
const must = async (args: string[], message: string, output?: (chunk: string) => void) => {
if (!(await this.run(args, output, 30 * 60_000)).ok) throw new ModelError(message)
}
if (operation.action === 'start') await this.loadCatalog()
const owned = await this.owned(grid)
const candidate = [...this.candidates, ...await this.known(grid, owned)].find(c => c.id === operation.modelId)
const instance = owned.find(o => candidate ? o.file === candidate.file : operation.modelId === `local:${o.file}`)
if (operation.action === 'stop') {
if (instance) {
if (instance.siblings > 1) throw new ModelError('Open Model Manager to stop this model. Other models share its engine.')
await must(['--remote', 'leave', grid, '--engine', instance.selector], 'The model could not stop. Try again.')
if ((await this.owned(grid)).some(o => o.file === instance.file)) throw new ModelError('The model is still stopping. Check again in a moment.')
}
} else {
if (!candidate || (this.catalogError && candidate.pull)) throw new ModelError('This model could not be checked. Refresh and try again.')
if (!instance) {
// The pinned Grid runtime respawns its union when adding --serve.
// Do not interrupt existing engines as a side effect of a simple Start.
if (this.blockers.has(grid)) throw new ModelError(this.blockers.get(grid)!)
// Recheck the machine immediately before downloading or allocating.
const currentDevice = obj(await this.json(['device-info', '--json']))
const budget = num(currentDevice.usable_bytes) ?? 0
if (candidate.pull) {
const current = compatibleModels(await this.catalog({ ...currentDevice, usable_bytes: Math.max(0, budget) })).find(c => c.id === candidate.id)
if (!current || current.size < candidate.size) throw new ModelError('Stop a running model to make room, then try again.')
candidate.context = Math.min(candidate.context, current.context)
} else if (candidate.size * 1.25 + 2 * GiB > budget) {
throw new ModelError('Stop a running model to make room, then try again.')
}
if (!await this.downloaded(candidate)) {
if (!candidate.pull) throw new ModelError('The downloaded file is no longer available. Open Model Manager to restore it.')
await mkdir(join(this.home, 'models'), { recursive: true })
const disk = await statfs(join(this.home, 'models'))
if (disk.bavail * disk.bsize < candidate.size + GiB) throw new ModelError('Free up disk space, then start again.')
await change('downloading')
let last = 0, progressWrites = Promise.resolve()
await must(['pull', candidate.pull], 'The download stopped. Start again to resume.', chunk => {
const matches = [...chunk.matchAll(/(\d+(?:\.\d+)?)\s*%/g)]
const percent = matches.length ? Number(matches.at(-1)![1]) : NaN
if (Number.isFinite(percent) && percent >= 0 && percent <= 100 && Date.now() - last > 500) {
last = Date.now(); operation.progress = percent / 100
progressWrites = progressWrites.then(() => this.save(grid, operation)).catch(() => {})
}
})
await progressWrites
if (!await this.downloaded(candidate)) throw new ModelError('The download is incomplete. Start again to resume.')
}
await change('starting')
const override = this.processEnv.LLAMA_SERVER
const installed = override ? binaryOnPath(override, this.processEnv)
: binaryOnPath(join(this.home, 'bin', 'llama-server'), this.processEnv) || binaryOnPath('llama-server', this.processEnv)
if (!installed) {
if (override) throw new ModelError('The local engine needs attention. Open Model Manager.')
await must(['engine', 'install', 'llama.cpp'], 'The model engine could not start. Try again.')
}
await must(['--remote', 'join', grid, '--serve', candidate.file,
'--max-concurrency', '1', '--ctx-size', String(candidate.context), '--reasoning-budget', '0'],
'The model could not start. Try again.')
}
await change('verifying')
await this.verify(grid, instance?.aliases[0] || candidate.file)
}
operation.phase = 'done'
await this.save(grid, operation)
}
private async verify(grid: string, model: string): Promise<void> {
const info = await this.run(['--remote', 'info', grid, '--env'])
const exports: Record<string, string> = {}
for (const match of info.stdout.matchAll(/^export\s+(OPENAI_BASE_URL|OPENAI_API_KEY)=(.*)$/gm)) exports[match[1]] = match[2].trim().replace(/^["']|["']$/g, '')
if (!info.ok || !exports.OPENAI_BASE_URL || !exports.OPENAI_API_KEY) throw new ModelError('The model is starting, but could not be checked. Try again shortly.')
const deadline = Date.now() + 180_000
do {
try {
const response = await this.request(`${exports.OPENAI_BASE_URL.replace(/\/$/, '')}/chat/completions`, {
method: 'POST', headers: { authorization: `Bearer ${exports.OPENAI_API_KEY}`, 'content-type': 'application/json' },
body: JSON.stringify({ model, messages: [{ role: 'user', content: 'Reply with the single word: ok' }], max_tokens: 8 }),
signal: AbortSignal.timeout(30_000), redirect: 'error',
})
const body = response.ok ? obj(await response.json()) : {}
if (/^\s*ok[.!]?\s*$/i.test(str(obj(rows(body.choices)[0]?.message).content))) return
} catch { /* registration and first warmup take time; status remains verifying */ }
await new Promise(resolve => setTimeout(resolve, 1500))
} while (Date.now() < deadline)
throw new ModelError('The model did not answer. Stop it, then start again.')
}
}
class ModelError extends Error {}