Loading files
442335385This file is pinned to the selected release. Download the original to open it in your design tool.
/**
* Cross-instance data bus over Redis pub/sub.
*
* Behind a load balancer a given agent's web socket(s) and its single manager socket can land on
* DIFFERENT backend instances. Redis bridges that gap (this is the ONLY cross-instance state — see
* docs/plans/2026-07-09_invert-ws-transport-backend-hub-redis.md):
*
* - up:{machineId} — node→client events. Published by the backend holding the manager socket (B_m);
* delivered by every backend with ≥1 local client for the agent (≤ M subscribers).
* - down:{machineId} — client→node messages. Published by any client-holding backend; delivered by the
* single backend holding the manager socket (exactly 1 subscriber, B_m).
* - mgr:{managerId} — control/provisioning commands to a specific manager (Phase 3), delivered by the
* backend holding that manager's socket.
*
* ioredis requires a connection in subscribe mode to be dedicated (it can't run other commands), so
* we keep one `sub` connection for SUBSCRIBE and one `pub` connection for PUBLISH/keys.
*/
import { Redis } from 'ioredis'
import { env } from '../config/env.js'
import { logger } from '../utils/logger.js'
import type { Frame, UpBusMsg, DownBusMsg } from './tunnel.js'
// A connection in subscriber mode must never fail its (re)SUBSCRIBE commands, so the sub side keeps
// ioredis' "retry forever" setting.
const subOpts = { maxRetriesPerRequest: null as null, lazyConnect: false }
// The pub/KV side is the opposite: a command that cannot reach Redis must FAIL FAST, not wait. With the
// default offline queue, every `publish` issued during a Redis outage (every up-frame from every manager
// socket, every terminal packet) was stringified and parked in this process' heap until reconnect, and
// every `await pub.set(...)` in an attach path hung — pinning its closure — for the whole outage.
const pubOpts = {
maxRetriesPerRequest: 1,
enableOfflineQueue: false,
commandTimeout: 5_000,
lazyConnect: false,
}
export const pub = new Redis(env.REDIS_URL, pubOpts)
const sub = new Redis(env.REDIS_URL, subOpts)
const terminalSub = new Redis(env.REDIS_URL, subOpts)
// App-proxy replies (`appup:{streamId}`) can be multi-MiB base64 bodies. Redis cuts a subscriber that
// exceeds `client-output-buffer-limit pubsub`; on the shared connection that took every chat / presence
// subscription of this instance down with it (ioredis re-subscribes, but everything in between is lost).
// Isolated here, a heavy app stream can only hurt app streams.
const appSub = new Redis(env.REDIS_URL, subOpts)
/** Every fan-out publish is best-effort: Redis unreachable → dropped with a warning, 0 subscribers.
* Callers already read 0 as "nobody holds this machine right now", which is exactly the truth. */
async function safePublish(channel: string, payload: string | Buffer): Promise<number> {
try {
return await pub.publish(channel, payload)
} catch (err) {
logger.warn('[bus] publish dropped', { channel, error: err instanceof Error ? err.message : String(err) })
return 0
}
}
pub.on('error', (err: Error) => logger.error('[bus] pub redis error', err))
sub.on('error', (err: Error) => logger.error('[bus] sub redis error', err))
terminalSub.on('error', (err: Error) => logger.error('[bus] terminal sub redis error', err))
appSub.on('error', (err: Error) => logger.error('[bus] app sub redis error', err))
export async function closeBus(): Promise<void> {
await Promise.allSettled([pub.quit(), sub.quit(), terminalSub.quit(), appSub.quit()])
}
type Cb = (msg: unknown) => void
/** One refcounted JSON subscriber registry per subscribe-mode connection. */
interface JsonSubscriber {
conn: Redis
channelSubs: Map<string, Set<Cb>>
channelSubscribePromises: Map<string, Promise<void>>
}
function createJsonSubscriber(conn: Redis, label: string): JsonSubscriber {
const registry: JsonSubscriber = { conn, channelSubs: new Map(), channelSubscribePromises: new Map() }
conn.on('message', (channel: string, payload: string) => {
const set = registry.channelSubs.get(channel)
if (!set || set.size === 0) return
let msg: unknown
try {
msg = JSON.parse(payload)
} catch (err) {
logger.error(`[bus] bad JSON on channel (${label})`, err, { channel })
return
}
for (const cb of set) {
try {
cb(msg)
} catch (err) {
logger.error(`[bus] subscriber callback threw (${label})`, err, { channel })
}
}
})
return registry
}
const mainSubscriber = createJsonSubscriber(sub, 'main')
const appSubscriber = createJsonSubscriber(appSub, 'app')
/** Refcounted subscribe: SUBSCRIBEs on the first callback for a channel, UNSUBSCRIBEs on the last. */
async function addSubOn(reg: JsonSubscriber, channel: string, cb: Cb): Promise<() => void> {
const { conn, channelSubs, channelSubscribePromises } = reg
let set = channelSubs.get(channel)
let subscribePromise = channelSubscribePromises.get(channel)
if (!set) {
set = new Set()
channelSubs.set(channel, set)
subscribePromise = conn.subscribe(channel)
.then(() => undefined)
.catch((err) => {
logger.error('[bus] subscribe failed', err, { channel })
if (channelSubs.get(channel) === set) channelSubs.delete(channel)
throw err
})
.finally(() => {
if (channelSubscribePromises.get(channel) === subscribePromise) channelSubscribePromises.delete(channel)
})
channelSubscribePromises.set(channel, subscribePromise)
}
set.add(cb)
if (subscribePromise) {
try {
await subscribePromise
} catch (err) {
const s = channelSubs.get(channel)
if (s === set) {
s.delete(cb)
if (s.size === 0) channelSubs.delete(channel)
}
throw err
}
}
return () => {
const s = channelSubs.get(channel)
if (!s) return
s.delete(cb)
if (s.size === 0) {
channelSubs.delete(channel)
conn.unsubscribe(channel).catch(() => { /* ignore */ })
}
}
}
const addSub = (channel: string, cb: Cb): Promise<() => void> => addSubOn(mainSubscriber, channel, cb)
const upChannel = (machineId: string): string => `up:${machineId}`
// Transition-only mirror of `up:`. The machine-list watchers (webWs.watchAgents / deviceWs.watchMachines)
// used to subscribe `up:{machineId}` for EVERY machine the user owns just to pick out node_status —
// which meant parsing and dispatching the full turn/delta stream of machines nobody was looking at.
const statusChannel = (machineId: string): string => `status:${machineId}`
const STATUS_TYPES = new Set(['node_status', 'machine_app_status'])
const downChannel = (machineId: string): string => `down:${machineId}`
const mgrChannel = (managerId: string): string => `mgr:${managerId}`
const appDownChannel = (machineId: string): string => `appdown:${machineId}`
const appUpChannel = (streamId: string): string => `appup:${streamId}`
const presenceKey = (machineId: string): string => `machine:${machineId}:mgr`
const terminalUpChannel = (machineId: string): string => `termup:${machineId}`
const terminalDownChannel = (machineId: string): string => `termdown:${machineId}`
type BinaryCb = (payload: Buffer) => void
const binarySubs = new Map<string, Set<BinaryCb>>()
terminalSub.on('messageBuffer', (channelRaw: Buffer, payload: Buffer) => {
const callbacks = binarySubs.get(channelRaw.toString('utf8'))
if (!callbacks) return
for (const callback of callbacks) {
try { callback(Buffer.from(payload)) } catch (err) { logger.error('[bus] binary subscriber callback threw', err) }
}
})
async function addBinarySub(channel: string, callback: BinaryCb): Promise<() => void> {
let callbacks = binarySubs.get(channel)
if (!callbacks) {
callbacks = new Set()
binarySubs.set(channel, callbacks)
await terminalSub.subscribe(channel)
}
callbacks.add(callback)
return () => {
const current = binarySubs.get(channel)
if (!current) return
current.delete(callback)
if (current.size === 0) {
binarySubs.delete(channel)
void terminalSub.unsubscribe(channel)
}
}
}
export function subscribeTerminalUp(machineId: string, callback: BinaryCb): Promise<() => void> {
return addBinarySub(terminalUpChannel(machineId), callback)
}
export function subscribeTerminalDown(machineId: string, callback: BinaryCb): Promise<() => void> {
return addBinarySub(terminalDownChannel(machineId), callback)
}
export function publishTerminalUp(machineId: string, payload: Uint8Array): Promise<number> {
return safePublish(terminalUpChannel(machineId), Buffer.from(payload))
}
export function publishTerminalDown(machineId: string, payload: Uint8Array): Promise<number> {
return safePublish(terminalDownChannel(machineId), Buffer.from(payload))
}
// ── up / down data channels ──────────────────────────────────────────────────────────────────────
export function subscribeUp(machineId: string, cb: (msg: UpBusMsg) => void): Promise<() => void> {
return addSub(upChannel(machineId), cb as Cb)
}
export function subscribeDown(machineId: string, cb: (msg: DownBusMsg) => void): Promise<() => void> {
return addSub(downChannel(machineId), cb as Cb)
}
/** A grant change invalidates observer sockets across backend instances. No content on this channel. */
export function subscribeShareChanged(id: string, cb: () => void): Promise<() => void> {
return addSub(`harness-share:${id}`, cb)
}
export function publishShareChanged(id: string): Promise<number> {
return safePublish(`harness-share:${id}`, '{}')
}
export function publishUp(machineId: string, msg: UpBusMsg): Promise<number> {
const type = (msg.frame as { type?: unknown } | undefined)?.type
if (typeof type === 'string' && STATUS_TYPES.has(type)) void safePublish(statusChannel(machineId), JSON.stringify(msg.frame))
return safePublish(upChannel(machineId), JSON.stringify(msg))
}
/** node_status / machine_app_status frames only (the bare frame, not the UpBusMsg envelope). */
export function subscribeStatus(machineId: string, cb: (frame: Frame) => void): Promise<() => void> {
return addSub(statusChannel(machineId), cb as Cb)
}
export function publishDown(machineId: string, msg: DownBusMsg): Promise<number> {
return safePublish(downChannel(machineId), JSON.stringify(msg))
}
// ── app-proxy tunnel channels ────────────────────────────────────────────────────────────────────
// App traffic is routed by MACHINEID (not managerId): an app belongs to a machine, and the machine's
// node manager socket has exactly ONE backend subscriber (B_m), so appdown:{machineId} has a single
// delivery path (no duplication under clustering) and rides the machine's sharded pool socket. Responses flow back
// on a per-STREAM channel so they reach the exact backend instance terminating the public request.
/** client→app frames (app_req, app_body, app_ws_open/msg/close, app_abort), delivered by B_m. */
export function subscribeAppDown(machineId: string, cb: (msg: unknown) => void): Promise<() => void> {
return addSub(appDownChannel(machineId), cb)
}
export function publishAppDown(machineId: string, msg: unknown): Promise<number> {
return safePublish(appDownChannel(machineId), JSON.stringify(msg))
}
/** app→client frames (app_res, app_res_body, app_ws_msg/close, app_abort) → the origin instance. */
export function subscribeAppUp(streamId: string, cb: (msg: unknown) => void): Promise<() => void> {
return addSubOn(appSubscriber, appUpChannel(streamId), cb)
}
export function publishAppUp(streamId: string, msg: unknown): Promise<number> {
return safePublish(appUpChannel(streamId), JSON.stringify(msg))
}
// ── manager command channel (Phase 3) ──────────────────────────────────────────────────────────
export function subscribeMgr(managerId: string, cb: (msg: unknown) => void): Promise<() => void> {
return addSub(mgrChannel(managerId), cb)
}
export function publishMgr(managerId: string, msg: unknown): Promise<number> {
return safePublish(mgrChannel(managerId), JSON.stringify(msg))
}
// Per-request reply channel for the provisioning RPC (backend → mgr:{managerId} → manager → reply).
const replyChannel = (requestId: string): string => `mgrreply:${requestId}`
export function subscribeReply(requestId: string, cb: (msg: unknown) => void): Promise<() => void> {
return addSub(replyChannel(requestId), cb)
}
export function publishReply(requestId: string, msg: unknown): Promise<number> {
return safePublish(replyChannel(requestId), JSON.stringify(msg))
}
// ── agent→manager presence (which manager currently owns an agent's node) ─────────────────────────
/** Set/refresh the presence key. Called on register + on each manager ping. */
export async function setAgentPresence(machineId: string, managerId: string, ttlSec = 30): Promise<void> {
try {
await pub.set(presenceKey(machineId), managerId, 'EX', ttlSec)
} catch (err) {
logger.error('[bus] setAgentPresence failed', err, { machineId })
}
}
export async function getAgentPresence(machineId: string): Promise<string | null> {
try {
return await pub.get(presenceKey(machineId))
} catch (err) {
logger.error('[bus] getAgentPresence failed', err, { machineId })
return null
}
}
/** One MGET for a whole machine list (the watchers seed N machines at once). Missing/failed → null. */
export async function getAgentPresenceMany(machineIds: string[]): Promise<Map<string, string | null>> {
const out = new Map<string, string | null>()
if (machineIds.length === 0) return out
try {
const vals = await pub.mget(...machineIds.map(presenceKey))
machineIds.forEach((id, i) => out.set(id, vals[i] ?? null))
} catch (err) {
logger.error('[bus] getAgentPresenceMany failed', err, { count: machineIds.length })
for (const id of machineIds) out.set(id, null)
}
return out
}
// A device's last-selected machine, so the backend can restore the active machine after a reconnect. A
// multi_machine device renders agents by attaching to every machine WITHOUT sending machine_select, so on a
// fresh connection `activeMachineId` would otherwise stay null and voice/RPCs fail with "no machine selected".
const deviceLastMachineKey = (deviceId: string): string => `devlastmachine:${deviceId}`
export async function setDeviceLastMachine(deviceId: string, machineId: string, ttlSec = 30 * 24 * 3600): Promise<void> {
try { await pub.set(deviceLastMachineKey(deviceId), machineId, 'EX', ttlSec) } catch (err) { logger.error('[bus] setDeviceLastMachine failed', err, { deviceId }) }
}
export async function getDeviceLastMachine(deviceId: string): Promise<string | null> {
try { return await pub.get(deviceLastMachineKey(deviceId)) } catch { return null }
}
// ── provider-machine recap cache (the device's tile text) ─────────────────────────────────────────
// Redis rather than a process Map because the backend runs FOUR cluster workers: the worker that ran
// the turn is usually not the one the device's `agent_recent` lands on, so an in-heap cache would
// answer "no recap" most of the time and every worker restart would blank the tiles. Only the
// provider path writes here — every other machine kind persists its own recap node-side.
// A whole small array under one key, not LPUSH/LTRIM: `n` is capped at 5 by the schema, so the value
// is tiny, and every other value in this file is a plain JSON `set`. Two turns finishing on the SAME
// agent at once can lose an entry in the read-modify-write; that is no worse than `activeTasks`,
// which is already one-per-machine, and a lost tile self-heals on the next turn.
const machineRecapKey = (machineId: string, agentId: string): string => `machine:${machineId}:recap:${agentId}`
const MACHINE_RECAP_KEEP = 5
export async function pushMachineRecap<T>(machineId: string, agentId: string, entry: T, ttlSec = 30 * 24 * 3600): Promise<void> {
if (!agentId) return
try {
const existing = await getMachineRecaps<T>(machineId, agentId, MACHINE_RECAP_KEEP)
const next = [entry, ...existing].slice(0, MACHINE_RECAP_KEEP)
await pub.set(machineRecapKey(machineId, agentId), JSON.stringify(next), 'EX', ttlSec)
} catch (err) {
// Best-effort: a recap that fails to cache costs a tile after a device reboot, never a turn.
logger.error('[bus] pushMachineRecap failed', err, { machineId, agentId })
}
}
export async function getMachineRecaps<T>(machineId: string, agentId: string, n: number): Promise<T[]> {
if (!agentId) return []
try {
const raw = await pub.get(machineRecapKey(machineId, agentId))
if (!raw) return []
const parsed = JSON.parse(raw) as unknown
return Array.isArray(parsed) ? (parsed.slice(0, n) as T[]) : []
} catch {
return []
}
}
// ── Resumable voice upload: park a partial live-stream so it survives a WS drop + reconnect ──────────
// Streaming voice has no local device copy; a WS blip mid-recording used to lose the whole (up to ~10-min)
// ramble. On a drop the backend PARKS the accumulated audio here keyed by the STABLE deviceId (not the
// per-connection closure), so a reconnect — which may land on a DIFFERENT backend instance (prod is
// multi-instance, non-sticky) — can `resumeVoiceUpload` and continue appending. Grace TTL ~60s. Written
// once per DROP (not per chunk). meta is the commit marker (set AFTER the pcm), so a torn write reads as
// absent. See docs/plans resumable-voice-upload.
export interface ParkedVoiceMeta {
uploadId: string
mode: 'turn' | 'ask'
route: boolean
goal: boolean
loop: boolean
autonomy: 'plan' | 'auto'
agentId: string | null
sessionId: string | null
requestId: string | null
sr: number
lang: string
size: number
machineId: string | null
}
const voiceUploadPcmKey = (deviceId: string): string => `voiceupload:${deviceId}:pcm`
const voiceUploadMetaKey = (deviceId: string): string => `voiceupload:${deviceId}:meta`
export async function parkVoiceUpload(deviceId: string, meta: ParkedVoiceMeta, pcm: Buffer, ttlSec = 60): Promise<void> {
try {
await pub.set(voiceUploadPcmKey(deviceId), pcm, 'EX', ttlSec) // binary value (ioredis Buffer)
await pub.set(voiceUploadMetaKey(deviceId), JSON.stringify(meta), 'EX', ttlSec) // commit marker last
} catch (err) { logger.error('[bus] parkVoiceUpload failed', err, { deviceId, size: pcm.length }) }
}
export async function resumeVoiceUpload(deviceId: string): Promise<{ meta: ParkedVoiceMeta; pcm: Buffer } | null> {
try {
const metaStr = await pub.get(voiceUploadMetaKey(deviceId))
if (!metaStr) return null
const pcm = await pub.getBuffer(voiceUploadPcmKey(deviceId))
if (!pcm) return null
return { meta: JSON.parse(metaStr) as ParkedVoiceMeta, pcm }
} catch (err) { logger.error('[bus] resumeVoiceUpload failed', err, { deviceId }); return null }
}
export async function evictVoiceUpload(deviceId: string): Promise<void> {
try { await pub.del(voiceUploadMetaKey(deviceId), voiceUploadPcmKey(deviceId)) } catch { /* best effort */ }
}
export async function clearAgentPresence(machineId: string): Promise<void> {
try {
await pub.del(presenceKey(machineId))
} catch (err) {
logger.error('[bus] clearAgentPresence failed', err, { machineId })
}
}
// ── one-computer-per-machine ownership ────────────────────────────────────────────────────────────
// A SEPARATE key from presence (which stays `machine:{id}:mgr` = 'remote' for the online/status path):
// `machine:{id}:owner` = the connecting COMPUTER's stable id. Atomic compare-and-set so a SECOND
// computer (different computerId) is rejected while the first holds the machine, but the SAME computer
// reconnecting after a blip/restart (its 30s key may still be alive) always reclaims. The value is the
// computerId — not a shared token — so the delete on teardown is conditional (a superseded/other socket
// can't free the real owner's claim). Mirrors the device-presence compare-and-delete pattern below.
const ownerKey = (machineId: string): string => `machine:${machineId}:owner`
/** Claim or refresh ownership. Returns true when this computer owns the machine afterwards (it was free
* or already ours), false when a DIFFERENT computer currently holds it. Also used as the refresh call. */
export async function claimMachineOwner(machineId: string, computerId: string, ttlSec = 30): Promise<boolean> {
try {
const res = await pub.eval(
"local cur = redis.call('get', KEYS[1]) " +
"if (not cur) or (cur == ARGV[1]) then redis.call('set', KEYS[1], ARGV[1], 'EX', ARGV[2]) return 1 else return 0 end",
1, ownerKey(machineId), computerId, String(ttlSec),
)
return res === 1
} catch (err) {
// On a Redis error, fail OPEN (allow) — losing the backend link should never lock a user out of
// their own machine; single-computer enforcement is best-effort liveness, not a security boundary.
logger.error('[bus] claimMachineOwner failed', err, { machineId })
return true
}
}
/** Conditional delete: only frees the claim if it still holds THIS computerId (compare-and-delete). */
export async function releaseMachineOwner(machineId: string, computerId: string): Promise<void> {
try {
await pub.eval(
"if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end",
1, ownerKey(machineId), computerId,
)
} catch (err) {
logger.error('[bus] releaseMachineOwner failed', err, { machineId })
}
}
// ── desktop-app state on a remote machine ─────────────────────────────────────────────────────────
// `machine:{id}:app` = "<engine>:<state>" — which desktop app this COMPUTER drives and whether it is
// running right now (see docs/specs/machine-adapter-cursor-desktop-integration.md). Written by the
// adapter socket, refreshed on the same 15s heartbeat that renews presence, and given the SAME 30s
// TTL as `machine:{id}:mgr` so the claim can never outlive the socket that made it: if a worker is
// killed without running cleanup, both keys expire together and the machine reads offline rather
// than "Cursor open" forever. Deliberately NOT a Prisma column — the value's useful life is 30s and
// a DB row would survive a crashed worker and lie.
const appStateKey = (machineId: string): string => `machine:${machineId}:app`
export async function setMachineAppState(
machineId: string, engine: string, state: string, ttlSec = 30,
): Promise<void> {
try {
await pub.set(appStateKey(machineId), `${engine}:${state}`, 'EX', ttlSec)
} catch (err) {
logger.error('[bus] setMachineAppState failed', err, { machineId })
}
}
/** null when unknown (no adapter, legacy adapter, or the key expired). */
export async function getMachineAppState(
machineId: string,
): Promise<{ engine: string; state: string } | null> {
try {
const raw = await pub.get(appStateKey(machineId))
if (!raw) return null
const sep = raw.indexOf(':')
if (sep <= 0 || sep === raw.length - 1) return null
return { engine: raw.slice(0, sep), state: raw.slice(sep + 1) }
} catch (err) {
logger.error('[bus] getMachineAppState failed', err, { machineId })
return null
}
}
function parseAppState(raw: string | null): { engine: string; state: string } | null {
if (!raw) return null
const sep = raw.indexOf(':')
if (sep <= 0 || sep === raw.length - 1) return null
return { engine: raw.slice(0, sep), state: raw.slice(sep + 1) }
}
/** MGET variant of getMachineAppState for the list seeds. */
export async function getMachineAppStateMany(machineIds: string[]): Promise<Map<string, { engine: string; state: string } | null>> {
const out = new Map<string, { engine: string; state: string } | null>()
if (machineIds.length === 0) return out
try {
const vals = await pub.mget(...machineIds.map(appStateKey))
machineIds.forEach((id, i) => out.set(id, parseAppState(vals[i] ?? null)))
} catch (err) {
logger.error('[bus] getMachineAppStateMany failed', err, { count: machineIds.length })
for (const id of machineIds) out.set(id, null)
}
return out
}
export async function clearMachineAppState(machineId: string): Promise<void> {
try {
await pub.del(appStateKey(machineId))
} catch (err) {
logger.error('[bus] clearMachineAppState failed', err, { machineId })
}
}
// ── device presence (is a paired device's socket currently connected?) ─────────────────────────────
// One key PER DEVICE (`device:{deviceId}:conn`) — a user with many devices has one independent key,
// refresh loop and supersede scope per device. The VALUE is the connection's own token so that when
// a device reconnects (new socket, possibly on another worker) the OLD socket's teardown cannot
// delete the NEW socket's key: the delete is conditional on the token still being ours.
const devicePresenceKey = (deviceId: string): string => `device:${deviceId}:conn`
const deviceStatusChannel = (userId: string): string => `devstatus:${userId}`
const deviceMachineListChannel = (userId: string): string => `devmachines:${userId}`
const deviceE2eePairChannel = (userId: string): string => `deve2eepair:${userId}`
/** Set/refresh this connection's presence. Called on device-ws open + every ~15s while connected. */
export async function setDevicePresence(deviceId: string, connToken: string, ttlSec = 45): Promise<void> {
try {
await pub.set(devicePresenceKey(deviceId), connToken, 'EX', ttlSec)
} catch (err) {
logger.error('[bus] setDevicePresence failed', err, { deviceId })
}
}
/** Conditional delete: only removes the key if it still holds THIS connection's token.
* Returns true when the key is gone afterwards (deleted by us or already absent). */
export async function clearDevicePresence(deviceId: string, connToken: string): Promise<boolean> {
try {
await pub.eval(
"if redis.call('get', KEYS[1]) == ARGV[1] then return redis.call('del', KEYS[1]) else return 0 end",
1, devicePresenceKey(deviceId), connToken,
)
return !(await pub.get(devicePresenceKey(deviceId)))
} catch (err) {
logger.error('[bus] clearDevicePresence failed', err, { deviceId })
return false
}
}
export async function getDevicePresence(deviceId: string): Promise<boolean> {
try {
return !!(await pub.get(devicePresenceKey(deviceId)))
} catch (err) {
logger.error('[bus] getDevicePresence failed', err, { deviceId })
return false
}
}
export interface DeviceStatusMsg { deviceId: string; online: boolean; lastSeenAt?: string }
/** Per-USER transition channel (one sub per watching web socket); messages are PER-DEVICE. */
export function publishDeviceStatus(userId: string, msg: DeviceStatusMsg): Promise<number> {
return safePublish(deviceStatusChannel(userId), JSON.stringify(msg))
}
export function subscribeDeviceStatus(userId: string, cb: (msg: DeviceStatusMsg) => void): Promise<() => void> {
return addSub(deviceStatusChannel(userId), cb as Cb)
}
// Per-USER machine-list invalidation for connected devices (and web machine-list watchers). Machine
// create/delete/rename happens on normal REST workers, while device/web sockets may live on another
// worker, so Redis fans out "re-query the DB now".
export interface DeviceMachineListChangedMsg {
reason: 'created' | 'deleted' | 'renamed' | 'updated' | 'environment_changed'
autonomousEnv?: 'prod' | 'stag'
}
export function publishDeviceMachineListChanged(userId: string, msg: DeviceMachineListChangedMsg): Promise<number> {
return safePublish(deviceMachineListChannel(userId), JSON.stringify(msg))
}
export function subscribeDeviceMachineListChanged(userId: string, cb: (msg: DeviceMachineListChangedMsg) => void): Promise<() => void> {
return addSub(deviceMachineListChannel(userId), cb as Cb)
}
export interface DeviceE2eePairMsg {
kind: 'pending' | 'cleared'
machineId: string
machineName?: string | null
label?: string
pairId: string
expiresAt?: number
computerFingerprint?: string | null
result?: 'paired' | 'failed' | 'cancelled'
}
export function publishDeviceE2eePair(userId: string, msg: DeviceE2eePairMsg): Promise<number> {
return safePublish(deviceE2eePairChannel(userId), JSON.stringify(msg))
}
export function subscribeDeviceE2eePair(userId: string, cb: (msg: DeviceE2eePairMsg) => void): Promise<() => void> {
return addSub(deviceE2eePairChannel(userId), cb as Cb)
}
// ── per-device control channel (backend → the one connected device socket, any worker) ────────────
// Unlike pushDeviceRevoked (rides up:{machineId}, reaches only a hub-ATTACHED commander conn — i.e. a
// device that has selected a machine), this reaches the device-ws relay itself, so a device parked on
// the machine PICKER is covered too.
const deviceControlChannel = (deviceId: string): string => `devctl:${deviceId}`
export interface DeviceControlMsg { action: 'revoked' | 'superseded'; by?: string }
export function publishDeviceControl(deviceId: string, msg: DeviceControlMsg): Promise<number> {
return safePublish(deviceControlChannel(deviceId), JSON.stringify(msg))
}
export function subscribeDeviceControl(deviceId: string, cb: (msg: DeviceControlMsg) => void): Promise<() => void> {
return addSub(deviceControlChannel(deviceId), cb as Cb)
}
// ── instance-affinity for the data-plane mesh (Phase 2) ────────────────────────────────────────────
// machine:{machineId}:appinst → the backend INSTANCE id holding that agent's app-pool manager socket.
// inst:{instanceId}:mesh → that instance's internal mesh endpoint (host:port). Both TTL-refreshed.
const appInstKey = (machineId: string): string => `machine:${machineId}:appinst`
const meshKey = (instanceId: string): string => `inst:${instanceId}:mesh`
export async function setAppInstance(machineId: string, instanceId: string, ttlSec = 30): Promise<void> {
try { await pub.set(appInstKey(machineId), instanceId, 'EX', ttlSec) } catch (err) { logger.error('[bus] setAppInstance failed', err, { machineId }) }
}
export async function getAppInstance(machineId: string): Promise<string | null> {
try { return await pub.get(appInstKey(machineId)) } catch (err) { logger.error('[bus] getAppInstance failed', err, { machineId }); return null }
}
export async function clearAppInstance(machineId: string): Promise<void> {
try { await pub.del(appInstKey(machineId)) } catch (err) { logger.error('[bus] clearAppInstance failed', err, { machineId }) }
}
export async function setMeshEndpoint(instanceId: string, endpoint: string, ttlSec = 30): Promise<void> {
try { await pub.set(meshKey(instanceId), endpoint, 'EX', ttlSec) } catch (err) { logger.error('[bus] setMeshEndpoint failed', err, { instanceId }) }
}
export async function getMeshEndpoint(instanceId: string): Promise<string | null> {
try { return await pub.get(meshKey(instanceId)) } catch (err) { logger.error('[bus] getMeshEndpoint failed', err, { instanceId }); return null }
}
// ── global per-agent client-count registry ─────────────────────────────────────────────────────────
// Drives the node's `__clients` aggregate lifecycle. `registry.clientCounts` is PER-PROCESS, but the
// node needs the CROSS-INSTANCE total (web on B1 + device on B2 must both count). We keep a HASH per
// agent, field = backend INSTANCE_ID → "ui,commander". Each field carries a per-field TTL (HEXPIRE,
// Redis ≥7.4) so a CRASHED instance's contribution ages out on its own — else the node would keep a
// commander/web aggregate warm for a dead client-holder. Live instances refresh on the hub heartbeat
// (25s < 45s TTL). Only B_m reads the sum and emits the frame (see hub.recomputeAndSendClients).
const clientsKey = (machineId: string): string => `machine:${machineId}:clients`
const commanderJoinGenerationKey = (machineId: string): string => `machine:${machineId}:commander-join-generation`
const CLIENT_COUNT_TTL_SEC = 45
/** Write THIS instance's local {ui,commander,commanderActive} for an agent (HDEL when zero), with a fresh
* per-field TTL. Field format is `ui,commander,commanderActive`; the trailing field is optional so an old
* instance's `ui,commander` still parses (missing → 0). */
export async function setAgentClientCount(machineId: string, instanceId: string, ui: number, commander: number, commanderActive = 0): Promise<void> {
const key = clientsKey(machineId)
try {
if (ui === 0 && commander === 0) { await pub.hdel(key, instanceId); return }
await pub.hset(key, instanceId, `${ui},${commander},${commanderActive}`)
} catch (err) {
logger.error('[bus] setAgentClientCount failed', err, { machineId, instanceId })
return
}
// Per-field expiry (Redis 7.4+): FIELDS 1 <field>. Refreshed each heartbeat; a dead instance's field
// simply expires. `pub.call` avoids depending on ioredis' typed hexpire signature. On Redis < 7.4
// (HEXPIRE unknown) fall back to a whole-key TTL — coarser (any live instance's heartbeat keeps the
// whole key warm), but enough that a crashed sole instance still ages out. The count was already
// written above, so a TTL failure must NOT surface as a "client count failed" error.
try {
await pub.call('HEXPIRE', key, String(CLIENT_COUNT_TTL_SEC), 'FIELDS', '1', instanceId)
} catch {
try { await pub.expire(key, CLIENT_COUNT_TTL_SEC) } catch { /* best-effort */ }
}
}
/** Heartbeat variant: every local agent's counts in ONE pipeline instead of 2 round trips per agent.
* Same field format and per-field TTL as setAgentClientCount; HEXPIRE failures fall back to a key TTL. */
export async function setAgentClientCountsBatch(
instanceId: string,
rows: Array<{ machineId: string; ui: number; commander: number; commanderActive: number }>,
): Promise<void> {
if (rows.length === 0) return
const p = pub.pipeline()
// Remember which pipeline result index each HEXPIRE lands at, so the fallback only re-expires the keys
// whose HEXPIRE actually failed (rather than every key on any single failure).
const hexpireByKey: Array<{ key: string; index: number }> = []
let cmdIndex = 0
for (const r of rows) {
const key = clientsKey(r.machineId)
if (r.ui === 0 && r.commander === 0) { p.hdel(key, instanceId); cmdIndex += 1; continue }
p.hset(key, instanceId, `${r.ui},${r.commander},${r.commanderActive}`)
p.call('HEXPIRE', key, String(CLIENT_COUNT_TTL_SEC), 'FIELDS', '1', instanceId)
hexpireByKey.push({ key, index: cmdIndex + 1 }) // hset is cmdIndex, HEXPIRE is the next command
cmdIndex += 2
}
let results: Array<[Error | null, unknown]> | null
try {
results = await p.exec()
} catch (err) {
logger.error('[bus] setAgentClientCountsBatch failed', err, { count: rows.length })
return
}
// A HEXPIRE error (Redis < 7.4) → coarser whole-key TTL, exactly like the single-row path. Only the
// keys whose own HEXPIRE errored need it; a null result set (shouldn't happen post-exec) falls back for all.
const failedKeys = results
? hexpireByKey.filter(({ index }) => results![index]?.[0]).map(({ key }) => key)
: hexpireByKey.map(({ key }) => key)
if (failedKeys.length > 0) {
const fb = pub.pipeline()
for (const key of failedKeys) fb.expire(key, CLIENT_COUNT_TTL_SEC)
try { await fb.exec() } catch { /* best-effort */ }
}
}
function sumClientFields(h: Record<string, string>): { ui: number; commander: number; commanderActive: number } {
let ui = 0
let commander = 0
let commanderActive = 0
for (const v of Object.values(h)) {
const [u, c, a] = String(v).split(',')
ui += parseInt(u, 10) || 0
commander += parseInt(c, 10) || 0
commanderActive += parseInt(a, 10) || 0
}
return { ui, commander, commanderActive }
}
/** Sum of every live instance's counts for an agent (expired fields already dropped Redis-side). */
export async function getAgentClientTotals(machineId: string): Promise<{ ui: number; commander: number; commanderActive: number }> {
try {
return sumClientFields(await pub.hgetall(clientsKey(machineId)))
} catch (err) {
logger.error('[bus] getAgentClientTotals failed', err, { machineId })
return { ui: 0, commander: 0, commanderActive: 0 }
}
}
/** Totals + commander join generation in one round trip (the `__clients` recompute reads both). */
export async function getAgentClientState(machineId: string): Promise<{
totals: { ui: number; commander: number; commanderActive: number }
commanderJoinGeneration: number | undefined
}> {
try {
const res = await pub.pipeline().hgetall(clientsKey(machineId)).get(commanderJoinGenerationKey(machineId)).exec()
const [hErr, h] = res?.[0] ?? [null, {}]
const [gErr, g] = res?.[1] ?? [null, null]
if (hErr) throw hErr
if (gErr) throw gErr
const generation = g == null ? undefined : Number(g)
return {
totals: sumClientFields((h ?? {}) as Record<string, string>),
commanderJoinGeneration: generation != null && Number.isSafeInteger(generation) && generation >= 0 ? generation : undefined,
}
} catch (err) {
logger.error('[bus] getAgentClientState failed', err, { machineId })
return { totals: { ui: 0, commander: 0, commanderActive: 0 }, commanderJoinGeneration: undefined }
}
}
/** Monotonic machine-wide generation. Unlike the aggregate count, this changes for EVERY commander attach,
* including a leave/rejoin that is coalesced to the same total before the node sees a count snapshot. */
export async function bumpCommanderJoinGeneration(machineId: string): Promise<number | undefined> {
try {
return await pub.incr(commanderJoinGenerationKey(machineId))
} catch (err) {
logger.error('[bus] bumpCommanderJoinGeneration failed', err, { machineId })
return undefined
}
}
export async function getCommanderJoinGeneration(machineId: string): Promise<number | undefined> {
try {
const raw = await pub.get(commanderJoinGenerationKey(machineId))
if (raw == null) return undefined
const generation = Number(raw)
return Number.isSafeInteger(generation) && generation >= 0 ? generation : undefined
} catch (err) {
logger.error('[bus] getCommanderJoinGeneration failed', err, { machineId })
return undefined
}
}