365 lines
12 KiB
TypeScript
365 lines
12 KiB
TypeScript
import { createHmac, timingSafeEqual } from 'crypto'
|
|
import { eventBus, type ServerEvent } from './event-bus'
|
|
import { logger } from './logger'
|
|
|
|
interface Webhook {
|
|
id: number
|
|
name: string
|
|
url: string
|
|
secret: string | null
|
|
events: string // JSON array
|
|
enabled: number
|
|
consecutive_failures?: number
|
|
}
|
|
|
|
interface DeliverOpts {
|
|
attempt?: number
|
|
parentDeliveryId?: number | null
|
|
allowRetry?: boolean
|
|
}
|
|
|
|
interface DeliveryResult {
|
|
success: boolean
|
|
status_code: number | null
|
|
response_body: string | null
|
|
error: string | null
|
|
duration_ms: number
|
|
delivery_id?: number
|
|
}
|
|
|
|
// Backoff schedule in seconds: 30s, 5m, 30m, 2h, 8h
|
|
const BACKOFF_SECONDS = [30, 300, 1800, 7200, 28800]
|
|
|
|
const MAX_RETRIES = parseInt(process.env.MC_WEBHOOK_MAX_RETRIES || '5', 10) || 5
|
|
|
|
// Map event bus events to webhook event types
|
|
const EVENT_MAP: Record<string, string> = {
|
|
'activity.created': 'activity', // Dynamically becomes activity.<type>
|
|
'notification.created': 'notification', // Dynamically becomes notification.<type>
|
|
'agent.status_changed': 'agent.status_change',
|
|
'audit.security': 'security', // Dynamically becomes security.<action>
|
|
'task.created': 'activity.task_created',
|
|
'task.updated': 'activity.task_updated',
|
|
'task.deleted': 'activity.task_deleted',
|
|
}
|
|
|
|
/**
|
|
* Compute the next retry delay in seconds, with ±20% jitter.
|
|
*/
|
|
export function nextRetryDelay(attempt: number): number {
|
|
const base = BACKOFF_SECONDS[Math.min(attempt, BACKOFF_SECONDS.length - 1)]
|
|
const jitter = base * 0.2 * (2 * Math.random() - 1) // ±20%
|
|
return Math.round(base + jitter)
|
|
}
|
|
|
|
/**
|
|
* Verify a webhook signature using constant-time comparison.
|
|
* Consumers can use this to validate incoming webhook deliveries.
|
|
*/
|
|
export function verifyWebhookSignature(
|
|
secret: string,
|
|
rawBody: string,
|
|
signatureHeader: string | null | undefined
|
|
): boolean {
|
|
if (!signatureHeader || !secret) return false
|
|
|
|
const expected = `sha256=${createHmac('sha256', secret).update(rawBody).digest('hex')}`
|
|
|
|
// Constant-time comparison
|
|
const sigBuf = Buffer.from(signatureHeader)
|
|
const expectedBuf = Buffer.from(expected)
|
|
|
|
if (sigBuf.length !== expectedBuf.length) {
|
|
// Compare expected against a dummy buffer of matching length to avoid timing leak
|
|
const dummy = Buffer.alloc(expectedBuf.length)
|
|
timingSafeEqual(expectedBuf, dummy)
|
|
return false
|
|
}
|
|
|
|
return timingSafeEqual(sigBuf, expectedBuf)
|
|
}
|
|
|
|
/**
|
|
* Subscribe to the event bus and fire webhooks for matching events.
|
|
* Called once during server initialization.
|
|
*/
|
|
export function initWebhookListener() {
|
|
eventBus.on('server-event', (event: ServerEvent) => {
|
|
const mapping = EVENT_MAP[event.type]
|
|
if (!mapping) return
|
|
|
|
// Build the specific webhook event type
|
|
let webhookEventType: string
|
|
if (mapping === 'activity' && event.data?.type) {
|
|
webhookEventType = `activity.${event.data.type}`
|
|
} else if (mapping === 'notification' && event.data?.type) {
|
|
webhookEventType = `notification.${event.data.type}`
|
|
} else if (mapping === 'security' && event.data?.action) {
|
|
webhookEventType = `security.${event.data.action}`
|
|
} else {
|
|
webhookEventType = mapping
|
|
}
|
|
|
|
// Also fire agent.error for error status specifically
|
|
const isAgentError = event.type === 'agent.status_changed' && event.data?.status === 'error'
|
|
|
|
fireWebhooksAsync(webhookEventType, event.data).catch((err) => {
|
|
logger.error({ err }, 'Webhook dispatch error')
|
|
})
|
|
|
|
if (isAgentError) {
|
|
fireWebhooksAsync('agent.error', event.data).catch((err) => {
|
|
logger.error({ err }, 'Webhook dispatch error')
|
|
})
|
|
}
|
|
})
|
|
}
|
|
|
|
/**
|
|
* Fire all matching webhooks for an event type (public for test endpoint).
|
|
*/
|
|
export function fireWebhooks(eventType: string, payload: Record<string, any>) {
|
|
fireWebhooksAsync(eventType, payload).catch((err) => {
|
|
logger.error({ err }, 'Webhook dispatch error')
|
|
})
|
|
}
|
|
|
|
async function fireWebhooksAsync(eventType: string, payload: Record<string, any>) {
|
|
let webhooks: Webhook[]
|
|
try {
|
|
// Lazy import to avoid circular dependency
|
|
const { getDatabase } = await import('./db')
|
|
const db = getDatabase()
|
|
webhooks = db.prepare(
|
|
'SELECT * FROM webhooks WHERE enabled = 1'
|
|
).all() as Webhook[]
|
|
} catch {
|
|
return // DB not ready or table doesn't exist yet
|
|
}
|
|
|
|
if (webhooks.length === 0) return
|
|
|
|
const matchingWebhooks = webhooks.filter((wh) => {
|
|
try {
|
|
const events: string[] = JSON.parse(wh.events)
|
|
return events.includes('*') || events.includes(eventType)
|
|
} catch {
|
|
return false
|
|
}
|
|
})
|
|
|
|
await Promise.allSettled(
|
|
matchingWebhooks.map((wh) => deliverWebhook(wh, eventType, payload, { allowRetry: true }))
|
|
)
|
|
}
|
|
|
|
/**
|
|
* Public wrapper for API routes (test endpoint, manual retry).
|
|
* Returns delivery result fields for the response.
|
|
*/
|
|
export async function deliverWebhookPublic(
|
|
webhook: Webhook,
|
|
eventType: string,
|
|
payload: Record<string, any>,
|
|
opts?: DeliverOpts
|
|
): Promise<DeliveryResult> {
|
|
return deliverWebhook(webhook, eventType, payload, opts ?? { allowRetry: false })
|
|
}
|
|
|
|
async function deliverWebhook(
|
|
webhook: Webhook,
|
|
eventType: string,
|
|
payload: Record<string, any>,
|
|
opts: DeliverOpts = {}
|
|
): Promise<DeliveryResult> {
|
|
const { attempt = 0, parentDeliveryId = null, allowRetry = true } = opts
|
|
|
|
const body = JSON.stringify({
|
|
event: eventType,
|
|
timestamp: Math.floor(Date.now() / 1000),
|
|
data: payload,
|
|
})
|
|
|
|
const headers: Record<string, string> = {
|
|
'Content-Type': 'application/json',
|
|
'User-Agent': 'MissionControl-Webhook/1.0',
|
|
'X-MC-Event': eventType,
|
|
}
|
|
|
|
// HMAC signature if secret is configured
|
|
if (webhook.secret) {
|
|
const sig = createHmac('sha256', webhook.secret).update(body).digest('hex')
|
|
headers['X-MC-Signature'] = `sha256=${sig}`
|
|
}
|
|
|
|
const start = Date.now()
|
|
let statusCode: number | null = null
|
|
let responseBody: string | null = null
|
|
let error: string | null = null
|
|
|
|
try {
|
|
const controller = new AbortController()
|
|
const timeout = setTimeout(() => controller.abort(), 10000)
|
|
|
|
const res = await fetch(webhook.url, {
|
|
method: 'POST',
|
|
headers,
|
|
body,
|
|
signal: controller.signal,
|
|
})
|
|
|
|
clearTimeout(timeout)
|
|
statusCode = res.status
|
|
responseBody = await res.text().catch(() => null)
|
|
if (responseBody && responseBody.length > 1000) {
|
|
responseBody = responseBody.slice(0, 1000) + '...'
|
|
}
|
|
} catch (err: any) {
|
|
error = err.name === 'AbortError' ? 'Timeout (10s)' : err.message
|
|
}
|
|
|
|
const durationMs = Date.now() - start
|
|
const success = statusCode !== null && statusCode >= 200 && statusCode < 300
|
|
let deliveryId: number | undefined
|
|
|
|
// Log delivery attempt and handle retry/circuit-breaker logic
|
|
try {
|
|
const { getDatabase } = await import('./db')
|
|
const db = getDatabase()
|
|
|
|
const insertResult = db.prepare(`
|
|
INSERT INTO webhook_deliveries (webhook_id, event_type, payload, status_code, response_body, error, duration_ms, attempt, is_retry, parent_delivery_id)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
`).run(
|
|
webhook.id,
|
|
eventType,
|
|
body,
|
|
statusCode,
|
|
responseBody,
|
|
error,
|
|
durationMs,
|
|
attempt,
|
|
attempt > 0 ? 1 : 0,
|
|
parentDeliveryId
|
|
)
|
|
deliveryId = Number(insertResult.lastInsertRowid)
|
|
|
|
// Update webhook last_fired
|
|
db.prepare(`
|
|
UPDATE webhooks SET last_fired_at = unixepoch(), last_status = ?, updated_at = unixepoch()
|
|
WHERE id = ?
|
|
`).run(statusCode ?? -1, webhook.id)
|
|
|
|
// Circuit breaker + retry scheduling (skip for test deliveries)
|
|
if (allowRetry) {
|
|
if (success) {
|
|
// Reset consecutive failures on success
|
|
db.prepare(`UPDATE webhooks SET consecutive_failures = 0 WHERE id = ?`).run(webhook.id)
|
|
} else {
|
|
// Increment consecutive failures
|
|
db.prepare(`UPDATE webhooks SET consecutive_failures = consecutive_failures + 1 WHERE id = ?`).run(webhook.id)
|
|
|
|
if (attempt < MAX_RETRIES - 1) {
|
|
// Schedule retry
|
|
const delaySec = nextRetryDelay(attempt)
|
|
const nextRetryAt = Math.floor(Date.now() / 1000) + delaySec
|
|
db.prepare(`UPDATE webhook_deliveries SET next_retry_at = ? WHERE id = ?`).run(nextRetryAt, deliveryId)
|
|
} else {
|
|
// Exhausted retries — trip circuit breaker
|
|
const wh = db.prepare(`SELECT consecutive_failures FROM webhooks WHERE id = ?`).get(webhook.id) as { consecutive_failures: number } | undefined
|
|
if (wh && wh.consecutive_failures >= MAX_RETRIES) {
|
|
db.prepare(`UPDATE webhooks SET enabled = 0, updated_at = unixepoch() WHERE id = ?`).run(webhook.id)
|
|
logger.warn({ webhookId: webhook.id, name: webhook.name }, 'Webhook circuit breaker tripped — disabled after exhausting retries')
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Prune old deliveries (keep last 200 per webhook)
|
|
db.prepare(`
|
|
DELETE FROM webhook_deliveries
|
|
WHERE webhook_id = ? AND id NOT IN (
|
|
SELECT id FROM webhook_deliveries WHERE webhook_id = ? ORDER BY created_at DESC LIMIT 200
|
|
)
|
|
`).run(webhook.id, webhook.id)
|
|
} catch (logErr) {
|
|
logger.error({ err: logErr, webhookId: webhook.id }, 'Webhook delivery logging/pruning failed')
|
|
}
|
|
|
|
return { success, status_code: statusCode, response_body: responseBody, error, duration_ms: durationMs, delivery_id: deliveryId }
|
|
}
|
|
|
|
/**
|
|
* Process pending webhook retries. Called by the scheduler.
|
|
* Picks up deliveries where next_retry_at has passed and re-delivers them.
|
|
*/
|
|
export async function processWebhookRetries(): Promise<{ ok: boolean; message: string }> {
|
|
try {
|
|
const { getDatabase } = await import('./db')
|
|
const db = getDatabase()
|
|
const now = Math.floor(Date.now() / 1000)
|
|
|
|
// Find deliveries ready for retry (limit batch to 50)
|
|
const pendingRetries = db.prepare(`
|
|
SELECT wd.id, wd.webhook_id, wd.event_type, wd.payload, wd.attempt,
|
|
w.id as w_id, w.name as w_name, w.url as w_url, w.secret as w_secret,
|
|
w.events as w_events, w.enabled as w_enabled, w.consecutive_failures as w_consecutive_failures
|
|
FROM webhook_deliveries wd
|
|
JOIN webhooks w ON w.id = wd.webhook_id AND w.enabled = 1
|
|
WHERE wd.next_retry_at IS NOT NULL AND wd.next_retry_at <= ?
|
|
LIMIT 50
|
|
`).all(now) as Array<{
|
|
id: number; webhook_id: number; event_type: string; payload: string; attempt: number
|
|
w_id: number; w_name: string; w_url: string; w_secret: string | null
|
|
w_events: string; w_enabled: number; w_consecutive_failures: number
|
|
}>
|
|
|
|
if (pendingRetries.length === 0) {
|
|
return { ok: true, message: 'No pending retries' }
|
|
}
|
|
|
|
// Clear next_retry_at immediately to prevent double-processing
|
|
const clearStmt = db.prepare(`UPDATE webhook_deliveries SET next_retry_at = NULL WHERE id = ?`)
|
|
for (const row of pendingRetries) {
|
|
clearStmt.run(row.id)
|
|
}
|
|
|
|
// Re-deliver each
|
|
let succeeded = 0
|
|
let failed = 0
|
|
for (const row of pendingRetries) {
|
|
const webhook: Webhook = {
|
|
id: row.w_id,
|
|
name: row.w_name,
|
|
url: row.w_url,
|
|
secret: row.w_secret,
|
|
events: row.w_events,
|
|
enabled: row.w_enabled,
|
|
consecutive_failures: row.w_consecutive_failures,
|
|
}
|
|
|
|
// Parse the original payload from the stored JSON body
|
|
let parsedPayload: Record<string, any>
|
|
try {
|
|
const parsed = JSON.parse(row.payload)
|
|
parsedPayload = parsed.data ?? parsed
|
|
} catch {
|
|
parsedPayload = {}
|
|
}
|
|
|
|
const result = await deliverWebhook(webhook, row.event_type, parsedPayload, {
|
|
attempt: row.attempt + 1,
|
|
parentDeliveryId: row.id,
|
|
allowRetry: true,
|
|
})
|
|
|
|
if (result.success) succeeded++
|
|
else failed++
|
|
}
|
|
|
|
return { ok: true, message: `Processed ${pendingRetries.length} retries (${succeeded} ok, ${failed} failed)` }
|
|
} catch (err: any) {
|
|
return { ok: false, message: `Webhook retry failed: ${err.message}` }
|
|
}
|
|
}
|