1681 lines
60 KiB
Go
1681 lines
60 KiB
Go
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"database/sql"
|
|
"encoding/json"
|
|
"fmt"
|
|
"html/template"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
_ "modernc.org/sqlite"
|
|
)
|
|
|
|
const (
|
|
fireworksKey = "fw_RVcDe4c6mN4utKLsgA7hTm"
|
|
fireworksModel = "accounts/fireworks/models/gpt-oss-20b"
|
|
fireworksURL = "https://api.fireworks.ai/inference/v1/chat/completions"
|
|
)
|
|
|
|
var systemPrompt = `You are the Vault1984 NOC assistant. Vault1984 is a global network of security POPs (Points of Presence) across 21 regions.
|
|
|
|
Your job is to:
|
|
- Help users report network problems or outages
|
|
- Answer questions about Vault1984 infrastructure status
|
|
- Collect details about issues (region, symptoms, timestamps)
|
|
- Be concise and professional
|
|
|
|
If someone reports a problem, acknowledge it, collect: affected region, what they observed, and when it started. Keep responses short.`
|
|
|
|
const version = "v0.4"
|
|
|
|
const (
|
|
colorGreen = "#76AD2A"
|
|
colorOrange = "#E86235"
|
|
colorRed = "#E04343"
|
|
colorGray = "#D8D6D0"
|
|
)
|
|
|
|
// Schema
|
|
const schema = `
|
|
CREATE TABLE IF NOT EXISTS nodes (
|
|
id TEXT PRIMARY KEY,
|
|
name TEXT NOT NULL,
|
|
region TEXT NOT NULL,
|
|
ip TEXT NOT NULL DEFAULT '',
|
|
status TEXT NOT NULL DEFAULT 'planned'
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS uptime (
|
|
node_id TEXT NOT NULL,
|
|
date TEXT NOT NULL, -- YYYY-MM-DD
|
|
status TEXT NOT NULL, -- operational | degraded | outage | planned
|
|
PRIMARY KEY (node_id, date)
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS incidents (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
title TEXT NOT NULL,
|
|
status TEXT NOT NULL, -- resolved | monitoring | investigating
|
|
date TEXT NOT NULL, -- display date e.g. "Mar 4, 2026"
|
|
node_ids TEXT NOT NULL DEFAULT '',
|
|
created_at INTEGER NOT NULL DEFAULT (strftime('%s','now'))
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS incident_updates (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
incident_id INTEGER NOT NULL REFERENCES incidents(id),
|
|
ts TEXT NOT NULL, -- ISO8601 e.g. "2026-03-04 16:08 UTC"
|
|
status TEXT NOT NULL, -- investigating | identified | monitoring | resolved
|
|
body TEXT NOT NULL
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS telemetry (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
node_id TEXT NOT NULL,
|
|
received_at INTEGER NOT NULL DEFAULT (strftime('%s','now')),
|
|
version TEXT NOT NULL DEFAULT '',
|
|
hostname TEXT NOT NULL DEFAULT '',
|
|
uptime_seconds INTEGER NOT NULL DEFAULT 0,
|
|
cpu_percent REAL NOT NULL DEFAULT 0,
|
|
memory_total_mb INTEGER NOT NULL DEFAULT 0,
|
|
memory_used_mb INTEGER NOT NULL DEFAULT 0,
|
|
disk_total_mb INTEGER NOT NULL DEFAULT 0,
|
|
disk_used_mb INTEGER NOT NULL DEFAULT 0,
|
|
load_1m REAL NOT NULL DEFAULT 0,
|
|
vault_count INTEGER NOT NULL DEFAULT 0,
|
|
vault_size_mb REAL NOT NULL DEFAULT 0,
|
|
vault_entries INTEGER NOT NULL DEFAULT 0,
|
|
mode TEXT NOT NULL DEFAULT ''
|
|
);
|
|
`
|
|
|
|
type Node struct {
|
|
ID string
|
|
Name string
|
|
Region string
|
|
IP string
|
|
Status string
|
|
History []DayStatus
|
|
}
|
|
|
|
type DayStatus struct {
|
|
Date string
|
|
Status string
|
|
}
|
|
|
|
type IncidentUpdate struct {
|
|
TS string
|
|
Status string
|
|
Body string
|
|
}
|
|
|
|
type Incident struct {
|
|
ID int64
|
|
Title string
|
|
Status string
|
|
Date string
|
|
NodeIDs string
|
|
Updates []IncidentUpdate
|
|
}
|
|
|
|
type PageData struct {
|
|
UpdatedAt string
|
|
OverallOK bool
|
|
Nodes []*Node
|
|
Incidents []Incident
|
|
}
|
|
|
|
func (n *Node) UptimePct() string {
|
|
if n.Status == "planned" {
|
|
return "—"
|
|
}
|
|
total, ok := 0, 0
|
|
for _, h := range n.History {
|
|
if h.Status == "planned" || h.Status == "nodata" {
|
|
continue
|
|
}
|
|
total++
|
|
if h.Status == "operational" {
|
|
ok++
|
|
}
|
|
}
|
|
if total == 0 {
|
|
return "—"
|
|
}
|
|
return fmt.Sprintf("%.2f%%", float64(ok)/float64(total)*100)
|
|
}
|
|
|
|
func (n *Node) SVGBars() template.HTML {
|
|
var sb strings.Builder
|
|
sb.WriteString(`<svg viewBox="0 0 448 34" width="100%" height="34" style="display:block">`)
|
|
for i, day := range n.History {
|
|
x := i * 5
|
|
color := colorGray
|
|
switch day.Status {
|
|
case "operational":
|
|
color = colorGreen
|
|
case "degraded":
|
|
color = colorOrange
|
|
case "outage":
|
|
color = colorRed
|
|
case "nodata":
|
|
color = "none"
|
|
}
|
|
title := fmt.Sprintf("%s: %s", day.Date, day.Status)
|
|
sb.WriteString(fmt.Sprintf(
|
|
`<rect x="%d" y="0" width="3" height="34" rx="1" fill="%s"><title>%s</title></rect>`,
|
|
x, color, title,
|
|
))
|
|
}
|
|
sb.WriteString(`</svg>`)
|
|
return template.HTML(sb.String())
|
|
}
|
|
|
|
func (n *Node) StatusLabel() string {
|
|
switch n.Status {
|
|
case "operational":
|
|
return "Operational"
|
|
case "degraded":
|
|
return "Degraded Performance"
|
|
case "outage":
|
|
return "Major Outage"
|
|
case "planned":
|
|
return "Planned"
|
|
}
|
|
return "Unknown"
|
|
}
|
|
|
|
func (n *Node) StatusColor() string {
|
|
switch n.Status {
|
|
case "operational":
|
|
return colorGreen
|
|
case "degraded":
|
|
return colorOrange
|
|
case "outage":
|
|
return colorRed
|
|
}
|
|
return colorGray
|
|
}
|
|
|
|
func nodeOrder(n *Node) int {
|
|
if n.ID == "hq-zurich" {
|
|
return 0
|
|
}
|
|
if n.Status == "operational" {
|
|
return 1
|
|
}
|
|
return 2
|
|
}
|
|
|
|
var db *sql.DB
|
|
|
|
// seedData populates the DB on first run
|
|
func seedData() {
|
|
// Insert nodes
|
|
nodeList := []struct{ id, name, region, ip, status string }{
|
|
{"virginia", "Virginia", "us-east-1", "18.209.55.127", "operational"},
|
|
{"singapore", "Singapore", "ap-southeast-1", "47.129.4.217", "operational"},
|
|
{"zurich", "Zürich", "eu-central-2", "16.18.20.81", "operational"},
|
|
{"ncalifornia", "N. California", "us-west-1", "", "planned"},
|
|
{"montreal", "Montreal", "ca-central-1", "", "planned"},
|
|
{"mexicocity", "Mexico City", "mx-central-1", "", "planned"},
|
|
{"saopaulo", "São Paulo", "sa-east-1", "", "planned"},
|
|
{"london", "London", "eu-west-2", "", "planned"},
|
|
{"paris", "Paris", "eu-west-3", "", "planned"},
|
|
{"spain", "Spain", "eu-south-2", "", "planned"},
|
|
{"stockholm", "Stockholm", "eu-north-1", "", "planned"},
|
|
{"uae", "UAE", "me-central-1", "", "planned"},
|
|
{"telaviv", "Tel Aviv", "il-central-1", "", "planned"},
|
|
{"capetown", "Cape Town", "af-south-1", "", "planned"},
|
|
{"mumbai", "Mumbai", "ap-south-1", "", "planned"},
|
|
{"jakarta", "Jakarta", "ap-southeast-3", "", "planned"},
|
|
{"malaysia", "Malaysia", "ap-southeast-5", "", "planned"},
|
|
{"sydney", "Sydney", "ap-southeast-2", "", "planned"},
|
|
{"seoul", "Seoul", "ap-northeast-2", "", "planned"},
|
|
{"hongkong", "Hong Kong", "ap-east-1", "", "planned"},
|
|
{"tokyo", "Tokyo", "ap-northeast-1", "", "planned"},
|
|
}
|
|
for _, n := range nodeList {
|
|
db.Exec(`INSERT OR IGNORE INTO nodes(id,name,region,ip,status) VALUES(?,?,?,?,?)`,
|
|
n.id, n.name, n.region, n.ip, n.status)
|
|
}
|
|
|
|
// Planned nodes: seed gray bars so the UI renders correctly
|
|
var plannedIDs []string
|
|
rows, _ := db.Query(`SELECT id FROM nodes WHERE status='planned'`)
|
|
for rows.Next() {
|
|
var id string
|
|
rows.Scan(&id)
|
|
plannedIDs = append(plannedIDs, id)
|
|
}
|
|
rows.Close()
|
|
now := time.Now()
|
|
for _, nodeID := range plannedIDs {
|
|
for i := 89; i >= 0; i-- {
|
|
date := now.AddDate(0, 0, -i).Format("2006-01-02")
|
|
db.Exec(`INSERT OR IGNORE INTO uptime(node_id,date,status) VALUES(?,?,?)`, nodeID, date, "planned")
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func loadNodes() []*Node {
|
|
rows, err := db.Query(`SELECT id,name,region,ip,status FROM nodes`)
|
|
if err != nil {
|
|
log.Printf("loadNodes error: %v", err)
|
|
return nil
|
|
}
|
|
defer rows.Close()
|
|
|
|
var nodes []*Node
|
|
for rows.Next() {
|
|
n := &Node{}
|
|
rows.Scan(&n.ID, &n.Name, &n.Region, &n.IP, &n.Status)
|
|
nodes = append(nodes, n)
|
|
}
|
|
|
|
// Load 90-day history for each node
|
|
now := time.Now()
|
|
for _, n := range nodes {
|
|
n.History = make([]DayStatus, 90)
|
|
// Build date map from DB
|
|
histMap := map[string]string{}
|
|
hrows, _ := db.Query(`SELECT date,status FROM uptime WHERE node_id=? ORDER BY date`, n.ID)
|
|
for hrows.Next() {
|
|
var date, status string
|
|
hrows.Scan(&date, &status)
|
|
histMap[date] = status
|
|
}
|
|
hrows.Close()
|
|
|
|
for i := 0; i < 90; i++ {
|
|
date := now.AddDate(0, 0, -(89 - i)).Format("2006-01-02")
|
|
status, ok := histMap[date]
|
|
if !ok {
|
|
if n.Status == "planned" {
|
|
status = "planned"
|
|
} else {
|
|
status = "nodata" // live node, no record yet — render transparent
|
|
}
|
|
}
|
|
n.History[i] = DayStatus{Date: date, Status: status}
|
|
}
|
|
}
|
|
|
|
sort.SliceStable(nodes, func(i, j int) bool {
|
|
oi, oj := nodeOrder(nodes[i]), nodeOrder(nodes[j])
|
|
if oi != oj {
|
|
return oi < oj
|
|
}
|
|
return nodes[i].Name < nodes[j].Name
|
|
})
|
|
return nodes
|
|
}
|
|
|
|
func loadIncidents() []Incident {
|
|
rows, err := db.Query(`SELECT id,title,status,date,node_ids FROM incidents ORDER BY created_at DESC`)
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
defer rows.Close()
|
|
var list []Incident
|
|
for rows.Next() {
|
|
var inc Incident
|
|
rows.Scan(&inc.ID, &inc.Title, &inc.Status, &inc.Date, &inc.NodeIDs)
|
|
list = append(list, inc)
|
|
}
|
|
rows.Close()
|
|
|
|
// Load updates for each incident
|
|
for i := range list {
|
|
urows, err := db.Query(
|
|
`SELECT ts,status,body FROM incident_updates WHERE incident_id=? ORDER BY id ASC`,
|
|
list[i].ID,
|
|
)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
for urows.Next() {
|
|
var u IncidentUpdate
|
|
urows.Scan(&u.TS, &u.Status, &u.Body)
|
|
list[i].Updates = append(list[i].Updates, u)
|
|
}
|
|
urows.Close()
|
|
}
|
|
return list
|
|
}
|
|
|
|
func buildPage() PageData {
|
|
nodes := loadNodes()
|
|
ok := true
|
|
for _, n := range nodes {
|
|
if n.Status == "outage" || n.Status == "degraded" {
|
|
ok = false
|
|
break
|
|
}
|
|
}
|
|
return PageData{
|
|
UpdatedAt: time.Now().UTC().Format("Jan 2, 2006 15:04 UTC"),
|
|
OverallOK: ok,
|
|
Nodes: nodes,
|
|
Incidents: loadIncidents(),
|
|
}
|
|
}
|
|
|
|
var tpl = template.Must(template.New("page").Parse(`<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Vault1984 Status</title>
|
|
<style>
|
|
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif;
|
|
background: #FAF9F5;
|
|
color: #141413;
|
|
line-height: 1.5;
|
|
}
|
|
a { color: #141413; }
|
|
.wrap { max-width: 860px; margin: 0 auto; padding: 32px 20px 64px; }
|
|
.site-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: 20px; }
|
|
.site-header .updated { font-size: 12px; color: #87867F; }
|
|
.wordmark { font-family: "SF Mono", "Fira Code", "JetBrains Mono", monospace; font-size: 22px; font-weight: 700; letter-spacing: -0.02em; color: #141413; text-decoration: none; }
|
|
.wordmark .n { color: #22C55E; }
|
|
.banner { border-radius: 6px; padding: 14px 24px; margin-bottom: 24px; display: flex; align-items: center; gap: 14px; }
|
|
.banner.ok { background: #76AD2A; }
|
|
.banner.warn { background: #E86235; }
|
|
.banner h2 { font-size: 20px; font-weight: 700; color: #fff; }
|
|
.uptime-header { font-size: 12px; color: #87867F; text-align: right; margin-bottom: 6px; }
|
|
.section-title { font-size: 12px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.08em; color: #87867F; margin-bottom: 6px; padding-bottom: 6px; border-bottom: 1px solid #DEDCD1; }
|
|
.component { border-bottom: 1px solid #DEDCD1; padding: 8px 0 6px; }
|
|
.component:last-child { border-bottom: none; }
|
|
.component-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: 6px; }
|
|
.component-name { font-size: 15px; font-weight: 500; }
|
|
.component-region { font-size: 12px; color: #87867F; margin-left: 8px; }
|
|
.component-status { font-size: 13px; font-weight: 500; }
|
|
.bar-row { margin-bottom: 2px; }
|
|
.bar-meta { display: flex; justify-content: space-between; font-size: 11px; color: #87867F; margin-top: 2px; }
|
|
.incidents { margin-top: 36px; }
|
|
.incident { border-top: 1px solid #DEDCD1; padding: 14px 0; }
|
|
.incident-header { display: flex; justify-content: space-between; align-items: baseline; margin-bottom: 4px; }
|
|
.incident-title { font-size: 15px; font-weight: 600; }
|
|
.incident-date { font-size: 12px; color: #87867F; }
|
|
.incident-status { display: inline-block; font-size: 11px; font-weight: 600; text-transform: uppercase; letter-spacing: 0.06em; padding: 2px 8px; border-radius: 10px; margin-bottom: 4px; }
|
|
.incident-status.resolved { background: rgba(118,173,42,.15); color: #76AD2A; }
|
|
.incident-status.monitoring { background: rgba(250,167,42,.15); color: #FAA72A; }
|
|
.incident-status.investigating{ background: rgba(224,67,67,.15); color: #E04343; }
|
|
.incident-body { font-size: 13px; color: #87867F; }
|
|
.incident-nodes { font-size: 11px; color: #87867F; margin-bottom: 10px; }
|
|
.timeline { margin-top: 12px; padding-left: 16px; border-left: 2px solid #DEDCD1; }
|
|
.timeline-entry { position: relative; padding: 0 0 14px 18px; }
|
|
.timeline-entry:last-child { padding-bottom: 0; }
|
|
.timeline-dot {
|
|
position: absolute; left: -7px; top: 4px;
|
|
width: 12px; height: 12px; border-radius: 50%;
|
|
border: 2px solid #FAF9F5;
|
|
}
|
|
.timeline-dot.resolved { background: #76AD2A; }
|
|
.timeline-dot.monitoring { background: #FAA72A; }
|
|
.timeline-dot.identified { background: #E86235; }
|
|
.timeline-dot.investigating{ background: #E04343; }
|
|
.timeline-ts { font-size: 11px; color: #87867F; margin-bottom: 2px; }
|
|
.timeline-status { font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: 0.05em; margin-bottom: 3px; }
|
|
.timeline-status.resolved { color: #76AD2A; }
|
|
.timeline-status.monitoring { color: #FAA72A; }
|
|
.timeline-status.identified { color: #E86235; }
|
|
.timeline-status.investigating{ color: #E04343; }
|
|
.timeline-body { font-size: 13px; color: #141413; }
|
|
footer { margin-top: 40px; text-align: center; font-size: 12px; color: #87867F; }
|
|
|
|
/* Chat widget */
|
|
#chat-btn {
|
|
position: fixed; bottom: 24px; right: 24px;
|
|
background: #141413; color: #FAF9F5;
|
|
border: none; border-radius: 50px;
|
|
padding: 12px 20px; font-size: 14px; font-weight: 600;
|
|
cursor: pointer; box-shadow: 0 2px 12px rgba(0,0,0,0.18);
|
|
display: flex; align-items: center; gap: 8px;
|
|
z-index: 100;
|
|
}
|
|
#chat-btn:hover { background: #2d2d2b; }
|
|
#chat-widget {
|
|
display: none; position: fixed; bottom: 76px; right: 24px;
|
|
width: 340px; background: #FAF9F5;
|
|
border: 1px solid #DEDCD1; border-radius: 12px;
|
|
box-shadow: 0 8px 32px rgba(0,0,0,0.14);
|
|
flex-direction: column; z-index: 100;
|
|
overflow: hidden;
|
|
}
|
|
#chat-widget.open { display: flex; }
|
|
#chat-header {
|
|
background: #141413; color: #FAF9F5;
|
|
padding: 14px 16px; font-size: 14px; font-weight: 600;
|
|
display: flex; justify-content: space-between; align-items: center;
|
|
}
|
|
#chat-close { background: none; border: none; color: #FAF9F5; cursor: pointer; font-size: 18px; line-height: 1; }
|
|
#chat-messages {
|
|
flex: 1; overflow-y: auto; padding: 16px; min-height: 220px; max-height: 320px;
|
|
display: flex; flex-direction: column; gap: 10px;
|
|
}
|
|
.msg { max-width: 85%; padding: 8px 12px; border-radius: 12px; font-size: 13px; line-height: 1.5; }
|
|
.msg.user { background: #141413; color: #FAF9F5; align-self: flex-end; border-radius: 12px 12px 3px 12px; }
|
|
.msg.bot { background: #EDECE8; color: #141413; align-self: flex-start; border-radius: 12px 12px 12px 3px; }
|
|
.msg.thinking { color: #87867F; font-style: italic; }
|
|
#chat-input-row {
|
|
display: flex; gap: 8px; padding: 12px;
|
|
border-top: 1px solid #DEDCD1; background: #FAF9F5;
|
|
}
|
|
#chat-input {
|
|
flex: 1; border: 1px solid #DEDCD1; border-radius: 8px;
|
|
padding: 8px 12px; font-size: 13px; background: #fff;
|
|
outline: none; font-family: inherit;
|
|
resize: none;
|
|
}
|
|
#chat-input:focus { border-color: #141413; }
|
|
#chat-send {
|
|
background: #141413; color: #FAF9F5; border: none;
|
|
border-radius: 8px; padding: 8px 14px; cursor: pointer;
|
|
font-size: 13px; font-weight: 600;
|
|
}
|
|
#chat-send:hover { background: #2d2d2b; }
|
|
#chat-send:disabled { opacity: 0.4; cursor: default; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="wrap">
|
|
<div class="site-header">
|
|
<a href="https://vault1984.com" class="wordmark">vault<span class="n">1984</span></a>
|
|
<span class="updated">Updated {{.UpdatedAt}}</span>
|
|
</div>
|
|
{{if .OverallOK}}
|
|
<div class="banner ok"><span>✓</span><h2>All Systems Operational</h2></div>
|
|
{{else}}
|
|
<div class="banner warn"><span>⚠</span><h2>Partial System Degradation</h2></div>
|
|
{{end}}
|
|
<div class="uptime-header">Uptime over the past 90 days.</div>
|
|
<div class="section-title">Regional Points of Presence</div>
|
|
{{range .Nodes}}
|
|
<div class="component">
|
|
<div class="component-header">
|
|
<span>
|
|
<span class="component-name">{{.Name}}</span>
|
|
<span class="component-region">{{.Region}}</span>
|
|
</span>
|
|
<span class="component-status" style="color:{{.StatusColor}}">{{.StatusLabel}}</span>
|
|
</div>
|
|
<div class="bar-row">{{.SVGBars}}</div>
|
|
<div class="bar-meta">
|
|
<span>90 days ago</span>
|
|
<span>{{.UptimePct}} uptime</span>
|
|
<span>Today</span>
|
|
</div>
|
|
</div>
|
|
{{end}}
|
|
{{if .Incidents}}
|
|
<div class="incidents">
|
|
<div class="section-title">Past Incidents</div>
|
|
{{range .Incidents}}
|
|
<div class="incident">
|
|
<div class="incident-header">
|
|
<span class="incident-title">{{.Title}}</span>
|
|
<span class="incident-date">{{.Date}}</span>
|
|
</div>
|
|
<span class="incident-status {{.Status}}">{{.Status}}</span>
|
|
{{if .NodeIDs}}<p class="incident-nodes">Affected: {{.NodeIDs}}</p>{{end}}
|
|
{{if .Updates}}
|
|
<div class="timeline">
|
|
{{range .Updates}}
|
|
<div class="timeline-entry">
|
|
<div class="timeline-dot {{.Status}}"></div>
|
|
<div class="timeline-ts">{{.TS}}</div>
|
|
<div class="timeline-status {{.Status}}">{{.Status}}</div>
|
|
<div class="timeline-body">{{.Body}}</div>
|
|
</div>
|
|
{{end}}
|
|
</div>
|
|
{{end}}
|
|
</div>
|
|
{{end}}
|
|
</div>
|
|
{{end}}
|
|
<footer>vault1984.com — Global Network Operations Center</footer>
|
|
</div>
|
|
|
|
<!-- Chat Widget -->
|
|
<button id="chat-btn" onclick="toggleChat()">💬 Report a Problem</button>
|
|
|
|
<div id="chat-widget">
|
|
<div id="chat-header">
|
|
<span>🛡 Vault1984 Support</span>
|
|
<button id="chat-close" onclick="toggleChat()">✕</button>
|
|
</div>
|
|
<div id="chat-messages">
|
|
<div class="msg bot">Hi! Report a network issue or ask about our infrastructure. Which region are you seeing problems with?</div>
|
|
</div>
|
|
<div id="chat-input-row">
|
|
<textarea id="chat-input" rows="1" placeholder="Describe the issue…"></textarea>
|
|
<button id="chat-send" onclick="sendMsg()">Send</button>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
const history = [];
|
|
function toggleChat() {
|
|
const w = document.getElementById('chat-widget');
|
|
w.classList.toggle('open');
|
|
if (w.classList.contains('open')) document.getElementById('chat-input').focus();
|
|
}
|
|
document.getElementById('chat-input').addEventListener('keydown', e => {
|
|
if (e.key === 'Enter' && !e.shiftKey) { e.preventDefault(); sendMsg(); }
|
|
});
|
|
async function sendMsg() {
|
|
const input = document.getElementById('chat-input');
|
|
const text = input.value.trim();
|
|
if (!text) return;
|
|
input.value = '';
|
|
const send = document.getElementById('chat-send');
|
|
send.disabled = true;
|
|
|
|
appendMsg('user', text);
|
|
history.push({role:'user', content:text});
|
|
|
|
const thinking = appendMsg('bot', 'Thinking…', 'thinking');
|
|
|
|
try {
|
|
const res = await fetch('/api/chat', {
|
|
method: 'POST',
|
|
headers: {'Content-Type':'application/json'},
|
|
body: JSON.stringify({messages: history})
|
|
});
|
|
const data = await res.json();
|
|
thinking.remove();
|
|
const reply = data.reply || data.error || 'No response';
|
|
appendMsg('bot', reply);
|
|
history.push({role:'assistant', content:reply});
|
|
} catch(e) {
|
|
thinking.remove();
|
|
appendMsg('bot', 'Connection error. Try again.');
|
|
}
|
|
send.disabled = false;
|
|
input.focus();
|
|
}
|
|
function appendMsg(role, text, extra='') {
|
|
const msgs = document.getElementById('chat-messages');
|
|
const div = document.createElement('div');
|
|
div.className = 'msg ' + role + (extra ? ' '+extra : '');
|
|
div.textContent = text;
|
|
msgs.appendChild(div);
|
|
msgs.scrollTop = msgs.scrollHeight;
|
|
return div;
|
|
}
|
|
</script>
|
|
</body>
|
|
</html>
|
|
`))
|
|
|
|
// tailscaleOnly allows only requests from Tailscale (100.64.0.0/10) or localhost
|
|
func tailscaleOnly(next http.HandlerFunc) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
ip := r.RemoteAddr
|
|
if i := strings.LastIndex(ip, ":"); i >= 0 {
|
|
ip = ip[:i]
|
|
}
|
|
ip = strings.Trim(ip, "[]")
|
|
allowed := ip == "127.0.0.1" || ip == "::1" || strings.HasPrefix(ip, "100.")
|
|
if !allowed {
|
|
http.Error(w, "Forbidden", 403)
|
|
return
|
|
}
|
|
next(w, r)
|
|
}
|
|
}
|
|
|
|
func handleOpsDashboard(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
fmt.Fprint(w, opsDashboardHTML)
|
|
}
|
|
|
|
const opsDashboardHTML = `<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>vault1984 NOC</title>
|
|
<style>
|
|
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
|
:root {
|
|
--bg: #0d0d0f;
|
|
--surface: #16161a;
|
|
--border: #2a2a30;
|
|
--text: #e2e2e8;
|
|
--muted: #6b6b78;
|
|
--green: #3ecf8e;
|
|
--yellow: #f5a623;
|
|
--red: #e04343;
|
|
--blue: #4c8ef7;
|
|
--purple: #a78bfa;
|
|
}
|
|
body { background: var(--bg); color: var(--text); font-family: 'SF Mono', 'Fira Code', monospace; font-size: 13px; }
|
|
header {
|
|
display: flex; align-items: center; justify-content: space-between;
|
|
padding: 16px 24px; border-bottom: 1px solid var(--border);
|
|
background: var(--surface);
|
|
}
|
|
header h1 { font-size: 15px; font-weight: 600; letter-spacing: 0.05em; color: var(--text); }
|
|
header h1 span { color: var(--green); }
|
|
#last-updated { color: var(--muted); font-size: 11px; }
|
|
#status-dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; background: var(--green); margin-right: 6px; animation: pulse 2s infinite; }
|
|
@keyframes pulse { 0%,100%{opacity:1} 50%{opacity:.4} }
|
|
main { padding: 24px; }
|
|
.grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(380px, 1fr)); gap: 16px; margin-bottom: 24px; }
|
|
.card {
|
|
background: var(--surface); border: 1px solid var(--border); border-radius: 8px;
|
|
padding: 16px; position: relative; overflow: hidden;
|
|
}
|
|
.card.stale { border-color: var(--yellow); }
|
|
.card.offline { border-color: var(--red); }
|
|
.card-header { display: flex; align-items: center; justify-content: space-between; margin-bottom: 14px; }
|
|
.node-name { font-size: 14px; font-weight: 600; }
|
|
.node-meta { color: var(--muted); font-size: 11px; margin-top: 2px; }
|
|
.badge {
|
|
font-size: 10px; font-weight: 700; padding: 2px 8px; border-radius: 99px;
|
|
text-transform: uppercase; letter-spacing: 0.08em;
|
|
}
|
|
.badge.ok { background: rgba(62,207,142,.15); color: var(--green); }
|
|
.badge.stale { background: rgba(245,166,35,.15); color: var(--yellow); }
|
|
.badge.offline { background: rgba(224,67,67,.15); color: var(--red); }
|
|
.metrics { display: grid; grid-template-columns: 1fr 1fr; gap: 10px; margin-bottom: 14px; }
|
|
.metric label { display: block; color: var(--muted); font-size: 10px; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 4px; }
|
|
.metric .val { font-size: 20px; font-weight: 700; line-height: 1; }
|
|
.metric .val.good { color: var(--green); }
|
|
.metric .val.warn { color: var(--yellow); }
|
|
.metric .val.crit { color: var(--red); }
|
|
.metric .sub { color: var(--muted); font-size: 11px; margin-top: 2px; }
|
|
.bar-wrap { margin-bottom: 10px; }
|
|
.bar-wrap label { display: flex; justify-content: space-between; color: var(--muted); font-size: 10px; text-transform: uppercase; letter-spacing: 0.08em; margin-bottom: 4px; }
|
|
.bar-track { background: var(--border); border-radius: 2px; height: 4px; }
|
|
.bar-fill { height: 4px; border-radius: 2px; transition: width .4s; }
|
|
.bar-fill.green { background: var(--green); }
|
|
.bar-fill.yellow { background: var(--yellow); }
|
|
.bar-fill.red { background: var(--red); }
|
|
.sparkline-wrap { margin-top: 12px; border-top: 1px solid var(--border); padding-top: 12px; }
|
|
.sparkline-wrap label { color: var(--muted); font-size: 10px; text-transform: uppercase; letter-spacing: 0.08em; display: block; margin-bottom: 6px; }
|
|
canvas.spark { width: 100%; height: 40px; display: block; }
|
|
.card-footer { display: flex; justify-content: space-between; align-items: center; margin-top: 12px; padding-top: 10px; border-top: 1px solid var(--border); color: var(--muted); font-size: 10px; }
|
|
.vault-stat { color: var(--purple); font-weight: 600; }
|
|
#summary { display: flex; gap: 24px; margin-bottom: 20px; padding: 12px 16px; background: var(--surface); border: 1px solid var(--border); border-radius: 8px; }
|
|
#summary .s-item label { color: var(--muted); font-size: 10px; text-transform: uppercase; letter-spacing: .08em; }
|
|
#summary .s-item .s-val { font-size: 16px; font-weight: 700; color: var(--text); }
|
|
#error-msg { color: var(--red); font-size: 12px; padding: 8px; display: none; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<header>
|
|
<h1>⚡ vault1984 <span>NOC</span></h1>
|
|
<div style="display:flex;align-items:center;gap:16px">
|
|
<span id="error-msg"></span>
|
|
<span><span id="status-dot"></span><span id="last-updated">loading...</span></span>
|
|
</div>
|
|
</header>
|
|
<main>
|
|
<div id="summary">
|
|
<div class="s-item"><label>Nodes reporting</label><div class="s-val" id="s-nodes">—</div></div>
|
|
<div class="s-item"><label>Avg CPU</label><div class="s-val" id="s-cpu">—</div></div>
|
|
<div class="s-item"><label>Avg Mem</label><div class="s-val" id="s-mem">—</div></div>
|
|
<div class="s-item"><label>Total Vaults</label><div class="s-val" id="s-vaults">—</div></div>
|
|
</div>
|
|
<div class="grid" id="cards"></div>
|
|
</main>
|
|
<script>
|
|
const REFRESH_MS = 30000;
|
|
const history = {}; // node_id → [{ts, cpu, mem_used_mb, mem_total_mb}]
|
|
|
|
function colorClass(pct, warnAt=60, critAt=85) {
|
|
if (pct >= critAt) return 'crit';
|
|
if (pct >= warnAt) return 'warn';
|
|
return 'good';
|
|
}
|
|
function barColor(pct, warnAt=60, critAt=85) {
|
|
if (pct >= critAt) return 'red';
|
|
if (pct >= warnAt) return 'yellow';
|
|
return 'green';
|
|
}
|
|
function fmtUptime(s) {
|
|
if (!s) return '—';
|
|
const d = Math.floor(s/86400), h = Math.floor((s%86400)/3600), m = Math.floor((s%3600)/60);
|
|
if (d > 0) return d+'d '+h+'h';
|
|
if (h > 0) return h+'h '+m+'m';
|
|
return m+'m';
|
|
}
|
|
function fmtAgo(ts) {
|
|
const s = Math.round(Date.now()/1000 - ts);
|
|
if (s < 5) return 'just now';
|
|
if (s < 60) return s+'s ago';
|
|
if (s < 3600) return Math.floor(s/60)+'m ago';
|
|
return Math.floor(s/3600)+'h ago';
|
|
}
|
|
function drawSpark(canvas, points, key, color) {
|
|
const W = canvas.clientWidth || 348, H = 40;
|
|
canvas.width = W * devicePixelRatio; canvas.height = H * devicePixelRatio;
|
|
const ctx = canvas.getContext('2d');
|
|
ctx.scale(devicePixelRatio, devicePixelRatio);
|
|
if (!points || points.length < 2) return;
|
|
const vals = points.map(p => p[key]);
|
|
const min = 0, max = Math.max(...vals, 5);
|
|
const xs = (i) => (i / (points.length-1)) * W;
|
|
const ys = (v) => H - 2 - ((v - min)/(max - min)) * (H-4);
|
|
ctx.beginPath();
|
|
ctx.strokeStyle = color; ctx.lineWidth = 1.5;
|
|
ctx.moveTo(xs(0), ys(vals[0]));
|
|
for (let i=1;i<vals.length;i++) ctx.lineTo(xs(i), ys(vals[i]));
|
|
ctx.stroke();
|
|
// fill under
|
|
ctx.lineTo(xs(vals.length-1), H); ctx.lineTo(xs(0), H); ctx.closePath();
|
|
ctx.fillStyle = color.replace(')', ',0.1)').replace('rgb','rgba');
|
|
ctx.fill();
|
|
}
|
|
|
|
function renderCard(t, hist) {
|
|
if (t._pending) return ` + "`" + `
|
|
<div class="card" style="opacity:.5">
|
|
<div class="card-header">
|
|
<div><div class="node-name">${t.node_id}</div><div class="node-meta">awaiting first beat…</div></div>
|
|
<span class="badge" style="background:rgba(107,107,120,.2);color:#6b6b78">PENDING</span>
|
|
</div>
|
|
<div style="color:var(--muted);font-size:12px;text-align:center;padding:20px 0">No telemetry yet</div>
|
|
</div>` + "`" + `;
|
|
const ageS = Math.round(Date.now()/1000 - t.received_at);
|
|
const stale = ageS > 150; // missed 2 beats
|
|
const offline = ageS > 300;
|
|
const memPct = t.memory_total_mb ? Math.round(t.memory_used_mb/t.memory_total_mb*100) : 0;
|
|
const diskPct = t.disk_total_mb ? Math.round(t.disk_used_mb/t.disk_total_mb*100) : 0;
|
|
const cardClass = offline ? 'card offline' : stale ? 'card stale' : 'card';
|
|
const badgeClass = offline ? 'badge offline' : stale ? 'badge stale' : 'badge ok';
|
|
const badgeText = offline ? 'OFFLINE' : stale ? 'STALE' : 'LIVE';
|
|
return ` + "`" + `
|
|
<div class="${cardClass}" id="card-${t.node_id}">
|
|
<div class="card-header">
|
|
<div>
|
|
<div class="node-name">${t.node_id}</div>
|
|
<div class="node-meta">${t.hostname} · v${t.version}</div>
|
|
</div>
|
|
<span class="${badgeClass}">${badgeText}</span>
|
|
</div>
|
|
<div class="metrics">
|
|
<div class="metric">
|
|
<label>CPU</label>
|
|
<div class="val ${colorClass(t.cpu_percent)}">${t.cpu_percent.toFixed(1)}<span style="font-size:12px;font-weight:400">%</span></div>
|
|
<div class="sub">load ${t.load_1m.toFixed(2)}</div>
|
|
</div>
|
|
<div class="metric">
|
|
<label>Memory</label>
|
|
<div class="val ${colorClass(memPct)}">${memPct}<span style="font-size:12px;font-weight:400">%</span></div>
|
|
<div class="sub">${t.memory_used_mb} / ${t.memory_total_mb} MB</div>
|
|
</div>
|
|
</div>
|
|
<div class="bar-wrap">
|
|
<label><span>Mem</span><span>${memPct}%</span></label>
|
|
<div class="bar-track"><div class="bar-fill ${barColor(memPct)}" style="width:${memPct}%"></div></div>
|
|
</div>
|
|
<div class="bar-wrap">
|
|
<label><span>Disk</span><span>${diskPct}% · ${t.disk_used_mb.toLocaleString()} / ${t.disk_total_mb.toLocaleString()} MB</span></label>
|
|
<div class="bar-track"><div class="bar-fill ${barColor(diskPct,70,90)}" style="width:${diskPct}%"></div></div>
|
|
</div>
|
|
<div class="sparkline-wrap">
|
|
<label>CPU % — last ${hist ? hist.length : 0} samples</label>
|
|
<canvas class="spark" id="spark-cpu-${t.node_id}"></canvas>
|
|
</div>
|
|
<div class="sparkline-wrap">
|
|
<label>Mem % — last ${hist ? hist.length : 0} samples</label>
|
|
<canvas class="spark" id="spark-mem-${t.node_id}"></canvas>
|
|
</div>
|
|
<div class="card-footer">
|
|
<span>⬆ ${fmtUptime(t.uptime_seconds)}</span>
|
|
<span class="vault-stat">◈ ${t.vault_count} vaults · ${t.vault_size_mb.toFixed(1)} MB</span>
|
|
<span title="${new Date(t.received_at*1000).toISOString()}">⏱ ${fmtAgo(t.received_at)}</span>
|
|
</div>
|
|
</div>` + "`" + `;
|
|
}
|
|
|
|
async function fetchHistory(nodeId) {
|
|
try {
|
|
const r = await fetch('/api/telemetry/history?node='+nodeId+'&limit=60', {cache: 'no-store'});
|
|
const d = await r.json();
|
|
if (d.history) {
|
|
history[nodeId] = d.history.map(h => ({
|
|
ts: h.ts, cpu: h.cpu,
|
|
mem_pct: h.mem_total_mb ? Math.round(h.mem_used_mb/h.mem_total_mb*100) : 0
|
|
}));
|
|
}
|
|
} catch(e) {}
|
|
}
|
|
|
|
async function refresh() {
|
|
try {
|
|
// Fetch known live nodes + telemetry in parallel
|
|
const [tRes, nRes] = await Promise.all([
|
|
fetch('/api/telemetry', {cache: 'no-store'}),
|
|
fetch('/api/nodes', {cache: 'no-store'}),
|
|
]);
|
|
const tData = await tRes.json();
|
|
const nData = await nRes.json();
|
|
const telemetryMap = {};
|
|
for (const t of (tData.telemetry || [])) telemetryMap[t.node_id] = t;
|
|
// Build node list: all operational nodes, filled with telemetry if available
|
|
const liveNodeIds = (nData.nodes || [])
|
|
.filter(n => n.Status === 'operational')
|
|
.map(n => n.ID);
|
|
const nodes = liveNodeIds.map(id => telemetryMap[id] || { node_id: id, _pending: true });
|
|
document.getElementById('error-msg').style.display = 'none';
|
|
|
|
// Summary
|
|
document.getElementById('s-nodes').textContent = nodes.length;
|
|
if (nodes.length) {
|
|
const avgCPU = nodes.reduce((a,n)=>a+n.cpu_percent,0)/nodes.length;
|
|
const avgMem = nodes.reduce((a,n)=>a+(n.memory_total_mb?n.memory_used_mb/n.memory_total_mb*100:0),0)/nodes.length;
|
|
const totalVaults = nodes.reduce((a,n)=>a+n.vault_count,0);
|
|
document.getElementById('s-cpu').textContent = avgCPU.toFixed(1)+'%';
|
|
document.getElementById('s-mem').textContent = avgMem.toFixed(1)+'%';
|
|
document.getElementById('s-vaults').textContent = totalVaults;
|
|
}
|
|
|
|
// Fetch history for all nodes (parallel)
|
|
await Promise.all(nodes.map(n => fetchHistory(n.node_id)));
|
|
|
|
// Render cards
|
|
const container = document.getElementById('cards');
|
|
container.innerHTML = nodes.map(t => renderCard(t, history[t.node_id])).join('');
|
|
|
|
// Draw sparklines
|
|
nodes.forEach(t => {
|
|
const hist = history[t.node_id] || [];
|
|
const cpuCanvas = document.getElementById('spark-cpu-'+t.node_id);
|
|
const memCanvas = document.getElementById('spark-mem-'+t.node_id);
|
|
if (cpuCanvas) drawSpark(cpuCanvas, hist, 'cpu', 'rgb(76,142,247)');
|
|
if (memCanvas) drawSpark(memCanvas, hist, 'mem_pct', 'rgb(167,139,250)');
|
|
});
|
|
|
|
document.getElementById('last-updated').textContent =
|
|
'updated ' + new Date().toLocaleTimeString();
|
|
} catch(e) {
|
|
document.getElementById('error-msg').textContent = 'fetch error: '+e.message;
|
|
document.getElementById('error-msg').style.display = 'inline';
|
|
}
|
|
}
|
|
|
|
refresh();
|
|
setInterval(refresh, REFRESH_MS);
|
|
</script>
|
|
</body>
|
|
</html>
|
|
`
|
|
|
|
func handleAdminGUI(w http.ResponseWriter, r *http.Request) {
|
|
nodes := loadNodes()
|
|
incidents := loadIncidents()
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
adminTpl.Execute(w, map[string]interface{}{
|
|
"Nodes": nodes,
|
|
"Incidents": incidents,
|
|
"Now": time.Now().UTC().Format("Jan 2, 2006 15:04 UTC"),
|
|
})
|
|
}
|
|
|
|
func handleAdminNewIncident(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != "POST" { w.WriteHeader(405); return }
|
|
var p struct {
|
|
Title string `json:"title"`
|
|
Status string `json:"status"`
|
|
Body string `json:"body"`
|
|
NodeIDs string `json:"node_ids"`
|
|
}
|
|
decode(r, &p)
|
|
date := time.Now().UTC().Format("Jan 2, 2006")
|
|
res, err := db.Exec(`INSERT INTO incidents(title,status,date,node_ids) VALUES(?,?,?,?)`,
|
|
p.Title, p.Status, date, p.NodeIDs)
|
|
if err != nil { jsonErr(w, err); return }
|
|
id, _ := res.LastInsertId()
|
|
ts := time.Now().UTC().Format("Jan 2, 2006 15:04 UTC")
|
|
db.Exec(`INSERT INTO incident_updates(incident_id,ts,status,body) VALUES(?,?,?,?)`,
|
|
id, ts, p.Status, p.Body)
|
|
// Update affected nodes
|
|
if p.NodeIDs != "" {
|
|
for _, nid := range strings.Split(p.NodeIDs, ",") {
|
|
nid = strings.TrimSpace(nid)
|
|
if nid != "" {
|
|
db.Exec(`UPDATE nodes SET status=? WHERE id=?`, p.Status, nid)
|
|
}
|
|
}
|
|
}
|
|
log.Printf("admin: new incident #%d: %s", id, p.Title)
|
|
jsonOK(w)
|
|
}
|
|
|
|
func handleAdminUpdateIncident(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != "POST" { w.WriteHeader(405); return }
|
|
var p struct {
|
|
IncidentID int64 `json:"incident_id"`
|
|
Status string `json:"status"`
|
|
Body string `json:"body"`
|
|
}
|
|
decode(r, &p)
|
|
ts := time.Now().UTC().Format("Jan 2, 2006 15:04 UTC")
|
|
db.Exec(`INSERT INTO incident_updates(incident_id,ts,status,body) VALUES(?,?,?,?)`,
|
|
p.IncidentID, ts, p.Status, p.Body)
|
|
db.Exec(`UPDATE incidents SET status=? WHERE id=?`, p.Status, p.IncidentID)
|
|
log.Printf("admin: updated incident #%d → %s", p.IncidentID, p.Status)
|
|
jsonOK(w)
|
|
}
|
|
|
|
func handleAdminNodeStatus(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != "POST" { w.WriteHeader(405); return }
|
|
var p struct {
|
|
ID string `json:"id"`
|
|
Status string `json:"status"`
|
|
}
|
|
decode(r, &p)
|
|
today := time.Now().Format("2006-01-02")
|
|
db.Exec(`UPDATE nodes SET status=? WHERE id=?`, p.Status, p.ID)
|
|
db.Exec(`INSERT OR REPLACE INTO uptime(node_id,date,status) VALUES(?,?,?)`, p.ID, today, p.Status)
|
|
log.Printf("admin: node %s → %s", p.ID, p.Status)
|
|
jsonOK(w)
|
|
}
|
|
|
|
func decode(r *http.Request, v interface{}) {
|
|
ct := r.Header.Get("Content-Type")
|
|
if strings.Contains(ct, "application/json") {
|
|
json.NewDecoder(r.Body).Decode(v)
|
|
} else {
|
|
r.ParseForm()
|
|
// map form values into struct via JSON round-trip
|
|
m := map[string]string{}
|
|
for k, vs := range r.Form { if len(vs) > 0 { m[k] = vs[0] } }
|
|
b, _ := json.Marshal(m)
|
|
json.Unmarshal(b, v)
|
|
}
|
|
}
|
|
func jsonOK(w http.ResponseWriter) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
fmt.Fprintln(w, `{"ok":true}`)
|
|
}
|
|
func jsonErr(w http.ResponseWriter, err error) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(500)
|
|
json.NewEncoder(w).Encode(map[string]string{"error": err.Error()})
|
|
}
|
|
|
|
var adminTpl = template.Must(template.New("admin").Parse(`<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>Vault1984 Admin</title>
|
|
<style>
|
|
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
|
body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif; background: #0d1117; color: #c9d1d9; line-height: 1.5; }
|
|
.wrap { max-width: 860px; margin: 0 auto; padding: 32px 20px 64px; }
|
|
h1 { font-size: 22px; font-weight: 700; color: #58a6ff; margin-bottom: 8px; }
|
|
.subtitle { font-size: 13px; color: #6e7681; margin-bottom: 32px; }
|
|
h2 { font-size: 16px; font-weight: 600; color: #f0f6fc; margin-bottom: 14px; margin-top: 32px; padding-bottom: 8px; border-bottom: 1px solid #30363d; }
|
|
.card { background: #161b22; border: 1px solid #30363d; border-radius: 8px; padding: 20px; margin-bottom: 20px; }
|
|
label { display: block; font-size: 12px; color: #8b949e; margin-bottom: 4px; margin-top: 12px; }
|
|
label:first-child { margin-top: 0; }
|
|
input, select, textarea {
|
|
width: 100%; padding: 8px 10px; background: #0d1117; color: #c9d1d9;
|
|
border: 1px solid #30363d; border-radius: 6px; font-size: 13px;
|
|
font-family: inherit;
|
|
}
|
|
input:focus, select:focus, textarea:focus { outline: none; border-color: #58a6ff; }
|
|
textarea { resize: vertical; min-height: 70px; }
|
|
button {
|
|
margin-top: 16px; padding: 8px 18px;
|
|
background: #238636; color: #fff; border: none;
|
|
border-radius: 6px; font-size: 13px; font-weight: 600;
|
|
cursor: pointer;
|
|
}
|
|
button:hover { background: #2ea043; }
|
|
.toast { display: none; background: #238636; color: #fff; padding: 8px 14px; border-radius: 6px; font-size: 13px; margin-top: 10px; }
|
|
.node-grid { display: grid; grid-template-columns: repeat(auto-fill, minmax(180px, 1fr)); gap: 10px; }
|
|
.node-item { background: #0d1117; border: 1px solid #30363d; border-radius: 6px; padding: 10px 12px; }
|
|
.node-item strong { font-size: 13px; color: #f0f6fc; }
|
|
.node-item small { font-size: 11px; color: #6e7681; }
|
|
.node-item select { margin-top: 6px; }
|
|
.node-item button { margin-top: 8px; padding: 5px 12px; font-size: 12px; }
|
|
a { color: #58a6ff; text-decoration: none; font-size: 13px; }
|
|
a:hover { text-decoration: underline; }
|
|
.incident-item { padding: 8px 0; border-bottom: 1px solid #30363d; font-size: 13px; display: flex; justify-content: space-between; align-items: center; }
|
|
.incident-item:last-child { border: none; }
|
|
.badge { padding: 2px 8px; border-radius: 10px; font-size: 11px; font-weight: 600; }
|
|
.badge.resolved { background: rgba(63,185,80,.15); color: #3fb950; }
|
|
.badge.monitoring { background: rgba(210,153,34,.15); color: #d29922; }
|
|
.badge.investigating { background: rgba(248,81,73,.15); color: #f85149; }
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="wrap">
|
|
<h1>🛡 Vault1984 Admin</h1>
|
|
<div class="subtitle">{{.Now}} • <a href="/">← Public status page</a></div>
|
|
|
|
<!-- Node Status -->
|
|
<h2>Node Status</h2>
|
|
<div class="node-grid">
|
|
{{range .Nodes}}
|
|
<div class="node-item">
|
|
<strong>{{.Name}}</strong><br>
|
|
<small>{{.Region}}</small>
|
|
<select id="ns-{{.ID}}">
|
|
<option value="operational" {{if eq .Status "operational"}}selected{{end}}>Operational</option>
|
|
<option value="degraded" {{if eq .Status "degraded"}}selected{{end}}>Degraded</option>
|
|
<option value="outage" {{if eq .Status "outage"}}selected{{end}}>Outage</option>
|
|
<option value="planned" {{if eq .Status "planned"}}selected{{end}}>Planned</option>
|
|
</select>
|
|
<button onclick="setNode('{{.ID}}')">Update</button>
|
|
</div>
|
|
{{end}}
|
|
</div>
|
|
|
|
<!-- New Incident -->
|
|
<h2>Create Incident</h2>
|
|
<div class="card">
|
|
<label>Title</label>
|
|
<input id="inc-title" placeholder="e.g. Elevated latency in Singapore">
|
|
<label>Status</label>
|
|
<select id="inc-status">
|
|
<option value="investigating">Investigating</option>
|
|
<option value="identified">Identified</option>
|
|
<option value="monitoring">Monitoring</option>
|
|
<option value="resolved">Resolved</option>
|
|
</select>
|
|
<label>Affected nodes (comma-separated IDs, e.g. singapore,virginia)</label>
|
|
<input id="inc-nodes" placeholder="singapore,virginia">
|
|
<label>Initial update message</label>
|
|
<textarea id="inc-body" placeholder="We are investigating…"></textarea>
|
|
<button onclick="createIncident()">Create Incident</button>
|
|
<div class="toast" id="inc-toast">Incident created ✓</div>
|
|
</div>
|
|
|
|
<!-- Update Existing Incident -->
|
|
<h2>Post Update to Incident</h2>
|
|
<div class="card">
|
|
{{if .Incidents}}
|
|
<label>Incident</label>
|
|
<select id="upd-id">
|
|
{{range .Incidents}}
|
|
<option value="{{.ID}}">{{.Date}} — {{.Title}}</option>
|
|
{{end}}
|
|
</select>
|
|
<label>New Status</label>
|
|
<select id="upd-status">
|
|
<option value="investigating">Investigating</option>
|
|
<option value="identified">Identified</option>
|
|
<option value="monitoring">Monitoring</option>
|
|
<option value="resolved">Resolved</option>
|
|
</select>
|
|
<label>Update message</label>
|
|
<textarea id="upd-body" placeholder="A fix has been deployed…"></textarea>
|
|
<button onclick="postUpdate()">Post Update</button>
|
|
<div class="toast" id="upd-toast">Update posted ✓</div>
|
|
{{else}}
|
|
<p style="color:#6e7681;font-size:13px">No incidents yet.</p>
|
|
{{end}}
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
async function post(url, data) {
|
|
const r = await fetch(url, {method:'POST', headers:{'Content-Type':'application/json'}, body: JSON.stringify(data)});
|
|
return r.json();
|
|
}
|
|
function toast(id) {
|
|
const el = document.getElementById(id);
|
|
el.style.display = 'block';
|
|
setTimeout(() => el.style.display='none', 3000);
|
|
}
|
|
async function setNode(id) {
|
|
const status = document.getElementById('ns-'+id).value;
|
|
await post('/admin/node/status', {id, status});
|
|
toast('upd-toast');
|
|
}
|
|
async function createIncident() {
|
|
const title = document.getElementById('inc-title').value.trim();
|
|
const status = document.getElementById('inc-status').value;
|
|
const body = document.getElementById('inc-body').value.trim();
|
|
const node_ids = document.getElementById('inc-nodes').value.trim();
|
|
if (!title || !body) { alert('Title and message required'); return; }
|
|
await post('/admin/incident/new', {title, status, body, node_ids});
|
|
document.getElementById('inc-title').value = '';
|
|
document.getElementById('inc-body').value = '';
|
|
toast('inc-toast');
|
|
}
|
|
async function postUpdate() {
|
|
const incident_id = parseInt(document.getElementById('upd-id').value);
|
|
const status = document.getElementById('upd-status').value;
|
|
const body = document.getElementById('upd-body').value.trim();
|
|
if (!body) { alert('Message required'); return; }
|
|
await post('/admin/incident/update', {incident_id, status, body});
|
|
document.getElementById('upd-body').value = '';
|
|
toast('upd-toast');
|
|
}
|
|
</script>
|
|
</body>
|
|
</html>
|
|
`))
|
|
|
|
func handleChat(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != "POST" {
|
|
w.WriteHeader(405)
|
|
return
|
|
}
|
|
var req struct {
|
|
Messages []map[string]string `json:"messages"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
w.WriteHeader(400)
|
|
return
|
|
}
|
|
|
|
// Prepend system message
|
|
msgs := append([]map[string]string{{"role": "system", "content": systemPrompt}}, req.Messages...)
|
|
|
|
payload, _ := json.Marshal(map[string]interface{}{
|
|
"model": fireworksModel,
|
|
"messages": msgs,
|
|
"max_tokens": 300,
|
|
"temperature": 0.7,
|
|
})
|
|
|
|
fwReq, _ := http.NewRequest("POST", fireworksURL, bytes.NewReader(payload))
|
|
fwReq.Header.Set("Authorization", "Bearer "+fireworksKey)
|
|
fwReq.Header.Set("Content-Type", "application/json")
|
|
|
|
resp, err := http.DefaultClient.Do(fwReq)
|
|
if err != nil {
|
|
log.Printf("fireworks error: %v", err)
|
|
http.Error(w, `{"error":"upstream error"}`, 502)
|
|
return
|
|
}
|
|
defer resp.Body.Close()
|
|
body, _ := io.ReadAll(resp.Body)
|
|
|
|
var fw struct {
|
|
Choices []struct {
|
|
Message struct {
|
|
Content string `json:"content"`
|
|
} `json:"message"`
|
|
} `json:"choices"`
|
|
}
|
|
if err := json.Unmarshal(body, &fw); err != nil || len(fw.Choices) == 0 {
|
|
log.Printf("fireworks parse error: %s", body)
|
|
http.Error(w, `{"error":"parse error"}`, 500)
|
|
return
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(map[string]string{
|
|
"reply": strings.TrimSpace(fw.Choices[0].Message.Content),
|
|
})
|
|
}
|
|
|
|
// OTLP JSON metric receiver
|
|
func handleOTLP(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != "POST" {
|
|
w.WriteHeader(405)
|
|
return
|
|
}
|
|
|
|
var payload struct {
|
|
ResourceMetrics []struct {
|
|
Resource struct {
|
|
Attributes []struct {
|
|
Key string `json:"key"`
|
|
Value struct {
|
|
StringValue string `json:"stringValue"`
|
|
} `json:"value"`
|
|
} `json:"attributes"`
|
|
} `json:"resource"`
|
|
ScopeMetrics []struct {
|
|
Metrics []struct {
|
|
Name string `json:"name"`
|
|
Gauge struct {
|
|
DataPoints []struct {
|
|
AsDouble float64 `json:"asDouble"`
|
|
} `json:"dataPoints"`
|
|
} `json:"gauge"`
|
|
} `json:"metrics"`
|
|
} `json:"scopeMetrics"`
|
|
} `json:"resourceMetrics"`
|
|
}
|
|
|
|
if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
|
|
w.WriteHeader(400)
|
|
log.Printf("OTLP decode error: %v", err)
|
|
return
|
|
}
|
|
|
|
for _, rm := range payload.ResourceMetrics {
|
|
// Extract node.id from resource attributes
|
|
nodeID := ""
|
|
for _, a := range rm.Resource.Attributes {
|
|
if a.Key == "node.id" {
|
|
nodeID = a.Value.StringValue
|
|
}
|
|
}
|
|
if nodeID == "" {
|
|
continue
|
|
}
|
|
|
|
// Extract metrics
|
|
metrics := map[string]float64{}
|
|
for _, sm := range rm.ScopeMetrics {
|
|
for _, m := range sm.Metrics {
|
|
if len(m.Gauge.DataPoints) > 0 {
|
|
metrics[m.Name] = m.Gauge.DataPoints[0].AsDouble
|
|
}
|
|
}
|
|
}
|
|
|
|
cpu := metrics["system.cpu.utilization"]
|
|
mem := metrics["system.memory.utilization"]
|
|
disk := metrics["system.disk.utilization"]
|
|
|
|
// Determine status from thresholds
|
|
status := "operational"
|
|
if cpu > 0.9 || mem > 0.9 || disk > 0.9 {
|
|
status = "degraded"
|
|
}
|
|
|
|
// Update node status + today's uptime
|
|
today := time.Now().Format("2006-01-02")
|
|
db.Exec(`UPDATE nodes SET status=? WHERE id=?`, status, nodeID)
|
|
db.Exec(`INSERT OR REPLACE INTO uptime(node_id,date,status) VALUES(?,?,?)`, nodeID, today, status)
|
|
|
|
// Store latest metrics as JSON in a simple kv table (lazy: reuse uptime body col if available, or just log)
|
|
log.Printf("OTLP [%s] cpu=%.1f%% mem=%.1f%% disk=%.1f%% → %s",
|
|
nodeID, cpu*100, mem*100, disk*100, status)
|
|
}
|
|
|
|
w.Header().Set("Content-Type", "application/json")
|
|
fmt.Fprintln(w, `{"partialSuccess":{}}`)
|
|
}
|
|
|
|
func main() {
|
|
fmt.Printf("=== Vault1984 Dashboard %s ===\n", version)
|
|
|
|
|
|
var err error
|
|
db, err = sql.Open("sqlite", "./status.db")
|
|
if err != nil {
|
|
log.Fatalf("open db: %v", err)
|
|
}
|
|
db.SetMaxOpenConns(1) // SQLite: single writer
|
|
|
|
if _, err := db.Exec(schema); err != nil {
|
|
log.Fatalf("schema: %v", err)
|
|
}
|
|
seedData()
|
|
log.Println("DB ready: status.db")
|
|
|
|
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
|
tpl.Execute(w, buildPage())
|
|
})
|
|
|
|
http.HandleFunc("/api/status", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
nodes := loadNodes()
|
|
ok := true
|
|
for _, n := range nodes {
|
|
if n.Status == "outage" || n.Status == "degraded" {
|
|
ok = false
|
|
break
|
|
}
|
|
}
|
|
status := "operational"
|
|
if !ok {
|
|
status = "degraded"
|
|
}
|
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"status": status,
|
|
"updated": time.Now().UTC().Format(time.RFC3339),
|
|
"nodes": len(nodes),
|
|
})
|
|
})
|
|
|
|
http.HandleFunc("/api/chat", handleChat)
|
|
http.HandleFunc("/v1/metrics", handleOTLP)
|
|
http.HandleFunc("/download/agent", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Disposition", "attachment; filename=v1984-agent")
|
|
w.Header().Set("Content-Type", "application/octet-stream")
|
|
http.ServeFile(w, r, "./v1984-agent")
|
|
})
|
|
http.HandleFunc("/download/agent-arm64", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Disposition", "attachment; filename=v1984-agent")
|
|
w.Header().Set("Content-Type", "application/octet-stream")
|
|
http.ServeFile(w, r, "./v1984-agent-arm64")
|
|
})
|
|
http.HandleFunc("/download/vault1984", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Disposition", "attachment; filename=vault1984")
|
|
w.Header().Set("Content-Type", "application/octet-stream")
|
|
http.ServeFile(w, r, "./vault1984")
|
|
})
|
|
http.HandleFunc("/download/vault1984-arm64", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Disposition", "attachment; filename=vault1984")
|
|
w.Header().Set("Content-Type", "application/octet-stream")
|
|
http.ServeFile(w, r, "./vault1984-arm64")
|
|
})
|
|
|
|
http.HandleFunc("/api/nodes", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
json.NewEncoder(w).Encode(map[string]interface{}{
|
|
"nodes": loadNodes(),
|
|
"incidents": loadIncidents(),
|
|
})
|
|
})
|
|
|
|
// Admin routes — Tailscale only
|
|
http.HandleFunc("/admin", tailscaleOnly(handleAdminGUI))
|
|
http.HandleFunc("/admin/incident/new", tailscaleOnly(handleAdminNewIncident))
|
|
http.HandleFunc("/admin/incident/update", tailscaleOnly(handleAdminUpdateIncident))
|
|
http.HandleFunc("/admin/node/status", tailscaleOnly(handleAdminNodeStatus))
|
|
|
|
// vault1984 binary telemetry ingestion
|
|
http.HandleFunc("/telemetry", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != "POST" {
|
|
w.WriteHeader(405)
|
|
return
|
|
}
|
|
// Optional token auth: if TELEMETRY_TOKEN env is set, enforce it
|
|
if tok := os.Getenv("TELEMETRY_TOKEN"); tok != "" {
|
|
if r.Header.Get("X-Telemetry-Token") != tok && r.URL.Query().Get("token") != tok {
|
|
w.WriteHeader(401)
|
|
fmt.Fprintln(w, `{"error":"unauthorized"}`)
|
|
return
|
|
}
|
|
}
|
|
var p struct {
|
|
Version string `json:"version"`
|
|
Hostname string `json:"hostname"`
|
|
Uptime int64 `json:"uptime_seconds"`
|
|
System struct {
|
|
CPUPercent float64 `json:"cpu_percent"`
|
|
MemTotalMB int64 `json:"memory_total_mb"`
|
|
MemUsedMB int64 `json:"memory_used_mb"`
|
|
DiskTotalMB int64 `json:"disk_total_mb"`
|
|
DiskUsedMB int64 `json:"disk_used_mb"`
|
|
Load1m float64 `json:"load_1m"`
|
|
} `json:"system"`
|
|
Vaults struct {
|
|
Count int `json:"count"`
|
|
TotalSizeMB float64 `json:"total_size_mb"`
|
|
TotalEntries int `json:"total_entries"`
|
|
} `json:"vaults"`
|
|
Mode string `json:"mode"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&p); err != nil {
|
|
w.WriteHeader(400)
|
|
fmt.Fprintln(w, `{"error":"bad request"}`)
|
|
return
|
|
}
|
|
// Map hostname → node_id
|
|
// Priority: hostname matches node id → hostname matches node name → source IP → hostname as-is
|
|
nodeID := p.Hostname
|
|
var knownID string
|
|
// Try direct id match
|
|
db.QueryRow(`SELECT id FROM nodes WHERE id=?`, nodeID).Scan(&knownID)
|
|
if knownID == "" {
|
|
// Try matching by hostname against node name (case-insensitive)
|
|
db.QueryRow(`SELECT id FROM nodes WHERE lower(name)=lower(?) OR lower(id)=lower(?)`, nodeID, nodeID).Scan(&knownID)
|
|
}
|
|
if knownID == "" {
|
|
// Try matching by source IP (strips port)
|
|
remoteIP := r.RemoteAddr
|
|
if host, _, ok := strings.Cut(remoteIP, ":"); ok {
|
|
remoteIP = host
|
|
}
|
|
db.QueryRow(`SELECT id FROM nodes WHERE ip=?`, remoteIP).Scan(&knownID)
|
|
}
|
|
if knownID != "" {
|
|
nodeID = knownID
|
|
}
|
|
db.Exec(`INSERT INTO telemetry(node_id,version,hostname,uptime_seconds,cpu_percent,memory_total_mb,memory_used_mb,disk_total_mb,disk_used_mb,load_1m,vault_count,vault_size_mb,vault_entries,mode)
|
|
VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
|
|
nodeID, p.Version, p.Hostname, p.Uptime,
|
|
p.System.CPUPercent, p.System.MemTotalMB, p.System.MemUsedMB,
|
|
p.System.DiskTotalMB, p.System.DiskUsedMB, p.System.Load1m,
|
|
p.Vaults.Count, p.Vaults.TotalSizeMB, p.Vaults.TotalEntries,
|
|
p.Mode,
|
|
)
|
|
// Keep last 1000 rows per node (scoped to node_id)
|
|
db.Exec(`DELETE FROM telemetry WHERE node_id=? AND id NOT IN (SELECT id FROM telemetry WHERE node_id=? ORDER BY id DESC LIMIT 1000)`, nodeID, nodeID)
|
|
// Drive public status page: update node status + today's uptime record
|
|
today := time.Now().Format("2006-01-02")
|
|
db.Exec(`UPDATE nodes SET status='operational' WHERE id=?`, nodeID)
|
|
db.Exec(`INSERT OR REPLACE INTO uptime(node_id,date,status) VALUES(?,?,'operational')`, nodeID, today)
|
|
log.Printf("telemetry: %s (v%s) uptime=%ds cpu=%.1f%%", nodeID, p.Version, p.Uptime, p.System.CPUPercent)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
fmt.Fprintln(w, `{"ok":true}`)
|
|
})
|
|
|
|
// Latest telemetry snapshot per node
|
|
http.HandleFunc("/api/telemetry", func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.Header().Set("Cache-Control", "no-store, no-cache, must-revalidate")
|
|
rows, err := db.Query(`
|
|
SELECT t.node_id, t.received_at, t.version, t.hostname, t.uptime_seconds,
|
|
t.cpu_percent, t.memory_total_mb, t.memory_used_mb,
|
|
t.disk_total_mb, t.disk_used_mb, t.load_1m,
|
|
t.vault_count, t.vault_size_mb, t.vault_entries, t.mode
|
|
FROM telemetry t
|
|
INNER JOIN (
|
|
SELECT node_id, MAX(id) AS max_id FROM telemetry GROUP BY node_id
|
|
) latest ON t.id = latest.max_id
|
|
ORDER BY t.node_id
|
|
`)
|
|
if err != nil {
|
|
w.WriteHeader(500)
|
|
return
|
|
}
|
|
defer rows.Close()
|
|
type TRow struct {
|
|
NodeID string `json:"node_id"`
|
|
ReceivedAt int64 `json:"received_at"`
|
|
Version string `json:"version"`
|
|
Hostname string `json:"hostname"`
|
|
Uptime int64 `json:"uptime_seconds"`
|
|
CPUPct float64 `json:"cpu_percent"`
|
|
MemTotal int64 `json:"memory_total_mb"`
|
|
MemUsed int64 `json:"memory_used_mb"`
|
|
DiskTotal int64 `json:"disk_total_mb"`
|
|
DiskUsed int64 `json:"disk_used_mb"`
|
|
Load1m float64 `json:"load_1m"`
|
|
VaultCount int `json:"vault_count"`
|
|
VaultSize float64 `json:"vault_size_mb"`
|
|
VaultEntries int `json:"vault_entries"`
|
|
Mode string `json:"mode"`
|
|
}
|
|
var result []TRow
|
|
for rows.Next() {
|
|
var t TRow
|
|
rows.Scan(&t.NodeID, &t.ReceivedAt, &t.Version, &t.Hostname, &t.Uptime,
|
|
&t.CPUPct, &t.MemTotal, &t.MemUsed,
|
|
&t.DiskTotal, &t.DiskUsed, &t.Load1m,
|
|
&t.VaultCount, &t.VaultSize, &t.VaultEntries, &t.Mode)
|
|
result = append(result, t)
|
|
}
|
|
json.NewEncoder(w).Encode(map[string]interface{}{"telemetry": result})
|
|
})
|
|
|
|
// Telemetry history for a node: /api/telemetry/history?node=virginia&limit=60
|
|
http.HandleFunc("/api/telemetry/history", tailscaleOnly(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
nodeID := r.URL.Query().Get("node")
|
|
limit := 60
|
|
if l := r.URL.Query().Get("limit"); l != "" {
|
|
fmt.Sscanf(l, "%d", &limit)
|
|
}
|
|
if limit > 1000 { limit = 1000 }
|
|
type TRow struct {
|
|
ReceivedAt int64 `json:"ts"`
|
|
CPUPct float64 `json:"cpu"`
|
|
MemUsedMB int64 `json:"mem_used_mb"`
|
|
MemTotalMB int64 `json:"mem_total_mb"`
|
|
DiskUsedMB int64 `json:"disk_used_mb"`
|
|
DiskTotalMB int64 `json:"disk_total_mb"`
|
|
Load1m float64 `json:"load_1m"`
|
|
UptimeSec int64 `json:"uptime_seconds"`
|
|
}
|
|
rows, err := db.Query(`
|
|
SELECT received_at, cpu_percent, memory_used_mb, memory_total_mb,
|
|
disk_used_mb, disk_total_mb, load_1m, uptime_seconds
|
|
FROM telemetry WHERE node_id=?
|
|
ORDER BY id DESC LIMIT ?`, nodeID, limit)
|
|
if err != nil { w.WriteHeader(500); return }
|
|
defer rows.Close()
|
|
var result []TRow
|
|
for rows.Next() {
|
|
var t TRow
|
|
rows.Scan(&t.ReceivedAt, &t.CPUPct, &t.MemUsedMB, &t.MemTotalMB,
|
|
&t.DiskUsedMB, &t.DiskTotalMB, &t.Load1m, &t.UptimeSec)
|
|
result = append(result, t)
|
|
}
|
|
// Reverse to chronological order
|
|
for i, j := 0, len(result)-1; i < j; i, j = i+1, j-1 {
|
|
result[i], result[j] = result[j], result[i]
|
|
}
|
|
json.NewEncoder(w).Encode(map[string]interface{}{"node": nodeID, "history": result})
|
|
}))
|
|
|
|
// Internal NOC ops dashboard — Tailscale only
|
|
http.HandleFunc("/ops", tailscaleOnly(handleOpsDashboard))
|
|
|
|
// POPs push metrics here
|
|
http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method != "POST" {
|
|
w.WriteHeader(405)
|
|
return
|
|
}
|
|
var p struct {
|
|
ID string `json:"id"`
|
|
Status string `json:"status"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&p); err != nil {
|
|
w.WriteHeader(400)
|
|
return
|
|
}
|
|
today := time.Now().Format("2006-01-02")
|
|
db.Exec(`INSERT OR REPLACE INTO uptime(node_id,date,status) VALUES(?,?,?)`, p.ID, today, p.Status)
|
|
db.Exec(`UPDATE nodes SET status=? WHERE id=?`, p.Status, p.ID)
|
|
log.Printf("metrics: %s → %s", p.ID, p.Status)
|
|
w.Header().Set("Content-Type", "application/json")
|
|
fmt.Fprintln(w, `{"ok":true}`)
|
|
})
|
|
|
|
// Start POP health monitor goroutine
|
|
go runHealthMonitor()
|
|
|
|
log.Println("Listening on :8080")
|
|
log.Fatal(http.ListenAndServe(":8080", nil))
|
|
}
|
|
|
|
// runHealthMonitor posts to Kuma push URL every 60s and webhooks on problems.
|
|
// Configure via env: KUMA_PUSH_URL, ALERT_WEBHOOK_URL
|
|
func runHealthMonitor() {
|
|
kumaURL := os.Getenv("KUMA_PUSH_URL")
|
|
webhookURL := os.Getenv("ALERT_WEBHOOK_URL")
|
|
expectedNodes := []string{"virginia", "singapore", "zurich"}
|
|
staleThreshold := int64(300) // 5 min — Kuma fires after 2 missed 60s beats
|
|
alerting := false // track state to avoid webhook spam
|
|
|
|
if kumaURL == "" {
|
|
log.Println("health monitor: KUMA_PUSH_URL not set — monitoring disabled")
|
|
return
|
|
}
|
|
log.Printf("health monitor: posting to Kuma every 60s, alert webhook=%v", webhookURL != "")
|
|
|
|
ticker := time.NewTicker(60 * time.Second)
|
|
defer ticker.Stop()
|
|
|
|
doPost := func(url, body string) {
|
|
req, _ := http.NewRequest("GET", url, nil)
|
|
if body != "" {
|
|
req, _ = http.NewRequest("POST", url,
|
|
strings.NewReader(body))
|
|
req.Header.Set("Content-Type", "application/json")
|
|
}
|
|
http.DefaultClient.Timeout = 10 * time.Second
|
|
resp, err := http.DefaultClient.Do(req)
|
|
if err != nil {
|
|
log.Printf("health monitor: POST error: %v", err)
|
|
return
|
|
}
|
|
resp.Body.Close()
|
|
}
|
|
|
|
for range ticker.C {
|
|
now := time.Now().Unix()
|
|
type row struct {
|
|
nodeID string
|
|
receivedAt int64
|
|
}
|
|
rows, err := db.Query(`
|
|
SELECT t.node_id, t.received_at
|
|
FROM telemetry t
|
|
INNER JOIN (SELECT node_id, MAX(id) AS max_id FROM telemetry GROUP BY node_id) latest
|
|
ON t.id = latest.max_id
|
|
`)
|
|
if err != nil {
|
|
log.Printf("health monitor: db error: %v", err)
|
|
continue
|
|
}
|
|
|
|
latest := map[string]int64{}
|
|
for rows.Next() {
|
|
var r row
|
|
rows.Scan(&r.nodeID, &r.receivedAt)
|
|
latest[r.nodeID] = r.receivedAt
|
|
}
|
|
rows.Close()
|
|
|
|
var problems []string
|
|
for _, node := range expectedNodes {
|
|
ts, ok := latest[node]
|
|
if !ok {
|
|
problems = append(problems, node+": no data")
|
|
} else if now-ts > staleThreshold {
|
|
problems = append(problems, fmt.Sprintf("%s: silent %ds", node, now-ts))
|
|
}
|
|
}
|
|
|
|
if len(problems) > 0 {
|
|
// Don't push to Kuma — missing beat triggers Kuma alert
|
|
log.Printf("health monitor: PROBLEM — %v", problems)
|
|
if webhookURL != "" && !alerting {
|
|
alerting = true
|
|
body := fmt.Sprintf(`{"text":"⚠️ vault1984 POP alert\n%s"}`,
|
|
strings.Join(problems, "\\n"))
|
|
doPost(webhookURL, body)
|
|
}
|
|
} else {
|
|
// All healthy — push heartbeat to Kuma
|
|
alerting = false
|
|
oldest := int64(0)
|
|
for _, ts := range latest {
|
|
if age := now - ts; age > oldest {
|
|
oldest = age
|
|
}
|
|
}
|
|
pushURL := fmt.Sprintf("%s?status=up&msg=OK+oldest=%ds&ping=%d", kumaURL, oldest, oldest)
|
|
doPost(pushURL, "")
|
|
log.Printf("health monitor: OK — pushed to Kuma (oldest beat %ds)", oldest)
|
|
}
|
|
}
|
|
}
|