johan
/
clavitor
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: auto-commit uncommitted changes

This commit is contained in:
James 2026-03-27 06:03:32 -04:00
parent b7d6ec31c7
commit fc9f49bf18
13 changed files with 387 additions and 197 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

BIN
clavis/clavis-vault/clavitor-linux-amd64
View File

Binary file not shown.

BIN
clavis/clavis-vault/clavitor-linux-arm64
View File

Binary file not shown.

BIN
clavitor.com/clavitor-web-linux-amd64
View File

Binary file not shown.

BIN
clavitor.com/clavitor.db-shm
View File

Binary file not shown.

BIN
clavitor.com/clavitor.db-wal
View File

Binary file not shown.

54
clavitor.com/templates/hosted.tmpl
View File

@ -140,31 +140,33 @@
const pulseMax = isHQ ? 18 : 13;
const tooltip = isLive ? pop.city + ' · Live' : pop.city + ' · Planned · Q2 2026';
// Pulse ring 1
const r1 = document.createElementNS(ns, 'circle');
r1.setAttribute('cx', x); r1.setAttribute('cy', y);
r1.setAttribute('r', pulseR); r1.setAttribute('fill', 'none');
r1.setAttribute('stroke', pulseColor); r1.setAttribute('stroke-width', isHQ ? '2' : '1.5');
const a1 = document.createElementNS(ns, 'animate');
a1.setAttribute('attributeName', 'r'); a1.setAttribute('values', pulseR+';'+pulseMax+';'+pulseR);
a1.setAttribute('dur', '2.4s'); a1.setAttribute('begin', delay+'s'); a1.setAttribute('repeatCount', 'indefinite');
const a2 = document.createElementNS(ns, 'animate');
a2.setAttribute('attributeName', 'stroke-opacity'); a2.setAttribute('values', '0.6;0;0.6');
a2.setAttribute('dur', '2.4s'); a2.setAttribute('begin', delay+'s'); a2.setAttribute('repeatCount', 'indefinite');
r1.appendChild(a1); r1.appendChild(a2);
// Pulse rings — only for live/HQ pops
let r1, r2;
if (isLive || isHQ) {
r1 = document.createElementNS(ns, 'circle');
r1.setAttribute('cx', x); r1.setAttribute('cy', y);
r1.setAttribute('r', pulseR); r1.setAttribute('fill', 'none');
r1.setAttribute('stroke', pulseColor); r1.setAttribute('stroke-width', isHQ ? '2' : '1.5');
const a1 = document.createElementNS(ns, 'animate');
a1.setAttribute('attributeName', 'r'); a1.setAttribute('values', pulseR+';'+pulseMax+';'+pulseR);
a1.setAttribute('dur', '2.4s'); a1.setAttribute('begin', delay+'s'); a1.setAttribute('repeatCount', 'indefinite');
const a2 = document.createElementNS(ns, 'animate');
a2.setAttribute('attributeName', 'stroke-opacity'); a2.setAttribute('values', '0.6;0;0.6');
a2.setAttribute('dur', '2.4s'); a2.setAttribute('begin', delay+'s'); a2.setAttribute('repeatCount', 'indefinite');
r1.appendChild(a1); r1.appendChild(a2);
// Pulse ring 2
const r2 = document.createElementNS(ns, 'circle');
r2.setAttribute('cx', x); r2.setAttribute('cy', y);
r2.setAttribute('r', pulseR); r2.setAttribute('fill', 'none');
r2.setAttribute('stroke', pulseColor); r2.setAttribute('stroke-width', isHQ ? '1.5' : '1');
const a3 = document.createElementNS(ns, 'animate');
a3.setAttribute('attributeName', 'r'); a3.setAttribute('values', pulseR+';'+pulseMax+';'+pulseR);
a3.setAttribute('dur', '2.4s'); a3.setAttribute('begin', (delay+0.8)+'s'); a3.setAttribute('repeatCount', 'indefinite');
const a4 = document.createElementNS(ns, 'animate');
a4.setAttribute('attributeName', 'stroke-opacity'); a4.setAttribute('values', '0.4;0;0.4');
a4.setAttribute('dur', '2.4s'); a4.setAttribute('begin', (delay+0.8)+'s'); a4.setAttribute('repeatCount', 'indefinite');
r2.appendChild(a3); r2.appendChild(a4);
r2 = document.createElementNS(ns, 'circle');
r2.setAttribute('cx', x); r2.setAttribute('cy', y);
r2.setAttribute('r', pulseR); r2.setAttribute('fill', 'none');
r2.setAttribute('stroke', pulseColor); r2.setAttribute('stroke-width', isHQ ? '1.5' : '1');
const a3 = document.createElementNS(ns, 'animate');
a3.setAttribute('attributeName', 'r'); a3.setAttribute('values', pulseR+';'+pulseMax+';'+pulseR);
a3.setAttribute('dur', '2.4s'); a3.setAttribute('begin', (delay+0.8)+'s'); a3.setAttribute('repeatCount', 'indefinite');
const a4 = document.createElementNS(ns, 'animate');
a4.setAttribute('attributeName', 'stroke-opacity'); a4.setAttribute('values', '0.4;0;0.4');
a4.setAttribute('dur', '2.4s'); a4.setAttribute('begin', (delay+0.8)+'s'); a4.setAttribute('repeatCount', 'indefinite');
r2.appendChild(a3); r2.appendChild(a4);
}
// Square dot
const half = dotSize / 2;
@ -189,8 +191,8 @@
labelTitle.textContent = tooltip;
label.appendChild(labelTitle);
svg.appendChild(r1);
svg.appendChild(r2);
if (r1) svg.appendChild(r1);
if (r2) svg.appendChild(r2);
svg.appendChild(dot);
svg.appendChild(label);
}

17
operations/pop-sync/Makefile Normal file
View File

@ -0,0 +1,17 @@
PROD_HOST = root@zurich.inou.com
PROD_DIR = /opt/pop-sync
.PHONY: build deploy clean
build:
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o pop-sync-linux-amd64 .
deploy: build
ssh $(PROD_HOST) "mkdir -p $(PROD_DIR)"
scp pop-sync-linux-amd64 $(PROD_HOST):$(PROD_DIR)/pop-sync
scp env.prod $(PROD_HOST):$(PROD_DIR)/.env
ssh $(PROD_HOST) "chmod +x $(PROD_DIR)/pop-sync"
@echo "✓ deployed to $(PROD_HOST):$(PROD_DIR)"
clean:
rm -f pop-sync-linux-amd64

7
operations/pop-sync/env.prod Normal file
View File

@ -0,0 +1,7 @@
export CF_API_TOKEN=dSVz7JZtyK023q7kh4MMNmIggK1dahWdnBxVnP3O
export TS_API_KEY=tskey-api-k7uLSqJS1121CNTRL-JF18cV5ajMBfE71sBQ5ZMB85zZJtKzVx
export TS_AUTHKEY=tskey-auth-kuYFWy9jka11CNTRL-gskGJwSfcWfoF9XQNpmMWfNittrHzgcbc
export AWS_ACCESS_KEY_ID=AKIA4QTBZQ4D4VPQEWGL
export AWS_SECRET_ACCESS_KEY=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
export SERVE_IP=185.218.204.47
export VAULT_BINARY=/opt/pop-sync/clavitor-arm64

8
operations/pop-sync/go.mod
View File

@ -3,18 +3,18 @@ module clavitor.ai/pop-sync
go 1.26.1
require (
github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect
github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect
github.com/aws/aws-sdk-go-v2/config v1.32.12 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect
github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.0 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect
github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3 // indirect
github.com/aws/aws-sdk-go-v2/service/ssm v1.68.4 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect

8
operations/pop-sync/go.sum
View File

@ -1,5 +1,7 @@
github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k=
github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY=
github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o=
github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0=
github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g=
github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8=
@ -8,8 +10,12 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqb
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw=
github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY=
github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.0 h1:98Miqj16un1WLNyM1RjVDhXYumhqZrQfAeG8i4jPG6o=
@ -22,6 +28,8 @@ github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNB
github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE=
github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3 h1:bBoWhx8lsFLTXintRX64ZBXcmFZbGqUmaPUrjXECqIc=
github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3/go.mod h1:rcRkKbUJ2437WuXdq9fbj+MjTudYWzY9Ct8kiBbN8a8=
github.com/aws/aws-sdk-go-v2/service/ssm v1.68.4 h1:5Wg8AAAnIWM2LE/0KFGqllZff96bm4dBs+uerYFfReE=
github.com/aws/aws-sdk-go-v2/service/ssm v1.68.4/go.mod h1:nph0ypDLWm9D9iA9zOX39W/N+A4GqwzlxA13jzXVD4k=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o=
github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8=
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU=

490
operations/pop-sync/main.go
View File

@ -74,12 +74,12 @@ type Config struct {
CFToken string
TSKey string
TSAuthKey string
HansHost string
DryRun bool
JSONOut bool
Zone string
CFZoneID string
VaultSrc string // path to clovis-vault source
VaultSrc string // path to vault source (for local builds)
VaultBinary string // path to pre-built vault binary (takes precedence)
Nodes string // comma-separated node filter (empty = all)
AWSKeyID string
AWSSecretKey string
@ -126,6 +126,15 @@ func main() {
fatal("usage: pop-sync provision <city> [city2 ...] (e.g. pop-sync provision Tokyo Calgary)")
}
exitWith(cmdProvision(cfg, remaining))
case "bootstrap":
if len(remaining) < 2 {
fatal("usage: pop-sync bootstrap <city> <ip> [root-password] (omit password if SSH key installed)")
}
password := ""
if len(remaining) >= 3 {
password = remaining[2]
}
exitWith(cmdBootstrap(cfg, remaining[0], remaining[1], password))
case "maintenance":
if len(remaining) == 0 {
fatal("usage: pop-sync maintenance <on|off> [reason]")
@ -155,7 +164,7 @@ Flags:
-cf-token Cloudflare API token (or CF_API_TOKEN env)
-ts-key Tailscale API key (or TS_API_KEY env)
-ts-authkey Tailscale auth key for joining new nodes (or TS_AUTHKEY env)
-hans SSH target for Hans/SSM relay (default: johan@185.218.204.47)
-binary Path to pre-built vault binary (skips local build)
-vault-src Path to clovis-vault source (default: ../clavis/clavis-vault)
-zone DNS zone (default: clavitor.ai)
-nodes Comma-separated node filter, e.g. "use1,sg1" (default: all)
@ -197,10 +206,10 @@ func parseFlags() (Config, []string) {
cfg.TSKey = next()
case "-ts-authkey":
cfg.TSAuthKey = next()
case "-hans":
cfg.HansHost = next()
case "-vault-src":
cfg.VaultSrc = next()
case "-binary":
cfg.VaultBinary = next()
case "-zone":
cfg.Zone = next()
case "-cf-zone-id":
@ -233,8 +242,8 @@ func parseFlags() (Config, []string) {
if cfg.TSAuthKey == "" {
cfg.TSAuthKey = os.Getenv("TS_AUTHKEY")
}
if cfg.HansHost == "" {
cfg.HansHost = "johan@185.218.204.47"
if cfg.VaultBinary == "" {
cfg.VaultBinary = os.Getenv("VAULT_BINARY")
}
if cfg.VaultSrc == "" {
cfg.VaultSrc = "../clavis/clavis-vault"
@ -323,7 +332,7 @@ func cmdDeploy(cfg Config) []NodeResult {
// Step 1: Build
log(cfg, "\n--- Build ---")
binaryPath := buildVault(cfg)
binaryPath := resolveVaultBinary(cfg)
log(cfg, "Built: %s", binaryPath)
if cfg.DryRun {
@ -335,45 +344,28 @@ func cmdDeploy(cfg Config) []NodeResult {
return results
}
// Step 2: Upload binary to Hans and serve it temporarily via HTTP
// Step 2: Deploy via Tailscale SCP + SSH
log(cfg, "\n--- Deploy to %d nodes ---", len(pops))
hansDir := "/tmp/clavitor-serve"
hansFile := hansDir + "/clavitor"
log(cfg, "Uploading binary to Hans...")
sshExec(cfg.HansHost, "mkdir -p "+hansDir)
if err := scpToHost(cfg.HansHost, binaryPath, hansFile); err != nil {
fatal("scp to hans: %v", err)
}
// Start a temporary HTTP server on Hans (port 9876)
log(cfg, "Starting temporary file server on Hans:9876...")
sshExec(cfg.HansHost, "pkill -f 'http.server 9876' 2>/dev/null; exit 0")
sshExec(cfg.HansHost, "sudo iptables -I INPUT -p tcp --dport 9876 -j ACCEPT")
time.Sleep(500 * time.Millisecond)
sshBackground(cfg.HansHost, fmt.Sprintf(
"cd %s && exec python3 -m http.server 9876 --bind 0.0.0.0 >/dev/null 2>&1", hansDir))
time.Sleep(2 * time.Second)
hansPublicIP := "185.218.204.47"
downloadURL := fmt.Sprintf("http://%s:9876/clavitor", hansPublicIP)
results := parallelExec(pops, 4, func(p POP) NodeResult {
name := p.Subdomain()
r := NodeResult{Node: name, Action: "deploy"}
if p.InstanceID == "" {
r.Error = "no instance_id — cannot deploy via SSM"
log(cfg, " [%s] uploading binary...", name)
// SCP binary
if err := scpToNode(name, binaryPath, "/tmp/clavitor-new"); err != nil {
r.Error = fmt.Sprintf("scp: %v", err)
return r
}
log(cfg, " [%s] deploying via SSM...", name)
// Install via SSH
installScript := fmt.Sprintf(`set -e
mkdir -p /opt/clavitor/bin /opt/clavitor/data /opt/clavitor/certs
mv /tmp/clavitor-new /opt/clavitor/bin/clavitor
chmod +x /opt/clavitor/bin/clavitor
installCmds := []string{
"mkdir -p /opt/clavitor/bin /opt/clavitor/data",
fmt.Sprintf("curl -sf -o /tmp/clavitor-new %s", downloadURL),
"mv /tmp/clavitor-new /opt/clavitor/bin/clavitor",
"chmod +x /opt/clavitor/bin/clavitor",
fmt.Sprintf(`cat > /opt/clavitor/env << 'ENVEOF'
cat > /opt/clavitor/env << 'ENVEOF'
PORT=1984
VAULT_MODE=hosted
DATA_DIR=/opt/clavitor/data
@ -384,9 +376,9 @@ TLS_DOMAIN=%s
CF_API_TOKEN=dSVz7JZtyK023q7kh4MMNmIggK1dahWdnBxVnP3O
TLS_CERT_DIR=/opt/clavitor/certs
TLS_EMAIL=ops@clavitor.ai
ENVEOF`, p.DNS),
ENVEOF
`test -f /etc/systemd/system/clavitor.service || cat > /etc/systemd/system/clavitor.service << 'UNITEOF'
test -f /etc/systemd/system/clavitor.service || cat > /etc/systemd/system/clavitor.service << 'UNITEOF'
[Unit]
Description=Clavitor Vault
After=network-online.target
@ -403,18 +395,17 @@ User=root
[Install]
WantedBy=multi-user.target
UNITEOF`,
UNITEOF
"systemctl daemon-reload",
"systemctl enable clavitor",
"systemctl restart clavitor",
"sleep 2",
"systemctl is-active clavitor",
}
systemctl daemon-reload
systemctl enable clavitor
systemctl restart clavitor
sleep 2
systemctl is-active clavitor`, p.DNS)
out, err := ssmRunCommand(cfg.HansHost, p.InstanceID, p.RegionName, installCmds)
out, err := tsSshExec(name, installScript)
if err != nil {
r.Error = fmt.Sprintf("deploy: %v\n%s", err, out)
r.Error = fmt.Sprintf("install: %v\n%s", err, out)
log(cfg, " [%s] FAIL: %v", name, err)
return r
}
@ -425,16 +416,19 @@ UNITEOF`,
return r
})
// Kill the temp HTTP server, close firewall, clean up
sshExec(cfg.HansHost, "pkill -f 'http.server 9876' 2>/dev/null; exit 0")
sshExec(cfg.HansHost, "sudo iptables -D INPUT -p tcp --dport 9876 -j ACCEPT 2>/dev/null; exit 0")
sshExec(cfg.HansHost, "rm -rf "+hansDir)
outputResults(cfg, results)
return results
}
func buildVault(cfg Config) string {
func resolveVaultBinary(cfg Config) string {
// Pre-built binary takes precedence
if cfg.VaultBinary != "" {
if _, err := os.Stat(cfg.VaultBinary); err != nil {
fatal("vault binary not found: %s", cfg.VaultBinary)
}
return cfg.VaultBinary
}
// Fall back to building from source
srcDir, _ := filepath.Abs(cfg.VaultSrc)
outPath := filepath.Join(srcDir, "clavitor-linux-arm64")
@ -937,30 +931,21 @@ func provisionNode(cfg Config, pop POP) NodeResult {
}
}
// --- Step 5: Deploy vault binary ---
// --- Step 5: Deploy vault binary via Tailscale SCP + SSH ---
log(cfg, " [%s] deploying vault...", pop.City)
binaryPath := buildVault(cfg)
binaryPath := resolveVaultBinary(cfg)
hansDir := "/tmp/clavitor-serve"
hansFile := hansDir + "/clavitor"
sshExec(cfg.HansHost, "mkdir -p "+hansDir)
if err := scpToHost(cfg.HansHost, binaryPath, hansFile); err != nil {
r.Error = fmt.Sprintf("scp to hans: %v", err)
if err := scpToNode(sub, binaryPath, "/tmp/clavitor-new"); err != nil {
r.Error = fmt.Sprintf("scp binary: %v", err)
return r
}
sshExec(cfg.HansHost, "pkill -f 'http.server 9876' 2>/dev/null; exit 0")
sshExec(cfg.HansHost, "sudo iptables -I INPUT -p tcp --dport 9876 -j ACCEPT")
time.Sleep(500 * time.Millisecond)
sshBackground(cfg.HansHost, fmt.Sprintf("cd %s && exec python3 -m http.server 9876 --bind 0.0.0.0 >/dev/null 2>&1", hansDir))
time.Sleep(2 * time.Second)
downloadURL := fmt.Sprintf("http://185.218.204.47:9876/clavitor")
installCmds := []string{
"mkdir -p /opt/clavitor/bin /opt/clavitor/data /opt/clavitor/certs",
fmt.Sprintf("curl -sf -o /tmp/clavitor-new %s", downloadURL),
"mv /tmp/clavitor-new /opt/clavitor/bin/clavitor",
"chmod +x /opt/clavitor/bin/clavitor",
fmt.Sprintf(`cat > /opt/clavitor/env << 'ENVEOF'
installScript := fmt.Sprintf(`set -e
mkdir -p /opt/clavitor/bin /opt/clavitor/data /opt/clavitor/certs
mv /tmp/clavitor-new /opt/clavitor/bin/clavitor
chmod +x /opt/clavitor/bin/clavitor
cat > /opt/clavitor/env << 'ENVEOF'
PORT=1984
VAULT_MODE=hosted
DATA_DIR=/opt/clavitor/data
@ -971,8 +956,9 @@ TLS_DOMAIN=%s
CF_API_TOKEN=dSVz7JZtyK023q7kh4MMNmIggK1dahWdnBxVnP3O
TLS_CERT_DIR=/opt/clavitor/certs
TLS_EMAIL=ops@clavitor.ai
ENVEOF`, dns),
`cat > /etc/systemd/system/clavitor.service << 'UNITEOF'
ENVEOF
cat > /etc/systemd/system/clavitor.service << 'UNITEOF'
[Unit]
Description=Clavitor Vault
After=network-online.target
@ -989,20 +975,15 @@ User=root
[Install]
WantedBy=multi-user.target
UNITEOF`,
"systemctl daemon-reload",
"systemctl enable clavitor",
"systemctl restart clavitor",
"sleep 3",
"systemctl is-active clavitor",
}
UNITEOF
out, err := ssmRunCommand(cfg.HansHost, instanceID, region, installCmds)
// Clean up Hans
sshExec(cfg.HansHost, "pkill -f 'http.server 9876' 2>/dev/null; exit 0")
sshExec(cfg.HansHost, "sudo iptables -D INPUT -p tcp --dport 9876 -j ACCEPT 2>/dev/null; exit 0")
sshExec(cfg.HansHost, "rm -rf "+hansDir)
systemctl daemon-reload
systemctl enable clavitor
systemctl restart clavitor
sleep 3
systemctl is-active clavitor`, dns)
out, err := tsSshExec(sub, installScript)
if err != nil {
r.Error = fmt.Sprintf("deploy: %v\n%s", err, out)
return r
@ -1027,6 +1008,234 @@ UNITEOF`,
return r
}
// --- Subcommand: bootstrap ---
// For non-AWS instances: takes an IP + password, installs Tailscale, hardens, deploys vault.
func cmdBootstrap(cfg Config, city, ip, password string) []NodeResult {
// Find the POP in the DB
allPOPs, err := readPOPs(cfg.DBPath)
if err != nil {
fatal("reading DB: %v", err)
}
var pop *POP
for _, p := range allPOPs {
if strings.EqualFold(p.City, city) {
pop = &p
break
}
}
if pop == nil {
fatal("city %q not found in DB", city)
}
sub := pop.Subdomain()
if sub == "" {
sub = strings.ToLower(pop.Country) + "1"
}
dns := sub + "." + cfg.Zone
if pop.DNS == "" {
pop.DNS = dns
} else {
dns = pop.DNS
sub = pop.Subdomain()
}
r := NodeResult{Node: city, Action: "bootstrap"}
if cfg.DryRun {
r.OK = true
r.Message = fmt.Sprintf("would bootstrap %s (%s) as %s", city, ip, dns)
log(cfg, "DRY RUN: %s", r.Message)
return []NodeResult{r}
}
// Helper: run command on fresh instance (password or key auth)
sshRaw := func(command string) (string, error) {
var cmd *exec.Cmd
sshOpts := []string{"-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR"}
if password != "" {
cmd = exec.Command("sshpass", append([]string{"-p", password, "ssh"}, append(sshOpts, "root@"+ip, command)...)...)
} else {
cmd = exec.Command("ssh", append(sshOpts, "root@"+ip, command)...)
}
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if err != nil {
return stdout.String() + stderr.String(), err
}
return stdout.String(), nil
}
scpRaw := func(localPath, remotePath string) error {
sshOpts := []string{"-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR"}
var cmd *exec.Cmd
if password != "" {
cmd = exec.Command("sshpass", append([]string{"-p", password, "scp"}, append(sshOpts, localPath, "root@"+ip+":"+remotePath)...)...)
} else {
cmd = exec.Command("scp", append(sshOpts, localPath, "root@"+ip+":"+remotePath)...)
}
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("%v: %s", err, string(out))
}
return nil
}
// --- Step 1: Test connectivity ---
log(cfg, " [%s] connecting to %s...", city, ip)
out, err := sshRaw("hostname && uname -m")
if err != nil {
r.Error = fmt.Sprintf("ssh connect: %v\n%s", err, out)
return []NodeResult{r}
}
arch := "amd64"
if strings.Contains(out, "aarch64") {
arch = "arm64"
}
log(cfg, " [%s] connected (%s)", city, strings.TrimSpace(out))
// --- Step 2: Install Tailscale ---
log(cfg, " [%s] installing Tailscale...", city)
out, err = sshRaw("command -v tailscale >/dev/null 2>&1 && echo 'already installed' || (curl -fsSL https://tailscale.com/install.sh | sh)")
if err != nil {
r.Error = fmt.Sprintf("tailscale install: %v", err)
return []NodeResult{r}
}
log(cfg, " [%s] %s", city, lastLines(out, 1))
// --- Step 3: Join tailnet ---
log(cfg, " [%s] joining tailnet as %s...", city, sub)
if cfg.TSAuthKey == "" {
r.Error = "TS_AUTHKEY required for bootstrap"
return []NodeResult{r}
}
out, err = sshRaw(fmt.Sprintf("systemctl enable --now tailscaled 2>/dev/null; sleep 2; tailscale up --authkey=%s --hostname=%s --ssh --reset; tailscale ip -4", cfg.TSAuthKey, sub))
if err != nil {
r.Error = fmt.Sprintf("tailscale join: %v", err)
return []NodeResult{r}
}
tsIP := strings.TrimSpace(out)
log(cfg, " [%s] Tailscale IP: %s", city, tsIP)
// --- Step 4: Harden ---
log(cfg, " [%s] hardening...", city)
sshRaw(fmt.Sprintf("hostnamectl set-hostname %s 2>/dev/null; hostname %s 2>/dev/null", sub, sub))
sshRaw(`sed -i 's/^#*PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config && systemctl restart sshd 2>/dev/null || systemctl restart ssh 2>/dev/null`)
// Firewall: allow only 1984 + tailscale
sshRaw(`command -v ufw >/dev/null 2>&1 && (ufw default deny incoming; ufw allow in on tailscale0; ufw allow 1984/tcp; ufw --force enable) || (command -v firewall-cmd >/dev/null 2>&1 && (firewall-cmd --permanent --add-port=1984/tcp; firewall-cmd --reload) || iptables -A INPUT -p tcp --dport 1984 -j ACCEPT)`)
log(cfg, " [%s] hardened (password auth disabled, firewall set)", city)
// --- Step 5: Deploy vault ---
log(cfg, " [%s] deploying vault (%s)...", city, arch)
binaryPath := resolveVaultBinary(cfg)
// Check if we need the other arch
if arch == "amd64" {
amdPath := strings.Replace(binaryPath, "arm64", "amd64", 1)
if _, err := os.Stat(amdPath); err == nil {
binaryPath = amdPath
}
}
// Now use Tailscale SSH (password auth may already be disabled)
if err := scpToNode(sub, binaryPath, "/tmp/clavitor-new"); err != nil {
// Fallback to password SCP if Tailscale not ready yet
log(cfg, " [%s] Tailscale SCP not ready, using password...", city)
if err := scpRaw(binaryPath, "/tmp/clavitor-new"); err != nil {
r.Error = fmt.Sprintf("scp binary: %v", err)
return []NodeResult{r}
}
}
installScript := fmt.Sprintf(`set -e
mkdir -p /opt/clavitor/bin /opt/clavitor/data /opt/clavitor/certs
mv /tmp/clavitor-new /opt/clavitor/bin/clavitor
chmod +x /opt/clavitor/bin/clavitor
cat > /opt/clavitor/env << 'ENVEOF'
PORT=1984
VAULT_MODE=hosted
DATA_DIR=/opt/clavitor/data
TELEMETRY_FREQ=30
TELEMETRY_HOST=https://clavitor.ai/telemetry
TELEMETRY_TOKEN=clavitor-fleet-2026
TLS_DOMAIN=%s
CF_API_TOKEN=dSVz7JZtyK023q7kh4MMNmIggK1dahWdnBxVnP3O
TLS_CERT_DIR=/opt/clavitor/certs
TLS_EMAIL=ops@clavitor.ai
ENVEOF
cat > /etc/systemd/system/clavitor.service << 'UNITEOF'
[Unit]
Description=Clavitor Vault
After=network-online.target
Wants=network-online.target
[Service]
Type=simple
ExecStart=/opt/clavitor/bin/clavitor
EnvironmentFile=/opt/clavitor/env
WorkingDirectory=/opt/clavitor/data
Restart=always
RestartSec=5
User=root
[Install]
WantedBy=multi-user.target
UNITEOF
systemctl daemon-reload
systemctl enable clavitor
systemctl restart clavitor
sleep 3
systemctl is-active clavitor`, dns)
out, err = tsSshExec(sub, installScript)
if err != nil {
// Fallback to password SSH
out, err = sshRaw(installScript)
}
if err != nil {
r.Error = fmt.Sprintf("deploy: %v\n%s", err, out)
return []NodeResult{r}
}
log(cfg, " [%s] service: %s", city, strings.TrimSpace(out))
// --- Step 6: Update DB + DNS ---
log(cfg, " [%s] updating DB + DNS...", city)
localDB, _ := sql.Open("sqlite", cfg.DBPath)
localDB.Exec(`UPDATE pops SET ip=?, dns=?, status='live' WHERE pop_id=?`, ip, dns, pop.PopID)
localDB.Close()
if cfg.CFToken != "" {
zoneID := cfg.CFZoneID
if zoneID == "" {
zoneID, _ = cfResolveZoneID(cfg.CFToken, cfg.Zone)
}
if zoneID != "" {
cfCreateRecord(cfg.CFToken, zoneID, dns, ip)
}
}
// --- Step 7: Verify TLS ---
log(cfg, " [%s] verifying TLS...", city)
time.Sleep(5 * time.Second)
resp, err := http.Get(fmt.Sprintf("https://%s:1984/ping", dns))
if err == nil {
resp.Body.Close()
log(cfg, " [%s] TLS verified", city)
} else {
log(cfg, " [%s] TLS not ready yet: %v", city, err)
}
r.OK = true
r.Message = fmt.Sprintf("%s → %s (%s) — live", ip, dns, arch)
log(cfg, "\n [%s] DONE: %s\n", city, r.Message)
return []NodeResult{r}
}
func ensureSecurityGroup(ctx context.Context, client *ec2.Client, region, city string, cfg Config) (string, error) {
// Check if vault1984-pop exists in this region
descOut, err := client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{
@ -1081,39 +1290,14 @@ func nodeExec(cfg Config, pop POP, command string) (string, error) {
// Tailscale SSH failed — fall back to SSM if we have instance_id
if pop.InstanceID != "" {
log(cfg, " [%s] TS SSH failed, falling back to SSM...", pop.Subdomain())
return ssmRunCommand(cfg.HansHost, pop.InstanceID, pop.RegionName, []string{command})
return ssmRunCommand(pop.InstanceID, pop.RegionName, []string{command})
}
return out, err
}
// nodePushFile copies a local file to a node, trying SCP (Tailscale) first, falling back to SSM+S3-less transfer via Hans.
// nodePushFile copies a local file to a node via SCP (Tailscale).
func nodePushFile(cfg Config, pop POP, localPath, remotePath string) error {
err := scpToNode(pop.Subdomain(), localPath, remotePath)
if err == nil {
return nil
}
// Tailscale SCP failed — fall back: SCP to Hans, then Hans SCPs to node's public IP
if pop.InstanceID != "" && pop.IP != "" {
log(cfg, " [%s] TS SCP failed, falling back via Hans...", pop.Subdomain())
hansPath := fmt.Sprintf("/tmp/clavitor-deploy-%s", pop.Subdomain())
// Upload to Hans
if err := scpToHost(cfg.HansHost, localPath, hansPath); err != nil {
return fmt.Errorf("scp to hans: %w", err)
}
// Hans pushes to node via SSM (base64 chunks for small files, or just curl from Hans if reachable)
// For a 14MB binary, use Hans → node SCP via public IP with SSM to temporarily allow it
// Simplest: have the node pull from Hans via Tailscale
pullCmd := fmt.Sprintf("scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 %s:%s %s",
cfg.HansHost, hansPath, remotePath)
_, err := ssmRunCommand(cfg.HansHost, pop.InstanceID, pop.RegionName, []string{pullCmd})
// Clean up
sshExec(cfg.HansHost, "rm -f "+hansPath)
return err
}
return err
return scpToNode(pop.Subdomain(), localPath, remotePath)
}
func tsSshExec(hostname, command string) (string, error) {
@ -1151,20 +1335,6 @@ func scpToNode(hostname, localPath, remotePath string) error {
return nil
}
func scpToHost(host, localPath, remotePath string) error {
cmd := exec.Command("scp",
"-o", "StrictHostKeyChecking=no",
"-o", "ConnectTimeout=10",
localPath,
host+":"+remotePath,
)
out, err := cmd.CombinedOutput()
if err != nil {
return fmt.Errorf("%v: %s", err, string(out))
}
return nil
}
// --- SSM (for bootstrap only, before Tailscale is available) ---
func ssmBootstrapTailscale(cfg Config, pop POP) error {
@ -1172,7 +1342,7 @@ func ssmBootstrapTailscale(cfg Config, pop POP) error {
region := pop.RegionName
log(cfg, " Installing Tailscale...")
out, err := ssmRunCommand(cfg.HansHost, pop.InstanceID, region, []string{
out, err := ssmRunCommand(pop.InstanceID, region, []string{
"command -v tailscale >/dev/null 2>&1 && echo 'already installed' && exit 0",
"curl -fsSL https://tailscale.com/install.sh | sh",
})
@ -1182,7 +1352,7 @@ func ssmBootstrapTailscale(cfg Config, pop POP) error {
log(cfg, " %s", lastLines(out, 2))
log(cfg, " Joining tailnet as %s...", hostname)
out, err = ssmRunCommand(cfg.HansHost, pop.InstanceID, region, []string{
out, err = ssmRunCommand(pop.InstanceID, region, []string{
"systemctl enable --now tailscaled 2>/dev/null || true",
"sleep 2",
fmt.Sprintf("tailscale up --authkey=%s --hostname=%s --ssh --reset", cfg.TSAuthKey, hostname),
@ -1194,7 +1364,7 @@ func ssmBootstrapTailscale(cfg Config, pop POP) error {
log(cfg, " TS IP: %s", strings.TrimSpace(out))
log(cfg, " Setting hostname...")
_, _ = ssmRunCommand(cfg.HansHost, pop.InstanceID, region, []string{
_, _ = ssmRunCommand(pop.InstanceID, region, []string{
fmt.Sprintf("hostnamectl set-hostname %s", hostname),
fmt.Sprintf("grep -q '%s' /etc/hosts || echo '127.0.1.1 %s %s.%s' >> /etc/hosts", hostname, hostname, hostname, cfg.Zone),
})
@ -1202,41 +1372,43 @@ func ssmBootstrapTailscale(cfg Config, pop POP) error {
return nil
}
func ssmRunCommand(hansHost, instanceID, region string, commands []string) (string, error) {
func ssmRunCommand(instanceID, region string, commands []string) (string, error) {
params := map[string][]string{"commands": commands}
paramsJSON, _ := json.Marshal(params)
tmpFile := fmt.Sprintf("/tmp/ssm-%s-%d.json", instanceID, time.Now().UnixNano())
writeCmd := fmt.Sprintf("cat > %s << 'SSMEOF'\n%s\nSSMEOF", tmpFile, string(paramsJSON))
if _, err := sshExec(hansHost, writeCmd); err != nil {
return "", fmt.Errorf("writing params: %w", err)
}
os.WriteFile(tmpFile, paramsJSON, 0600)
defer os.Remove(tmpFile)
ssmCmd := fmt.Sprintf(
`aws ssm send-command --instance-ids %s --document-name AWS-RunShellScript --region %s --parameters file://%s --query Command.CommandId --output text && rm -f %s`,
instanceID, region, tmpFile, tmpFile,
)
cmdID, err := sshExec(hansHost, ssmCmd)
cmd := exec.Command("aws", "ssm", "send-command",
"--instance-ids", instanceID,
"--document-name", "AWS-RunShellScript",
"--region", region,
"--parameters", "file://"+tmpFile,
"--query", "Command.CommandId",
"--output", "text")
out, err := cmd.CombinedOutput()
if err != nil {
sshExec(hansHost, "rm -f "+tmpFile)
return "", fmt.Errorf("send-command: %w\n%s", err, cmdID)
return "", fmt.Errorf("send-command: %w\n%s", err, string(out))
}
cmdID = strings.TrimSpace(cmdID)
cmdID := strings.TrimSpace(string(out))
if cmdID == "" {
return "", fmt.Errorf("empty command ID returned")
}
for i := 0; i < 36; i++ {
time.Sleep(5 * time.Second)
pollCmd := fmt.Sprintf(
`aws ssm get-command-invocation --command-id %s --instance-id %s --region %s --query '[Status,StandardOutputContent,StandardErrorContent]' --output text 2>/dev/null`,
cmdID, instanceID, region,
)
result, err := sshExec(hansHost, pollCmd)
poll := exec.Command("aws", "ssm", "get-command-invocation",
"--command-id", cmdID,
"--instance-id", instanceID,
"--region", region,
"--query", "[Status,StandardOutputContent,StandardErrorContent]",
"--output", "text")
result, err := poll.CombinedOutput()
if err != nil {
continue
}
parts := strings.SplitN(strings.TrimSpace(result), "\t", 3)
parts := strings.SplitN(strings.TrimSpace(string(result)), "\t", 3)
if len(parts) < 1 {
continue
}
@ -1258,22 +1430,6 @@ func ssmRunCommand(hansHost, instanceID, region string, commands []string) (stri
return "", fmt.Errorf("timeout waiting for SSM command %s", cmdID)
}
// sshBackground starts a command on a remote host via SSH, forking to background.
func sshBackground(host, command string) {
exec.Command("ssh", "-f", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10", host, command).Run()
}
func sshExec(host, command string) (string, error) {
cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10", host, command)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if err != nil {
return stdout.String() + stderr.String(), err
}
return stdout.String(), nil
}
// --- DNS sync ---

BIN
operations/pop-sync/pop-sync
View File

Binary file not shown.

BIN
operations/pop-sync/pop-sync-linux-amd64 Executable file
View File

Binary file not shown.