diff --git a/clavis/clavis-vault/clavitor-linux-amd64 b/clavis/clavis-vault/clavitor-linux-amd64 index e1d6b45..6cc74f8 100755 Binary files a/clavis/clavis-vault/clavitor-linux-amd64 and b/clavis/clavis-vault/clavitor-linux-amd64 differ diff --git a/clavis/clavis-vault/clavitor-linux-arm64 b/clavis/clavis-vault/clavitor-linux-arm64 index 859afb8..c8f112c 100755 Binary files a/clavis/clavis-vault/clavitor-linux-arm64 and b/clavis/clavis-vault/clavitor-linux-arm64 differ diff --git a/clavitor.com/clavitor-web-linux-amd64 b/clavitor.com/clavitor-web-linux-amd64 index ec7c6da..3047ed5 100755 Binary files a/clavitor.com/clavitor-web-linux-amd64 and b/clavitor.com/clavitor-web-linux-amd64 differ diff --git a/clavitor.com/clavitor.db-shm b/clavitor.com/clavitor.db-shm index e6078b5..ac8ad67 100644 Binary files a/clavitor.com/clavitor.db-shm and b/clavitor.com/clavitor.db-shm differ diff --git a/clavitor.com/clavitor.db-wal b/clavitor.com/clavitor.db-wal index 24f7b84..8be596c 100644 Binary files a/clavitor.com/clavitor.db-wal and b/clavitor.com/clavitor.db-wal differ diff --git a/clavitor.com/templates/hosted.tmpl b/clavitor.com/templates/hosted.tmpl index b3dacd4..41419fb 100644 --- a/clavitor.com/templates/hosted.tmpl +++ b/clavitor.com/templates/hosted.tmpl @@ -140,31 +140,33 @@ const pulseMax = isHQ ? 18 : 13; const tooltip = isLive ? pop.city + ' · Live' : pop.city + ' · Planned · Q2 2026'; - // Pulse ring 1 - const r1 = document.createElementNS(ns, 'circle'); - r1.setAttribute('cx', x); r1.setAttribute('cy', y); - r1.setAttribute('r', pulseR); r1.setAttribute('fill', 'none'); - r1.setAttribute('stroke', pulseColor); r1.setAttribute('stroke-width', isHQ ? '2' : '1.5'); - const a1 = document.createElementNS(ns, 'animate'); - a1.setAttribute('attributeName', 'r'); a1.setAttribute('values', pulseR+';'+pulseMax+';'+pulseR); - a1.setAttribute('dur', '2.4s'); a1.setAttribute('begin', delay+'s'); a1.setAttribute('repeatCount', 'indefinite'); - const a2 = document.createElementNS(ns, 'animate'); - a2.setAttribute('attributeName', 'stroke-opacity'); a2.setAttribute('values', '0.6;0;0.6'); - a2.setAttribute('dur', '2.4s'); a2.setAttribute('begin', delay+'s'); a2.setAttribute('repeatCount', 'indefinite'); - r1.appendChild(a1); r1.appendChild(a2); + // Pulse rings — only for live/HQ pops + let r1, r2; + if (isLive || isHQ) { + r1 = document.createElementNS(ns, 'circle'); + r1.setAttribute('cx', x); r1.setAttribute('cy', y); + r1.setAttribute('r', pulseR); r1.setAttribute('fill', 'none'); + r1.setAttribute('stroke', pulseColor); r1.setAttribute('stroke-width', isHQ ? '2' : '1.5'); + const a1 = document.createElementNS(ns, 'animate'); + a1.setAttribute('attributeName', 'r'); a1.setAttribute('values', pulseR+';'+pulseMax+';'+pulseR); + a1.setAttribute('dur', '2.4s'); a1.setAttribute('begin', delay+'s'); a1.setAttribute('repeatCount', 'indefinite'); + const a2 = document.createElementNS(ns, 'animate'); + a2.setAttribute('attributeName', 'stroke-opacity'); a2.setAttribute('values', '0.6;0;0.6'); + a2.setAttribute('dur', '2.4s'); a2.setAttribute('begin', delay+'s'); a2.setAttribute('repeatCount', 'indefinite'); + r1.appendChild(a1); r1.appendChild(a2); - // Pulse ring 2 - const r2 = document.createElementNS(ns, 'circle'); - r2.setAttribute('cx', x); r2.setAttribute('cy', y); - r2.setAttribute('r', pulseR); r2.setAttribute('fill', 'none'); - r2.setAttribute('stroke', pulseColor); r2.setAttribute('stroke-width', isHQ ? '1.5' : '1'); - const a3 = document.createElementNS(ns, 'animate'); - a3.setAttribute('attributeName', 'r'); a3.setAttribute('values', pulseR+';'+pulseMax+';'+pulseR); - a3.setAttribute('dur', '2.4s'); a3.setAttribute('begin', (delay+0.8)+'s'); a3.setAttribute('repeatCount', 'indefinite'); - const a4 = document.createElementNS(ns, 'animate'); - a4.setAttribute('attributeName', 'stroke-opacity'); a4.setAttribute('values', '0.4;0;0.4'); - a4.setAttribute('dur', '2.4s'); a4.setAttribute('begin', (delay+0.8)+'s'); a4.setAttribute('repeatCount', 'indefinite'); - r2.appendChild(a3); r2.appendChild(a4); + r2 = document.createElementNS(ns, 'circle'); + r2.setAttribute('cx', x); r2.setAttribute('cy', y); + r2.setAttribute('r', pulseR); r2.setAttribute('fill', 'none'); + r2.setAttribute('stroke', pulseColor); r2.setAttribute('stroke-width', isHQ ? '1.5' : '1'); + const a3 = document.createElementNS(ns, 'animate'); + a3.setAttribute('attributeName', 'r'); a3.setAttribute('values', pulseR+';'+pulseMax+';'+pulseR); + a3.setAttribute('dur', '2.4s'); a3.setAttribute('begin', (delay+0.8)+'s'); a3.setAttribute('repeatCount', 'indefinite'); + const a4 = document.createElementNS(ns, 'animate'); + a4.setAttribute('attributeName', 'stroke-opacity'); a4.setAttribute('values', '0.4;0;0.4'); + a4.setAttribute('dur', '2.4s'); a4.setAttribute('begin', (delay+0.8)+'s'); a4.setAttribute('repeatCount', 'indefinite'); + r2.appendChild(a3); r2.appendChild(a4); + } // Square dot const half = dotSize / 2; @@ -189,8 +191,8 @@ labelTitle.textContent = tooltip; label.appendChild(labelTitle); - svg.appendChild(r1); - svg.appendChild(r2); + if (r1) svg.appendChild(r1); + if (r2) svg.appendChild(r2); svg.appendChild(dot); svg.appendChild(label); } diff --git a/operations/pop-sync/Makefile b/operations/pop-sync/Makefile new file mode 100644 index 0000000..afc7b52 --- /dev/null +++ b/operations/pop-sync/Makefile @@ -0,0 +1,17 @@ +PROD_HOST = root@zurich.inou.com +PROD_DIR = /opt/pop-sync + +.PHONY: build deploy clean + +build: + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o pop-sync-linux-amd64 . + +deploy: build + ssh $(PROD_HOST) "mkdir -p $(PROD_DIR)" + scp pop-sync-linux-amd64 $(PROD_HOST):$(PROD_DIR)/pop-sync + scp env.prod $(PROD_HOST):$(PROD_DIR)/.env + ssh $(PROD_HOST) "chmod +x $(PROD_DIR)/pop-sync" + @echo "✓ deployed to $(PROD_HOST):$(PROD_DIR)" + +clean: + rm -f pop-sync-linux-amd64 diff --git a/operations/pop-sync/env.prod b/operations/pop-sync/env.prod new file mode 100644 index 0000000..6494e85 --- /dev/null +++ b/operations/pop-sync/env.prod @@ -0,0 +1,7 @@ +export CF_API_TOKEN=dSVz7JZtyK023q7kh4MMNmIggK1dahWdnBxVnP3O +export TS_API_KEY=tskey-api-k7uLSqJS1121CNTRL-JF18cV5ajMBfE71sBQ5ZMB85zZJtKzVx +export TS_AUTHKEY=tskey-auth-kuYFWy9jka11CNTRL-gskGJwSfcWfoF9XQNpmMWfNittrHzgcbc +export AWS_ACCESS_KEY_ID=AKIA4QTBZQ4D4VPQEWGL +export AWS_SECRET_ACCESS_KEY=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 +export SERVE_IP=185.218.204.47 +export VAULT_BINARY=/opt/pop-sync/clavitor-arm64 diff --git a/operations/pop-sync/go.mod b/operations/pop-sync/go.mod index 6584d1c..76da1c3 100644 --- a/operations/pop-sync/go.mod +++ b/operations/pop-sync/go.mod @@ -3,18 +3,18 @@ module clavitor.ai/pop-sync go 1.26.1 require ( - github.com/aws/aws-sdk-go-v2 v1.41.4 // indirect + github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect github.com/aws/aws-sdk-go-v2/config v1.32.12 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.19.12 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.0 // indirect github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect - github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssm v1.68.4 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect diff --git a/operations/pop-sync/go.sum b/operations/pop-sync/go.sum index 38e3cbd..3a01bce 100644 --- a/operations/pop-sync/go.sum +++ b/operations/pop-sync/go.sum @@ -1,5 +1,7 @@ github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= +github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV/yY= +github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= @@ -8,8 +10,12 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqb github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21/go.mod h1:p+hz+PRAYlY3zcpJhPwXlLC4C+kqn70WIHwnzAfs6ps= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= github.com/aws/aws-sdk-go-v2/service/ec2 v1.296.0 h1:98Miqj16un1WLNyM1RjVDhXYumhqZrQfAeG8i4jPG6o= @@ -22,6 +28,8 @@ github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNB github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3 h1:bBoWhx8lsFLTXintRX64ZBXcmFZbGqUmaPUrjXECqIc= github.com/aws/aws-sdk-go-v2/service/ssm v1.68.3/go.mod h1:rcRkKbUJ2437WuXdq9fbj+MjTudYWzY9Ct8kiBbN8a8= +github.com/aws/aws-sdk-go-v2/service/ssm v1.68.4 h1:5Wg8AAAnIWM2LE/0KFGqllZff96bm4dBs+uerYFfReE= +github.com/aws/aws-sdk-go-v2/service/ssm v1.68.4/go.mod h1:nph0ypDLWm9D9iA9zOX39W/N+A4GqwzlxA13jzXVD4k= github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= diff --git a/operations/pop-sync/main.go b/operations/pop-sync/main.go index 7ca80d7..069de30 100644 --- a/operations/pop-sync/main.go +++ b/operations/pop-sync/main.go @@ -74,12 +74,12 @@ type Config struct { CFToken string TSKey string TSAuthKey string - HansHost string DryRun bool JSONOut bool Zone string CFZoneID string - VaultSrc string // path to clovis-vault source + VaultSrc string // path to vault source (for local builds) + VaultBinary string // path to pre-built vault binary (takes precedence) Nodes string // comma-separated node filter (empty = all) AWSKeyID string AWSSecretKey string @@ -126,6 +126,15 @@ func main() { fatal("usage: pop-sync provision [city2 ...] (e.g. pop-sync provision Tokyo Calgary)") } exitWith(cmdProvision(cfg, remaining)) + case "bootstrap": + if len(remaining) < 2 { + fatal("usage: pop-sync bootstrap [root-password] (omit password if SSH key installed)") + } + password := "" + if len(remaining) >= 3 { + password = remaining[2] + } + exitWith(cmdBootstrap(cfg, remaining[0], remaining[1], password)) case "maintenance": if len(remaining) == 0 { fatal("usage: pop-sync maintenance [reason]") @@ -155,7 +164,7 @@ Flags: -cf-token Cloudflare API token (or CF_API_TOKEN env) -ts-key Tailscale API key (or TS_API_KEY env) -ts-authkey Tailscale auth key for joining new nodes (or TS_AUTHKEY env) - -hans SSH target for Hans/SSM relay (default: johan@185.218.204.47) + -binary Path to pre-built vault binary (skips local build) -vault-src Path to clovis-vault source (default: ../clavis/clavis-vault) -zone DNS zone (default: clavitor.ai) -nodes Comma-separated node filter, e.g. "use1,sg1" (default: all) @@ -197,10 +206,10 @@ func parseFlags() (Config, []string) { cfg.TSKey = next() case "-ts-authkey": cfg.TSAuthKey = next() - case "-hans": - cfg.HansHost = next() case "-vault-src": cfg.VaultSrc = next() + case "-binary": + cfg.VaultBinary = next() case "-zone": cfg.Zone = next() case "-cf-zone-id": @@ -233,8 +242,8 @@ func parseFlags() (Config, []string) { if cfg.TSAuthKey == "" { cfg.TSAuthKey = os.Getenv("TS_AUTHKEY") } - if cfg.HansHost == "" { - cfg.HansHost = "johan@185.218.204.47" + if cfg.VaultBinary == "" { + cfg.VaultBinary = os.Getenv("VAULT_BINARY") } if cfg.VaultSrc == "" { cfg.VaultSrc = "../clavis/clavis-vault" @@ -323,7 +332,7 @@ func cmdDeploy(cfg Config) []NodeResult { // Step 1: Build log(cfg, "\n--- Build ---") - binaryPath := buildVault(cfg) + binaryPath := resolveVaultBinary(cfg) log(cfg, "Built: %s", binaryPath) if cfg.DryRun { @@ -335,45 +344,28 @@ func cmdDeploy(cfg Config) []NodeResult { return results } - // Step 2: Upload binary to Hans and serve it temporarily via HTTP + // Step 2: Deploy via Tailscale SCP + SSH log(cfg, "\n--- Deploy to %d nodes ---", len(pops)) - hansDir := "/tmp/clavitor-serve" - hansFile := hansDir + "/clavitor" - log(cfg, "Uploading binary to Hans...") - sshExec(cfg.HansHost, "mkdir -p "+hansDir) - if err := scpToHost(cfg.HansHost, binaryPath, hansFile); err != nil { - fatal("scp to hans: %v", err) - } - - // Start a temporary HTTP server on Hans (port 9876) - log(cfg, "Starting temporary file server on Hans:9876...") - sshExec(cfg.HansHost, "pkill -f 'http.server 9876' 2>/dev/null; exit 0") - sshExec(cfg.HansHost, "sudo iptables -I INPUT -p tcp --dport 9876 -j ACCEPT") - time.Sleep(500 * time.Millisecond) - sshBackground(cfg.HansHost, fmt.Sprintf( - "cd %s && exec python3 -m http.server 9876 --bind 0.0.0.0 >/dev/null 2>&1", hansDir)) - time.Sleep(2 * time.Second) - hansPublicIP := "185.218.204.47" - downloadURL := fmt.Sprintf("http://%s:9876/clavitor", hansPublicIP) results := parallelExec(pops, 4, func(p POP) NodeResult { name := p.Subdomain() r := NodeResult{Node: name, Action: "deploy"} - if p.InstanceID == "" { - r.Error = "no instance_id — cannot deploy via SSM" + log(cfg, " [%s] uploading binary...", name) + + // SCP binary + if err := scpToNode(name, binaryPath, "/tmp/clavitor-new"); err != nil { + r.Error = fmt.Sprintf("scp: %v", err) return r } - log(cfg, " [%s] deploying via SSM...", name) + // Install via SSH + installScript := fmt.Sprintf(`set -e +mkdir -p /opt/clavitor/bin /opt/clavitor/data /opt/clavitor/certs +mv /tmp/clavitor-new /opt/clavitor/bin/clavitor +chmod +x /opt/clavitor/bin/clavitor - installCmds := []string{ - "mkdir -p /opt/clavitor/bin /opt/clavitor/data", - fmt.Sprintf("curl -sf -o /tmp/clavitor-new %s", downloadURL), - "mv /tmp/clavitor-new /opt/clavitor/bin/clavitor", - "chmod +x /opt/clavitor/bin/clavitor", - - fmt.Sprintf(`cat > /opt/clavitor/env << 'ENVEOF' +cat > /opt/clavitor/env << 'ENVEOF' PORT=1984 VAULT_MODE=hosted DATA_DIR=/opt/clavitor/data @@ -384,9 +376,9 @@ TLS_DOMAIN=%s CF_API_TOKEN=dSVz7JZtyK023q7kh4MMNmIggK1dahWdnBxVnP3O TLS_CERT_DIR=/opt/clavitor/certs TLS_EMAIL=ops@clavitor.ai -ENVEOF`, p.DNS), +ENVEOF - `test -f /etc/systemd/system/clavitor.service || cat > /etc/systemd/system/clavitor.service << 'UNITEOF' +test -f /etc/systemd/system/clavitor.service || cat > /etc/systemd/system/clavitor.service << 'UNITEOF' [Unit] Description=Clavitor Vault After=network-online.target @@ -403,18 +395,17 @@ User=root [Install] WantedBy=multi-user.target -UNITEOF`, +UNITEOF - "systemctl daemon-reload", - "systemctl enable clavitor", - "systemctl restart clavitor", - "sleep 2", - "systemctl is-active clavitor", - } +systemctl daemon-reload +systemctl enable clavitor +systemctl restart clavitor +sleep 2 +systemctl is-active clavitor`, p.DNS) - out, err := ssmRunCommand(cfg.HansHost, p.InstanceID, p.RegionName, installCmds) + out, err := tsSshExec(name, installScript) if err != nil { - r.Error = fmt.Sprintf("deploy: %v\n%s", err, out) + r.Error = fmt.Sprintf("install: %v\n%s", err, out) log(cfg, " [%s] FAIL: %v", name, err) return r } @@ -425,16 +416,19 @@ UNITEOF`, return r }) - // Kill the temp HTTP server, close firewall, clean up - sshExec(cfg.HansHost, "pkill -f 'http.server 9876' 2>/dev/null; exit 0") - sshExec(cfg.HansHost, "sudo iptables -D INPUT -p tcp --dport 9876 -j ACCEPT 2>/dev/null; exit 0") - sshExec(cfg.HansHost, "rm -rf "+hansDir) - outputResults(cfg, results) return results } -func buildVault(cfg Config) string { +func resolveVaultBinary(cfg Config) string { + // Pre-built binary takes precedence + if cfg.VaultBinary != "" { + if _, err := os.Stat(cfg.VaultBinary); err != nil { + fatal("vault binary not found: %s", cfg.VaultBinary) + } + return cfg.VaultBinary + } + // Fall back to building from source srcDir, _ := filepath.Abs(cfg.VaultSrc) outPath := filepath.Join(srcDir, "clavitor-linux-arm64") @@ -937,30 +931,21 @@ func provisionNode(cfg Config, pop POP) NodeResult { } } - // --- Step 5: Deploy vault binary --- + // --- Step 5: Deploy vault binary via Tailscale SCP + SSH --- log(cfg, " [%s] deploying vault...", pop.City) - binaryPath := buildVault(cfg) + binaryPath := resolveVaultBinary(cfg) - hansDir := "/tmp/clavitor-serve" - hansFile := hansDir + "/clavitor" - sshExec(cfg.HansHost, "mkdir -p "+hansDir) - if err := scpToHost(cfg.HansHost, binaryPath, hansFile); err != nil { - r.Error = fmt.Sprintf("scp to hans: %v", err) + if err := scpToNode(sub, binaryPath, "/tmp/clavitor-new"); err != nil { + r.Error = fmt.Sprintf("scp binary: %v", err) return r } - sshExec(cfg.HansHost, "pkill -f 'http.server 9876' 2>/dev/null; exit 0") - sshExec(cfg.HansHost, "sudo iptables -I INPUT -p tcp --dport 9876 -j ACCEPT") - time.Sleep(500 * time.Millisecond) - sshBackground(cfg.HansHost, fmt.Sprintf("cd %s && exec python3 -m http.server 9876 --bind 0.0.0.0 >/dev/null 2>&1", hansDir)) - time.Sleep(2 * time.Second) - downloadURL := fmt.Sprintf("http://185.218.204.47:9876/clavitor") - installCmds := []string{ - "mkdir -p /opt/clavitor/bin /opt/clavitor/data /opt/clavitor/certs", - fmt.Sprintf("curl -sf -o /tmp/clavitor-new %s", downloadURL), - "mv /tmp/clavitor-new /opt/clavitor/bin/clavitor", - "chmod +x /opt/clavitor/bin/clavitor", - fmt.Sprintf(`cat > /opt/clavitor/env << 'ENVEOF' + installScript := fmt.Sprintf(`set -e +mkdir -p /opt/clavitor/bin /opt/clavitor/data /opt/clavitor/certs +mv /tmp/clavitor-new /opt/clavitor/bin/clavitor +chmod +x /opt/clavitor/bin/clavitor + +cat > /opt/clavitor/env << 'ENVEOF' PORT=1984 VAULT_MODE=hosted DATA_DIR=/opt/clavitor/data @@ -971,8 +956,9 @@ TLS_DOMAIN=%s CF_API_TOKEN=dSVz7JZtyK023q7kh4MMNmIggK1dahWdnBxVnP3O TLS_CERT_DIR=/opt/clavitor/certs TLS_EMAIL=ops@clavitor.ai -ENVEOF`, dns), - `cat > /etc/systemd/system/clavitor.service << 'UNITEOF' +ENVEOF + +cat > /etc/systemd/system/clavitor.service << 'UNITEOF' [Unit] Description=Clavitor Vault After=network-online.target @@ -989,20 +975,15 @@ User=root [Install] WantedBy=multi-user.target -UNITEOF`, - "systemctl daemon-reload", - "systemctl enable clavitor", - "systemctl restart clavitor", - "sleep 3", - "systemctl is-active clavitor", - } +UNITEOF - out, err := ssmRunCommand(cfg.HansHost, instanceID, region, installCmds) - // Clean up Hans - sshExec(cfg.HansHost, "pkill -f 'http.server 9876' 2>/dev/null; exit 0") - sshExec(cfg.HansHost, "sudo iptables -D INPUT -p tcp --dport 9876 -j ACCEPT 2>/dev/null; exit 0") - sshExec(cfg.HansHost, "rm -rf "+hansDir) +systemctl daemon-reload +systemctl enable clavitor +systemctl restart clavitor +sleep 3 +systemctl is-active clavitor`, dns) + out, err := tsSshExec(sub, installScript) if err != nil { r.Error = fmt.Sprintf("deploy: %v\n%s", err, out) return r @@ -1027,6 +1008,234 @@ UNITEOF`, return r } +// --- Subcommand: bootstrap --- +// For non-AWS instances: takes an IP + password, installs Tailscale, hardens, deploys vault. + +func cmdBootstrap(cfg Config, city, ip, password string) []NodeResult { + // Find the POP in the DB + allPOPs, err := readPOPs(cfg.DBPath) + if err != nil { + fatal("reading DB: %v", err) + } + + var pop *POP + for _, p := range allPOPs { + if strings.EqualFold(p.City, city) { + pop = &p + break + } + } + if pop == nil { + fatal("city %q not found in DB", city) + } + + sub := pop.Subdomain() + if sub == "" { + sub = strings.ToLower(pop.Country) + "1" + } + dns := sub + "." + cfg.Zone + if pop.DNS == "" { + pop.DNS = dns + } else { + dns = pop.DNS + sub = pop.Subdomain() + } + + r := NodeResult{Node: city, Action: "bootstrap"} + + if cfg.DryRun { + r.OK = true + r.Message = fmt.Sprintf("would bootstrap %s (%s) as %s", city, ip, dns) + log(cfg, "DRY RUN: %s", r.Message) + return []NodeResult{r} + } + + // Helper: run command on fresh instance (password or key auth) + sshRaw := func(command string) (string, error) { + var cmd *exec.Cmd + sshOpts := []string{"-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR"} + if password != "" { + cmd = exec.Command("sshpass", append([]string{"-p", password, "ssh"}, append(sshOpts, "root@"+ip, command)...)...) + } else { + cmd = exec.Command("ssh", append(sshOpts, "root@"+ip, command)...) + } + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + if err != nil { + return stdout.String() + stderr.String(), err + } + return stdout.String(), nil + } + + scpRaw := func(localPath, remotePath string) error { + sshOpts := []string{"-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10", "-o", "UserKnownHostsFile=/dev/null", "-o", "LogLevel=ERROR"} + var cmd *exec.Cmd + if password != "" { + cmd = exec.Command("sshpass", append([]string{"-p", password, "scp"}, append(sshOpts, localPath, "root@"+ip+":"+remotePath)...)...) + } else { + cmd = exec.Command("scp", append(sshOpts, localPath, "root@"+ip+":"+remotePath)...) + } + out, err := cmd.CombinedOutput() + if err != nil { + return fmt.Errorf("%v: %s", err, string(out)) + } + return nil + } + + // --- Step 1: Test connectivity --- + log(cfg, " [%s] connecting to %s...", city, ip) + out, err := sshRaw("hostname && uname -m") + if err != nil { + r.Error = fmt.Sprintf("ssh connect: %v\n%s", err, out) + return []NodeResult{r} + } + arch := "amd64" + if strings.Contains(out, "aarch64") { + arch = "arm64" + } + log(cfg, " [%s] connected (%s)", city, strings.TrimSpace(out)) + + // --- Step 2: Install Tailscale --- + log(cfg, " [%s] installing Tailscale...", city) + out, err = sshRaw("command -v tailscale >/dev/null 2>&1 && echo 'already installed' || (curl -fsSL https://tailscale.com/install.sh | sh)") + if err != nil { + r.Error = fmt.Sprintf("tailscale install: %v", err) + return []NodeResult{r} + } + log(cfg, " [%s] %s", city, lastLines(out, 1)) + + // --- Step 3: Join tailnet --- + log(cfg, " [%s] joining tailnet as %s...", city, sub) + if cfg.TSAuthKey == "" { + r.Error = "TS_AUTHKEY required for bootstrap" + return []NodeResult{r} + } + out, err = sshRaw(fmt.Sprintf("systemctl enable --now tailscaled 2>/dev/null; sleep 2; tailscale up --authkey=%s --hostname=%s --ssh --reset; tailscale ip -4", cfg.TSAuthKey, sub)) + if err != nil { + r.Error = fmt.Sprintf("tailscale join: %v", err) + return []NodeResult{r} + } + tsIP := strings.TrimSpace(out) + log(cfg, " [%s] Tailscale IP: %s", city, tsIP) + + // --- Step 4: Harden --- + log(cfg, " [%s] hardening...", city) + sshRaw(fmt.Sprintf("hostnamectl set-hostname %s 2>/dev/null; hostname %s 2>/dev/null", sub, sub)) + sshRaw(`sed -i 's/^#*PasswordAuthentication.*/PasswordAuthentication no/' /etc/ssh/sshd_config && systemctl restart sshd 2>/dev/null || systemctl restart ssh 2>/dev/null`) + // Firewall: allow only 1984 + tailscale + sshRaw(`command -v ufw >/dev/null 2>&1 && (ufw default deny incoming; ufw allow in on tailscale0; ufw allow 1984/tcp; ufw --force enable) || (command -v firewall-cmd >/dev/null 2>&1 && (firewall-cmd --permanent --add-port=1984/tcp; firewall-cmd --reload) || iptables -A INPUT -p tcp --dport 1984 -j ACCEPT)`) + log(cfg, " [%s] hardened (password auth disabled, firewall set)", city) + + // --- Step 5: Deploy vault --- + log(cfg, " [%s] deploying vault (%s)...", city, arch) + binaryPath := resolveVaultBinary(cfg) + // Check if we need the other arch + if arch == "amd64" { + amdPath := strings.Replace(binaryPath, "arm64", "amd64", 1) + if _, err := os.Stat(amdPath); err == nil { + binaryPath = amdPath + } + } + + // Now use Tailscale SSH (password auth may already be disabled) + if err := scpToNode(sub, binaryPath, "/tmp/clavitor-new"); err != nil { + // Fallback to password SCP if Tailscale not ready yet + log(cfg, " [%s] Tailscale SCP not ready, using password...", city) + if err := scpRaw(binaryPath, "/tmp/clavitor-new"); err != nil { + r.Error = fmt.Sprintf("scp binary: %v", err) + return []NodeResult{r} + } + } + + installScript := fmt.Sprintf(`set -e +mkdir -p /opt/clavitor/bin /opt/clavitor/data /opt/clavitor/certs +mv /tmp/clavitor-new /opt/clavitor/bin/clavitor +chmod +x /opt/clavitor/bin/clavitor + +cat > /opt/clavitor/env << 'ENVEOF' +PORT=1984 +VAULT_MODE=hosted +DATA_DIR=/opt/clavitor/data +TELEMETRY_FREQ=30 +TELEMETRY_HOST=https://clavitor.ai/telemetry +TELEMETRY_TOKEN=clavitor-fleet-2026 +TLS_DOMAIN=%s +CF_API_TOKEN=dSVz7JZtyK023q7kh4MMNmIggK1dahWdnBxVnP3O +TLS_CERT_DIR=/opt/clavitor/certs +TLS_EMAIL=ops@clavitor.ai +ENVEOF + +cat > /etc/systemd/system/clavitor.service << 'UNITEOF' +[Unit] +Description=Clavitor Vault +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +ExecStart=/opt/clavitor/bin/clavitor +EnvironmentFile=/opt/clavitor/env +WorkingDirectory=/opt/clavitor/data +Restart=always +RestartSec=5 +User=root + +[Install] +WantedBy=multi-user.target +UNITEOF + +systemctl daemon-reload +systemctl enable clavitor +systemctl restart clavitor +sleep 3 +systemctl is-active clavitor`, dns) + + out, err = tsSshExec(sub, installScript) + if err != nil { + // Fallback to password SSH + out, err = sshRaw(installScript) + } + if err != nil { + r.Error = fmt.Sprintf("deploy: %v\n%s", err, out) + return []NodeResult{r} + } + log(cfg, " [%s] service: %s", city, strings.TrimSpace(out)) + + // --- Step 6: Update DB + DNS --- + log(cfg, " [%s] updating DB + DNS...", city) + localDB, _ := sql.Open("sqlite", cfg.DBPath) + localDB.Exec(`UPDATE pops SET ip=?, dns=?, status='live' WHERE pop_id=?`, ip, dns, pop.PopID) + localDB.Close() + + if cfg.CFToken != "" { + zoneID := cfg.CFZoneID + if zoneID == "" { + zoneID, _ = cfResolveZoneID(cfg.CFToken, cfg.Zone) + } + if zoneID != "" { + cfCreateRecord(cfg.CFToken, zoneID, dns, ip) + } + } + + // --- Step 7: Verify TLS --- + log(cfg, " [%s] verifying TLS...", city) + time.Sleep(5 * time.Second) + resp, err := http.Get(fmt.Sprintf("https://%s:1984/ping", dns)) + if err == nil { + resp.Body.Close() + log(cfg, " [%s] TLS verified", city) + } else { + log(cfg, " [%s] TLS not ready yet: %v", city, err) + } + + r.OK = true + r.Message = fmt.Sprintf("%s → %s (%s) — live", ip, dns, arch) + log(cfg, "\n [%s] DONE: %s\n", city, r.Message) + return []NodeResult{r} +} + func ensureSecurityGroup(ctx context.Context, client *ec2.Client, region, city string, cfg Config) (string, error) { // Check if vault1984-pop exists in this region descOut, err := client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ @@ -1081,39 +1290,14 @@ func nodeExec(cfg Config, pop POP, command string) (string, error) { // Tailscale SSH failed — fall back to SSM if we have instance_id if pop.InstanceID != "" { log(cfg, " [%s] TS SSH failed, falling back to SSM...", pop.Subdomain()) - return ssmRunCommand(cfg.HansHost, pop.InstanceID, pop.RegionName, []string{command}) + return ssmRunCommand(pop.InstanceID, pop.RegionName, []string{command}) } return out, err } -// nodePushFile copies a local file to a node, trying SCP (Tailscale) first, falling back to SSM+S3-less transfer via Hans. +// nodePushFile copies a local file to a node via SCP (Tailscale). func nodePushFile(cfg Config, pop POP, localPath, remotePath string) error { - err := scpToNode(pop.Subdomain(), localPath, remotePath) - if err == nil { - return nil - } - - // Tailscale SCP failed — fall back: SCP to Hans, then Hans SCPs to node's public IP - if pop.InstanceID != "" && pop.IP != "" { - log(cfg, " [%s] TS SCP failed, falling back via Hans...", pop.Subdomain()) - hansPath := fmt.Sprintf("/tmp/clavitor-deploy-%s", pop.Subdomain()) - - // Upload to Hans - if err := scpToHost(cfg.HansHost, localPath, hansPath); err != nil { - return fmt.Errorf("scp to hans: %w", err) - } - - // Hans pushes to node via SSM (base64 chunks for small files, or just curl from Hans if reachable) - // For a 14MB binary, use Hans → node SCP via public IP with SSM to temporarily allow it - // Simplest: have the node pull from Hans via Tailscale - pullCmd := fmt.Sprintf("scp -o StrictHostKeyChecking=no -o ConnectTimeout=10 %s:%s %s", - cfg.HansHost, hansPath, remotePath) - _, err := ssmRunCommand(cfg.HansHost, pop.InstanceID, pop.RegionName, []string{pullCmd}) - // Clean up - sshExec(cfg.HansHost, "rm -f "+hansPath) - return err - } - return err + return scpToNode(pop.Subdomain(), localPath, remotePath) } func tsSshExec(hostname, command string) (string, error) { @@ -1151,20 +1335,6 @@ func scpToNode(hostname, localPath, remotePath string) error { return nil } -func scpToHost(host, localPath, remotePath string) error { - cmd := exec.Command("scp", - "-o", "StrictHostKeyChecking=no", - "-o", "ConnectTimeout=10", - localPath, - host+":"+remotePath, - ) - out, err := cmd.CombinedOutput() - if err != nil { - return fmt.Errorf("%v: %s", err, string(out)) - } - return nil -} - // --- SSM (for bootstrap only, before Tailscale is available) --- func ssmBootstrapTailscale(cfg Config, pop POP) error { @@ -1172,7 +1342,7 @@ func ssmBootstrapTailscale(cfg Config, pop POP) error { region := pop.RegionName log(cfg, " Installing Tailscale...") - out, err := ssmRunCommand(cfg.HansHost, pop.InstanceID, region, []string{ + out, err := ssmRunCommand(pop.InstanceID, region, []string{ "command -v tailscale >/dev/null 2>&1 && echo 'already installed' && exit 0", "curl -fsSL https://tailscale.com/install.sh | sh", }) @@ -1182,7 +1352,7 @@ func ssmBootstrapTailscale(cfg Config, pop POP) error { log(cfg, " %s", lastLines(out, 2)) log(cfg, " Joining tailnet as %s...", hostname) - out, err = ssmRunCommand(cfg.HansHost, pop.InstanceID, region, []string{ + out, err = ssmRunCommand(pop.InstanceID, region, []string{ "systemctl enable --now tailscaled 2>/dev/null || true", "sleep 2", fmt.Sprintf("tailscale up --authkey=%s --hostname=%s --ssh --reset", cfg.TSAuthKey, hostname), @@ -1194,7 +1364,7 @@ func ssmBootstrapTailscale(cfg Config, pop POP) error { log(cfg, " TS IP: %s", strings.TrimSpace(out)) log(cfg, " Setting hostname...") - _, _ = ssmRunCommand(cfg.HansHost, pop.InstanceID, region, []string{ + _, _ = ssmRunCommand(pop.InstanceID, region, []string{ fmt.Sprintf("hostnamectl set-hostname %s", hostname), fmt.Sprintf("grep -q '%s' /etc/hosts || echo '127.0.1.1 %s %s.%s' >> /etc/hosts", hostname, hostname, hostname, cfg.Zone), }) @@ -1202,41 +1372,43 @@ func ssmBootstrapTailscale(cfg Config, pop POP) error { return nil } -func ssmRunCommand(hansHost, instanceID, region string, commands []string) (string, error) { +func ssmRunCommand(instanceID, region string, commands []string) (string, error) { params := map[string][]string{"commands": commands} paramsJSON, _ := json.Marshal(params) tmpFile := fmt.Sprintf("/tmp/ssm-%s-%d.json", instanceID, time.Now().UnixNano()) - writeCmd := fmt.Sprintf("cat > %s << 'SSMEOF'\n%s\nSSMEOF", tmpFile, string(paramsJSON)) - if _, err := sshExec(hansHost, writeCmd); err != nil { - return "", fmt.Errorf("writing params: %w", err) - } + os.WriteFile(tmpFile, paramsJSON, 0600) + defer os.Remove(tmpFile) - ssmCmd := fmt.Sprintf( - `aws ssm send-command --instance-ids %s --document-name AWS-RunShellScript --region %s --parameters file://%s --query Command.CommandId --output text && rm -f %s`, - instanceID, region, tmpFile, tmpFile, - ) - cmdID, err := sshExec(hansHost, ssmCmd) + cmd := exec.Command("aws", "ssm", "send-command", + "--instance-ids", instanceID, + "--document-name", "AWS-RunShellScript", + "--region", region, + "--parameters", "file://"+tmpFile, + "--query", "Command.CommandId", + "--output", "text") + out, err := cmd.CombinedOutput() if err != nil { - sshExec(hansHost, "rm -f "+tmpFile) - return "", fmt.Errorf("send-command: %w\n%s", err, cmdID) + return "", fmt.Errorf("send-command: %w\n%s", err, string(out)) } - cmdID = strings.TrimSpace(cmdID) + cmdID := strings.TrimSpace(string(out)) if cmdID == "" { return "", fmt.Errorf("empty command ID returned") } for i := 0; i < 36; i++ { time.Sleep(5 * time.Second) - pollCmd := fmt.Sprintf( - `aws ssm get-command-invocation --command-id %s --instance-id %s --region %s --query '[Status,StandardOutputContent,StandardErrorContent]' --output text 2>/dev/null`, - cmdID, instanceID, region, - ) - result, err := sshExec(hansHost, pollCmd) + poll := exec.Command("aws", "ssm", "get-command-invocation", + "--command-id", cmdID, + "--instance-id", instanceID, + "--region", region, + "--query", "[Status,StandardOutputContent,StandardErrorContent]", + "--output", "text") + result, err := poll.CombinedOutput() if err != nil { continue } - parts := strings.SplitN(strings.TrimSpace(result), "\t", 3) + parts := strings.SplitN(strings.TrimSpace(string(result)), "\t", 3) if len(parts) < 1 { continue } @@ -1258,22 +1430,6 @@ func ssmRunCommand(hansHost, instanceID, region string, commands []string) (stri return "", fmt.Errorf("timeout waiting for SSM command %s", cmdID) } -// sshBackground starts a command on a remote host via SSH, forking to background. -func sshBackground(host, command string) { - exec.Command("ssh", "-f", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10", host, command).Run() -} - -func sshExec(host, command string) (string, error) { - cmd := exec.Command("ssh", "-o", "StrictHostKeyChecking=no", "-o", "ConnectTimeout=10", host, command) - var stdout, stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - err := cmd.Run() - if err != nil { - return stdout.String() + stderr.String(), err - } - return stdout.String(), nil -} // --- DNS sync --- diff --git a/operations/pop-sync/pop-sync b/operations/pop-sync/pop-sync index 1e93ac2..571d5a2 100755 Binary files a/operations/pop-sync/pop-sync and b/operations/pop-sync/pop-sync differ diff --git a/operations/pop-sync/pop-sync-linux-amd64 b/operations/pop-sync/pop-sync-linux-amd64 new file mode 100755 index 0000000..ce73502 Binary files /dev/null and b/operations/pop-sync/pop-sync-linux-amd64 differ