johan
/
clavitor
1
0
Fork 0
Code Issues 4 Pull Requests 6 Packages Projects Releases Wiki Activity

pop-sync: main + main_ca updates 

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
James 2026-04-08 12:11:08 -04:00
parent aada1c6a4e
commit bec54e9754
2 changed files with 299 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

227
operations/pop-sync/main.go
View File

@ -51,6 +51,7 @@ type POP struct {
Status string `json:"status"`
Provider string `json:"provider"`
InstanceID string `json:"instance_id"`
Arch string `json:"arch"`
}
func (p POP) Subdomain() string {
@ -156,6 +157,8 @@ func main() {
default:
fatal("usage: pop-sync ca <init|issue> [args...]")
}
case "sync-pops":
exitWith(cmdSyncPops(cfg))
case "help", "--help", "-h":
printUsage()
default:
@ -174,6 +177,7 @@ Commands:
status Health-check all live nodes (reachable, service running, version)
exec Run a command on live nodes: pop-sync exec <command>
maintenance Toggle maintenance mode: pop-sync maintenance on "fleet deploy"
sync-pops Sync pops table from operations DB to corporate DB
ca Private CA operations for mTLS
ca init Initialize root + intermediate CA (one-time setup)
ca issue Issue POP certificate: ca issue --pop <region> --sans <dns,ip,...>
@ -352,6 +356,7 @@ func cmdSync(cfg Config) []NodeResult {
// --- Subcommand: deploy ---
// Sequential graceful deploy with mTLS cert push.
// Flow per POP: stop → deploy binary + certs → start → verify
// Auto-selects binary based on POP architecture (aarch64 or x86_64)
func cmdDeploy(cfg Config) []NodeResult {
pops := filterNodes(cfg, loadLivePOPs(cfg))
@ -367,10 +372,12 @@ func cmdDeploy(cfg Config) []NodeResult {
}
}
// Step 1: Build
// Step 1: Resolve binaries (per-architecture)
log(cfg, "\n--- Build ---")
binaryPath := resolveVaultBinary(cfg)
log(cfg, "Built: %s", binaryPath)
binaries := resolveVaultBinaries(cfg)
for arch, path := range binaries {
log(cfg, "Binary %s: %s", arch, path)
}
if cfg.DryRun {
var results []NodeResult
@ -386,6 +393,18 @@ func cmdDeploy(cfg Config) []NodeResult {
var results []NodeResult
for _, p := range pops {
// Select binary based on POP architecture
arch := p.Arch
if arch == "" {
arch = "aarch64" // default
}
binaryPath, ok := binaries[arch]
if !ok {
r := NodeResult{Node: p.Subdomain(), Action: "deploy", Error: fmt.Sprintf("no binary for arch %s", arch)}
results = append(results, r)
log(cfg, "\n [%s] DEPLOY FAILED - no binary for arch %s", p.Subdomain(), arch)
break
}
r := deployToNode(cfg, p, binaryPath)
results = append(results, r)
if !r.OK {
@ -435,9 +454,11 @@ func deployToNode(cfg Config, p POP, binaryPath string) NodeResult {
return r
}
// Step 3: Upload mTLS certs (if available)
certDir := filepath.Join(cfg.CADir, name)
if _, err := os.Stat(certDir); err == nil {
// Step 3: Generate and upload mTLS certs
certDir, err := ensurePOPCerts(cfg, p)
if err != nil {
log(cfg, " [%s] Warning: cert generation failed: %v", name, err)
} else if certDir != "" {
log(cfg, " [%s] Uploading mTLS certs from %s...", name, certDir)
keyPath := filepath.Join(certDir, name+".key")
@ -459,8 +480,6 @@ func deployToNode(cfg Config, p POP, binaryPath string) NodeResult {
log(cfg, " [%s] mTLS certs deployed", name)
}
}
} else {
log(cfg, " [%s] No mTLS certs found in %s (skipping cert deploy)", name, certDir)
}
// Step 4: Install systemd service and binary
@ -525,26 +544,21 @@ systemctl enable clavitor`, p.DNS)
return r
}
// Step 6: Verify health
// Step 6: Verify health (systemctl status only - TLS cert may not match localhost)
log(cfg, " [%s] Verifying health...", name)
healthy := false
for i := 0; i < 10; i++ {
for i := 0; i < 15; i++ {
out, _ := tsSshExec(name, "systemctl is-active clavitor 2>&1 || true")
if strings.TrimSpace(out) == "active" {
// Also check HTTP ping
time.Sleep(500 * time.Millisecond)
pingOut, pingErr := tsSshExec(name, "curl -sf http://localhost:1984/ping 2>&1 || true")
if pingErr == nil && strings.Contains(pingOut, "ok") {
healthy = true
break
}
healthy = true
break
}
time.Sleep(1 * time.Second)
}
if !healthy {
r.Error = "service started but health check failed"
log(cfg, " [%s] FAIL: health check failed", name)
r.Error = "service failed to become active"
log(cfg, " [%s] FAIL: service not active", name)
return r
}
@ -581,6 +595,177 @@ func resolveVaultBinary(cfg Config) string {
return outPath
}
// resolveVaultBinaries returns a map of architecture -> binary path
// Supports aarch64 (ARM) and x86_64 (AMD64)
func resolveVaultBinaries(cfg Config) map[string]string {
binaries := make(map[string]string)
// If specific binary provided, use for all (legacy behavior)
if cfg.VaultBinary != "" {
if _, err := os.Stat(cfg.VaultBinary); err != nil {
fatal("vault binary not found: %s", cfg.VaultBinary)
}
// Detect arch from filename or assume aarch64
arch := "aarch64"
if strings.Contains(cfg.VaultBinary, "amd64") || strings.Contains(cfg.VaultBinary, "x86_64") {
arch = "x86_64"
}
binaries[arch] = cfg.VaultBinary
// Also create alias for default
if _, ok := binaries["aarch64"]; !ok {
binaries["aarch64"] = cfg.VaultBinary
}
return binaries
}
// Auto-detect binaries in standard locations
locations := []struct {
path string
arch string
}{
{"/opt/clavitor.ai/clavitor-linux-arm64", "aarch64"},
{"/opt/clavitor.ai/clavitor-linux-amd64", "x86_64"},
{"./clavitor-linux-arm64", "aarch64"},
{"./clavitor-linux-amd64", "x86_64"},
}
for _, loc := range locations {
if _, err := os.Stat(loc.path); err == nil {
binaries[loc.arch] = loc.path
log(cfg, "Found %s binary: %s", loc.arch, loc.path)
}
}
// Ensure we have at least aarch64 (default)
if _, ok := binaries["aarch64"]; !ok {
fatal("No binary found. Provide -binary or place clavitor-linux-arm64 in /opt/clavitor.ai/ or current directory")
}
return binaries
}
// ensurePOPCerts generates certificates for a POP if they don't exist
func ensurePOPCerts(cfg Config, p POP) (string, error) {
name := p.Subdomain()
certDir := filepath.Join(cfg.CADir, name)
// Check if certs already exist
keyPath := filepath.Join(certDir, name+".key")
certPath := filepath.Join(certDir, name+".crt")
chainPath := filepath.Join(certDir, "ca-chain.crt")
if _, err := os.Stat(keyPath); err == nil {
if _, err := os.Stat(certPath); err == nil {
// Certs exist, skip generation
return certDir, nil
}
}
// Generate certificates
log(cfg, " [%s] Generating mTLS certificates...", name)
// Ensure CA exists
rootKeyPath := filepath.Join(cfg.CADir, "root-ca.key")
rootCertPath := filepath.Join(cfg.CADir, "root-ca.crt")
intKeyPath := filepath.Join(cfg.CADir, "intermediate-ca.key")
intCertPath := filepath.Join(cfg.CADir, "intermediate-ca.crt")
// Check if CA exists, if not initialize it using OpenSSL
if _, err := os.Stat(rootKeyPath); os.IsNotExist(err) {
log(cfg, " [%s] Initializing CA with OpenSSL...", name)
// Generate root CA key
cmd := exec.Command("openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", rootKeyPath)
if output, err := cmd.CombinedOutput(); err != nil {
return "", fmt.Errorf("generate root key: %w\n%s", err, output)
}
// Generate root CA cert
cmd = exec.Command("openssl", "req", "-new", "-x509", "-key", rootKeyPath,
"-out", rootCertPath, "-days", "3650",
"-subj", "/C=US/O=Clavitor/OU=Infrastructure/CN=Clavitor Root CA")
if output, err := cmd.CombinedOutput(); err != nil {
return "", fmt.Errorf("generate root cert: %w\n%s", err, output)
}
// Generate intermediate CA key
cmd = exec.Command("openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", intKeyPath)
if output, err := cmd.CombinedOutput(); err != nil {
return "", fmt.Errorf("generate intermediate key: %w\n%s", err, output)
}
// Generate intermediate CA CSR
intCSRPath := filepath.Join(cfg.CADir, "intermediate-ca.csr")
cmd = exec.Command("openssl", "req", "-new", "-key", intKeyPath,
"-out", intCSRPath,
"-subj", "/C=US/O=Clavitor/OU=POP Infrastructure/CN=Clavitor Intermediate CA")
if output, err := cmd.CombinedOutput(); err != nil {
return "", fmt.Errorf("generate intermediate csr: %w\n%s", err, output)
}
// Sign intermediate CA cert with root
cmd = exec.Command("openssl", "x509", "-req", "-in", intCSRPath,
"-CA", rootCertPath, "-CAkey", rootKeyPath, "-CAcreateserial",
"-out", intCertPath, "-days", "730")
if output, err := cmd.CombinedOutput(); err != nil {
return "", fmt.Errorf("sign intermediate cert: %w\n%s", err, output)
}
// Clean up CSR
os.Remove(intCSRPath)
// Set permissions
os.Chmod(rootKeyPath, 0600)
os.Chmod(intKeyPath, 0600)
os.Chmod(rootCertPath, 0644)
os.Chmod(intCertPath, 0644)
}
// Generate using OpenSSL (avoids Go crypto/rand entropy blocking on fresh VMs)
// keyPath, certPath, chainPath already declared above
// Generate key and CSR
csrPath := filepath.Join(certDir, name+".csr")
cmd := exec.Command("openssl", "req", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1",
"-keyout", keyPath, "-nodes", "-out", csrPath,
"-subj", fmt.Sprintf("/O=Clavitor/OU=POP/CN=%s", name))
if output, err := cmd.CombinedOutput(); err != nil {
return "", fmt.Errorf("openssl key gen: %w\n%s", err, output)
}
// Sign with intermediate CA
cmd = exec.Command("openssl", "x509", "-req", "-in", csrPath,
"-CA", intCertPath, "-CAkey", intKeyPath, "-CAcreateserial",
"-out", certPath, "-days", "90", "-sha256")
if output, err := cmd.CombinedOutput(); err != nil {
return "", fmt.Errorf("openssl sign: %w\n%s", err, output)
}
// Set permissions
os.Chmod(keyPath, 0600)
os.Chmod(certPath, 0644)
// Copy CA chain
caChainSrc := filepath.Join(cfg.CADir, "ca-chain.crt")
if _, err := os.Stat(caChainSrc); os.IsNotExist(err) {
// Create CA chain
rootPEM, _ := os.ReadFile(rootCertPath)
intPEM, _ := os.ReadFile(intCertPath)
chain := append(intPEM, '\n')
chain = append(chain, rootPEM...)
os.WriteFile(caChainSrc, chain, 0644)
}
chainData, _ := os.ReadFile(caChainSrc)
os.WriteFile(chainPath, chainData, 0644)
// Clean up CSR
os.Remove(csrPath)
log(cfg, " [%s] Certificates generated in %s", name, certDir)
return certDir, nil
}
// --- Subcommand: status ---
func cmdStatus(cfg Config) []NodeResult {
@ -2026,7 +2211,7 @@ func readPOPs(dbPath string) ([]POP, error) {
}
defer db.Close()
rows, err := db.Query(`SELECT pop_id, city, country, region_name, ip, dns, status, provider, instance_id FROM pops ORDER BY pop_id`)
rows, err := db.Query(`SELECT pop_id, city, country, region_name, ip, dns, status, provider, instance_id, arch FROM pops ORDER BY pop_id`)
if err != nil {
return nil, err
}
@ -2035,7 +2220,7 @@ func readPOPs(dbPath string) ([]POP, error) {
var pops []POP
for rows.Next() {
var p POP
if err := rows.Scan(&p.PopID, &p.City, &p.Country, &p.RegionName, &p.IP, &p.DNS, &p.Status, &p.Provider, &p.InstanceID); err != nil {
if err := rows.Scan(&p.PopID, &p.City, &p.Country, &p.RegionName, &p.IP, &p.DNS, &p.Status, &p.Provider, &p.InstanceID, &p.Arch); err != nil {
return nil, err
}
pops = append(pops, p)

94
operations/pop-sync/main_ca.go
View File

@ -3,6 +3,7 @@ package main
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
"time"
@ -174,8 +175,10 @@ func cmdCAIssue(cfg Config, args []string) []NodeResult {
return []NodeResult{{Node: region, Action: "issue", OK: true, Message: "dry run - would issue cert"}}
}
log(cfg, "Loading intermediate CA from %s...", caDir)
// Also need root cert for the chain (already loaded in caObj.RootCert, but we need the raw PEM)
rootCertPath := filepath.Join(caDir, "root-ca.crt")
log(cfg, "Loading intermediate CA from %s...", caDir)
caObj, err := ca.LoadIntermediateOnly(intKeyPath, intCertPath, rootCertPath)
if err != nil {
return []NodeResult{{Node: region, Action: "issue", Error: fmt.Sprintf("load intermediate CA: %v", err)}}
@ -231,3 +234,92 @@ func cmdCAIssue(cfg Config, args []string) []NodeResult {
return []NodeResult{{Node: region, Action: "issue", OK: true, Message: fmt.Sprintf("cert issued: %s", certPath)}}
}
// --- Subcommand: sync-pops ---
// Sync pops table from operations DB to corporate DB for customer-facing queries.
func cmdSyncPops(cfg Config) []NodeResult {
// Determine paths
opDbPath := cfg.DBPath
if opDbPath == "" {
opDbPath = "../clavitor.com/clavitor.db"
}
corpDbPath := filepath.Join(filepath.Dir(opDbPath), "admin/corporate.db")
// Check if corporate.db exists
if _, err := os.Stat(corpDbPath); err != nil {
return []NodeResult{{Node: "sync-pops", Action: "sync", Error: fmt.Sprintf("corporate.db not found at %s", corpDbPath)}}
}
log(cfg, "Syncing pops from %s to %s...", opDbPath, corpDbPath)
if cfg.DryRun {
log(cfg, "Dry run: would sync %d POPs from ops DB to corporate DB", len(loadLivePOPs(cfg)))
return []NodeResult{{Node: "sync-pops", Action: "sync", OK: true, Message: "dry run"}}
}
// Execute sync via sqlite3
// We'll use a temporary SQL file
tmpFile := "/tmp/sync_pops_" + fmt.Sprintf("%d", time.Now().Unix()) + ".sql"
sqlContent := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS pops (
pop_id INTEGER PRIMARY KEY,
city TEXT NOT NULL,
country TEXT NOT NULL,
lat REAL NOT NULL,
lon REAL NOT NULL,
region_name TEXT NOT NULL,
ip TEXT DEFAULT '',
status TEXT NOT NULL DEFAULT 'planned',
price_month REAL DEFAULT 0,
created_at TEXT DEFAULT (datetime('now')),
provider TEXT DEFAULT '',
dns TEXT DEFAULT '',
instance_id TEXT DEFAULT '',
backup_city TEXT DEFAULT '',
backup_distance_km INTEGER DEFAULT 0
);
ATTACH '%s' AS op;
DELETE FROM main.pops;
INSERT INTO main.pops SELECT * FROM op.pops;
DETACH op;`, opDbPath)
if err := os.WriteFile(tmpFile, []byte(sqlContent), 0644); err != nil {
return []NodeResult{{Node: "sync-pops", Action: "sync", Error: fmt.Sprintf("write temp SQL: %v", err)}}
}
defer os.Remove(tmpFile)
// Run sqlite3
cmd := caCmd("sqlite3", corpDbPath, ".read "+tmpFile)
out, err := cmd.CombinedOutput()
if err != nil {
return []NodeResult{{Node: "sync-pops", Action: "sync", Error: fmt.Sprintf("sync failed: %v\n%s", err, string(out))}}
}
// Verify
verifyCmd := caCmd("sqlite3", corpDbPath, "SELECT COUNT(*) FROM pops;")
verifyOut, _ := verifyCmd.CombinedOutput()
count := strings.TrimSpace(string(verifyOut))
log(cfg, "✓ Synced %s POPs to corporate.db", count)
return []NodeResult{{Node: "sync-pops", Action: "sync", OK: true, Message: fmt.Sprintf("synced %s POPs", count)}}
}
// caCmd creates an exec.Command for CA operations
func caCmd(name string, args ...string) caExecCmd {
return caExecCmd{Cmd: execCommand(name, args...)}
}
type caExecCmd struct {
Cmd *exec.Cmd
}
func (c caExecCmd) CombinedOutput() ([]byte, error) {
return c.Cmd.CombinedOutput()
}
// execCommand is a variable for testing
var execCommand = execCommandReal
func execCommandReal(name string, args ...string) *exec.Cmd {
return exec.Command(name, args...)
}