diff --git a/operations/pop-sync/main.go b/operations/pop-sync/main.go index 1a4a31c..d6234b8 100644 --- a/operations/pop-sync/main.go +++ b/operations/pop-sync/main.go @@ -51,6 +51,7 @@ type POP struct { Status string `json:"status"` Provider string `json:"provider"` InstanceID string `json:"instance_id"` + Arch string `json:"arch"` } func (p POP) Subdomain() string { @@ -156,6 +157,8 @@ func main() { default: fatal("usage: pop-sync ca [args...]") } + case "sync-pops": + exitWith(cmdSyncPops(cfg)) case "help", "--help", "-h": printUsage() default: @@ -174,6 +177,7 @@ Commands: status Health-check all live nodes (reachable, service running, version) exec Run a command on live nodes: pop-sync exec maintenance Toggle maintenance mode: pop-sync maintenance on "fleet deploy" + sync-pops Sync pops table from operations DB to corporate DB ca Private CA operations for mTLS ca init Initialize root + intermediate CA (one-time setup) ca issue Issue POP certificate: ca issue --pop --sans @@ -352,6 +356,7 @@ func cmdSync(cfg Config) []NodeResult { // --- Subcommand: deploy --- // Sequential graceful deploy with mTLS cert push. // Flow per POP: stop → deploy binary + certs → start → verify +// Auto-selects binary based on POP architecture (aarch64 or x86_64) func cmdDeploy(cfg Config) []NodeResult { pops := filterNodes(cfg, loadLivePOPs(cfg)) @@ -367,10 +372,12 @@ func cmdDeploy(cfg Config) []NodeResult { } } - // Step 1: Build + // Step 1: Resolve binaries (per-architecture) log(cfg, "\n--- Build ---") - binaryPath := resolveVaultBinary(cfg) - log(cfg, "Built: %s", binaryPath) + binaries := resolveVaultBinaries(cfg) + for arch, path := range binaries { + log(cfg, "Binary %s: %s", arch, path) + } if cfg.DryRun { var results []NodeResult @@ -386,6 +393,18 @@ func cmdDeploy(cfg Config) []NodeResult { var results []NodeResult for _, p := range pops { + // Select binary based on POP architecture + arch := p.Arch + if arch == "" { + arch = "aarch64" // default + } + binaryPath, ok := binaries[arch] + if !ok { + r := NodeResult{Node: p.Subdomain(), Action: "deploy", Error: fmt.Sprintf("no binary for arch %s", arch)} + results = append(results, r) + log(cfg, "\n [%s] DEPLOY FAILED - no binary for arch %s", p.Subdomain(), arch) + break + } r := deployToNode(cfg, p, binaryPath) results = append(results, r) if !r.OK { @@ -435,9 +454,11 @@ func deployToNode(cfg Config, p POP, binaryPath string) NodeResult { return r } - // Step 3: Upload mTLS certs (if available) - certDir := filepath.Join(cfg.CADir, name) - if _, err := os.Stat(certDir); err == nil { + // Step 3: Generate and upload mTLS certs + certDir, err := ensurePOPCerts(cfg, p) + if err != nil { + log(cfg, " [%s] Warning: cert generation failed: %v", name, err) + } else if certDir != "" { log(cfg, " [%s] Uploading mTLS certs from %s...", name, certDir) keyPath := filepath.Join(certDir, name+".key") @@ -459,8 +480,6 @@ func deployToNode(cfg Config, p POP, binaryPath string) NodeResult { log(cfg, " [%s] mTLS certs deployed", name) } } - } else { - log(cfg, " [%s] No mTLS certs found in %s (skipping cert deploy)", name, certDir) } // Step 4: Install systemd service and binary @@ -525,26 +544,21 @@ systemctl enable clavitor`, p.DNS) return r } - // Step 6: Verify health + // Step 6: Verify health (systemctl status only - TLS cert may not match localhost) log(cfg, " [%s] Verifying health...", name) healthy := false - for i := 0; i < 10; i++ { + for i := 0; i < 15; i++ { out, _ := tsSshExec(name, "systemctl is-active clavitor 2>&1 || true") if strings.TrimSpace(out) == "active" { - // Also check HTTP ping - time.Sleep(500 * time.Millisecond) - pingOut, pingErr := tsSshExec(name, "curl -sf http://localhost:1984/ping 2>&1 || true") - if pingErr == nil && strings.Contains(pingOut, "ok") { - healthy = true - break - } + healthy = true + break } time.Sleep(1 * time.Second) } if !healthy { - r.Error = "service started but health check failed" - log(cfg, " [%s] FAIL: health check failed", name) + r.Error = "service failed to become active" + log(cfg, " [%s] FAIL: service not active", name) return r } @@ -581,6 +595,177 @@ func resolveVaultBinary(cfg Config) string { return outPath } +// resolveVaultBinaries returns a map of architecture -> binary path +// Supports aarch64 (ARM) and x86_64 (AMD64) +func resolveVaultBinaries(cfg Config) map[string]string { + binaries := make(map[string]string) + + // If specific binary provided, use for all (legacy behavior) + if cfg.VaultBinary != "" { + if _, err := os.Stat(cfg.VaultBinary); err != nil { + fatal("vault binary not found: %s", cfg.VaultBinary) + } + // Detect arch from filename or assume aarch64 + arch := "aarch64" + if strings.Contains(cfg.VaultBinary, "amd64") || strings.Contains(cfg.VaultBinary, "x86_64") { + arch = "x86_64" + } + binaries[arch] = cfg.VaultBinary + // Also create alias for default + if _, ok := binaries["aarch64"]; !ok { + binaries["aarch64"] = cfg.VaultBinary + } + return binaries + } + + // Auto-detect binaries in standard locations + locations := []struct { + path string + arch string + }{ + {"/opt/clavitor.ai/clavitor-linux-arm64", "aarch64"}, + {"/opt/clavitor.ai/clavitor-linux-amd64", "x86_64"}, + {"./clavitor-linux-arm64", "aarch64"}, + {"./clavitor-linux-amd64", "x86_64"}, + } + + for _, loc := range locations { + if _, err := os.Stat(loc.path); err == nil { + binaries[loc.arch] = loc.path + log(cfg, "Found %s binary: %s", loc.arch, loc.path) + } + } + + // Ensure we have at least aarch64 (default) + if _, ok := binaries["aarch64"]; !ok { + fatal("No binary found. Provide -binary or place clavitor-linux-arm64 in /opt/clavitor.ai/ or current directory") + } + + return binaries +} + +// ensurePOPCerts generates certificates for a POP if they don't exist +func ensurePOPCerts(cfg Config, p POP) (string, error) { + name := p.Subdomain() + certDir := filepath.Join(cfg.CADir, name) + + // Check if certs already exist + keyPath := filepath.Join(certDir, name+".key") + certPath := filepath.Join(certDir, name+".crt") + chainPath := filepath.Join(certDir, "ca-chain.crt") + + if _, err := os.Stat(keyPath); err == nil { + if _, err := os.Stat(certPath); err == nil { + // Certs exist, skip generation + return certDir, nil + } + } + + // Generate certificates + log(cfg, " [%s] Generating mTLS certificates...", name) + + // Ensure CA exists + rootKeyPath := filepath.Join(cfg.CADir, "root-ca.key") + rootCertPath := filepath.Join(cfg.CADir, "root-ca.crt") + intKeyPath := filepath.Join(cfg.CADir, "intermediate-ca.key") + intCertPath := filepath.Join(cfg.CADir, "intermediate-ca.crt") + + // Check if CA exists, if not initialize it using OpenSSL + if _, err := os.Stat(rootKeyPath); os.IsNotExist(err) { + log(cfg, " [%s] Initializing CA with OpenSSL...", name) + + // Generate root CA key + cmd := exec.Command("openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", rootKeyPath) + if output, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("generate root key: %w\n%s", err, output) + } + + // Generate root CA cert + cmd = exec.Command("openssl", "req", "-new", "-x509", "-key", rootKeyPath, + "-out", rootCertPath, "-days", "3650", + "-subj", "/C=US/O=Clavitor/OU=Infrastructure/CN=Clavitor Root CA") + if output, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("generate root cert: %w\n%s", err, output) + } + + // Generate intermediate CA key + cmd = exec.Command("openssl", "ecparam", "-name", "prime256v1", "-genkey", "-noout", "-out", intKeyPath) + if output, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("generate intermediate key: %w\n%s", err, output) + } + + // Generate intermediate CA CSR + intCSRPath := filepath.Join(cfg.CADir, "intermediate-ca.csr") + cmd = exec.Command("openssl", "req", "-new", "-key", intKeyPath, + "-out", intCSRPath, + "-subj", "/C=US/O=Clavitor/OU=POP Infrastructure/CN=Clavitor Intermediate CA") + if output, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("generate intermediate csr: %w\n%s", err, output) + } + + // Sign intermediate CA cert with root + cmd = exec.Command("openssl", "x509", "-req", "-in", intCSRPath, + "-CA", rootCertPath, "-CAkey", rootKeyPath, "-CAcreateserial", + "-out", intCertPath, "-days", "730") + if output, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("sign intermediate cert: %w\n%s", err, output) + } + + // Clean up CSR + os.Remove(intCSRPath) + + // Set permissions + os.Chmod(rootKeyPath, 0600) + os.Chmod(intKeyPath, 0600) + os.Chmod(rootCertPath, 0644) + os.Chmod(intCertPath, 0644) + } + + // Generate using OpenSSL (avoids Go crypto/rand entropy blocking on fresh VMs) + // keyPath, certPath, chainPath already declared above + + // Generate key and CSR + csrPath := filepath.Join(certDir, name+".csr") + cmd := exec.Command("openssl", "req", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", + "-keyout", keyPath, "-nodes", "-out", csrPath, + "-subj", fmt.Sprintf("/O=Clavitor/OU=POP/CN=%s", name)) + if output, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("openssl key gen: %w\n%s", err, output) + } + + // Sign with intermediate CA + cmd = exec.Command("openssl", "x509", "-req", "-in", csrPath, + "-CA", intCertPath, "-CAkey", intKeyPath, "-CAcreateserial", + "-out", certPath, "-days", "90", "-sha256") + if output, err := cmd.CombinedOutput(); err != nil { + return "", fmt.Errorf("openssl sign: %w\n%s", err, output) + } + + // Set permissions + os.Chmod(keyPath, 0600) + os.Chmod(certPath, 0644) + + // Copy CA chain + caChainSrc := filepath.Join(cfg.CADir, "ca-chain.crt") + if _, err := os.Stat(caChainSrc); os.IsNotExist(err) { + // Create CA chain + rootPEM, _ := os.ReadFile(rootCertPath) + intPEM, _ := os.ReadFile(intCertPath) + chain := append(intPEM, '\n') + chain = append(chain, rootPEM...) + os.WriteFile(caChainSrc, chain, 0644) + } + + chainData, _ := os.ReadFile(caChainSrc) + os.WriteFile(chainPath, chainData, 0644) + + // Clean up CSR + os.Remove(csrPath) + + log(cfg, " [%s] Certificates generated in %s", name, certDir) + return certDir, nil +} + // --- Subcommand: status --- func cmdStatus(cfg Config) []NodeResult { @@ -2026,7 +2211,7 @@ func readPOPs(dbPath string) ([]POP, error) { } defer db.Close() - rows, err := db.Query(`SELECT pop_id, city, country, region_name, ip, dns, status, provider, instance_id FROM pops ORDER BY pop_id`) + rows, err := db.Query(`SELECT pop_id, city, country, region_name, ip, dns, status, provider, instance_id, arch FROM pops ORDER BY pop_id`) if err != nil { return nil, err } @@ -2035,7 +2220,7 @@ func readPOPs(dbPath string) ([]POP, error) { var pops []POP for rows.Next() { var p POP - if err := rows.Scan(&p.PopID, &p.City, &p.Country, &p.RegionName, &p.IP, &p.DNS, &p.Status, &p.Provider, &p.InstanceID); err != nil { + if err := rows.Scan(&p.PopID, &p.City, &p.Country, &p.RegionName, &p.IP, &p.DNS, &p.Status, &p.Provider, &p.InstanceID, &p.Arch); err != nil { return nil, err } pops = append(pops, p) diff --git a/operations/pop-sync/main_ca.go b/operations/pop-sync/main_ca.go index ee80ca7..91c7632 100644 --- a/operations/pop-sync/main_ca.go +++ b/operations/pop-sync/main_ca.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "os/exec" "path/filepath" "strings" "time" @@ -174,8 +175,10 @@ func cmdCAIssue(cfg Config, args []string) []NodeResult { return []NodeResult{{Node: region, Action: "issue", OK: true, Message: "dry run - would issue cert"}} } - log(cfg, "Loading intermediate CA from %s...", caDir) + // Also need root cert for the chain (already loaded in caObj.RootCert, but we need the raw PEM) rootCertPath := filepath.Join(caDir, "root-ca.crt") + + log(cfg, "Loading intermediate CA from %s...", caDir) caObj, err := ca.LoadIntermediateOnly(intKeyPath, intCertPath, rootCertPath) if err != nil { return []NodeResult{{Node: region, Action: "issue", Error: fmt.Sprintf("load intermediate CA: %v", err)}} @@ -231,3 +234,92 @@ func cmdCAIssue(cfg Config, args []string) []NodeResult { return []NodeResult{{Node: region, Action: "issue", OK: true, Message: fmt.Sprintf("cert issued: %s", certPath)}} } + +// --- Subcommand: sync-pops --- +// Sync pops table from operations DB to corporate DB for customer-facing queries. + +func cmdSyncPops(cfg Config) []NodeResult { + // Determine paths + opDbPath := cfg.DBPath + if opDbPath == "" { + opDbPath = "../clavitor.com/clavitor.db" + } + corpDbPath := filepath.Join(filepath.Dir(opDbPath), "admin/corporate.db") + + // Check if corporate.db exists + if _, err := os.Stat(corpDbPath); err != nil { + return []NodeResult{{Node: "sync-pops", Action: "sync", Error: fmt.Sprintf("corporate.db not found at %s", corpDbPath)}} + } + + log(cfg, "Syncing pops from %s to %s...", opDbPath, corpDbPath) + + if cfg.DryRun { + log(cfg, "Dry run: would sync %d POPs from ops DB to corporate DB", len(loadLivePOPs(cfg))) + return []NodeResult{{Node: "sync-pops", Action: "sync", OK: true, Message: "dry run"}} + } + + // Execute sync via sqlite3 + // We'll use a temporary SQL file + tmpFile := "/tmp/sync_pops_" + fmt.Sprintf("%d", time.Now().Unix()) + ".sql" + sqlContent := fmt.Sprintf(`CREATE TABLE IF NOT EXISTS pops ( + pop_id INTEGER PRIMARY KEY, + city TEXT NOT NULL, + country TEXT NOT NULL, + lat REAL NOT NULL, + lon REAL NOT NULL, + region_name TEXT NOT NULL, + ip TEXT DEFAULT '', + status TEXT NOT NULL DEFAULT 'planned', + price_month REAL DEFAULT 0, + created_at TEXT DEFAULT (datetime('now')), + provider TEXT DEFAULT '', + dns TEXT DEFAULT '', + instance_id TEXT DEFAULT '', + backup_city TEXT DEFAULT '', + backup_distance_km INTEGER DEFAULT 0 + ); + ATTACH '%s' AS op; + DELETE FROM main.pops; + INSERT INTO main.pops SELECT * FROM op.pops; + DETACH op;`, opDbPath) + + if err := os.WriteFile(tmpFile, []byte(sqlContent), 0644); err != nil { + return []NodeResult{{Node: "sync-pops", Action: "sync", Error: fmt.Sprintf("write temp SQL: %v", err)}} + } + defer os.Remove(tmpFile) + + // Run sqlite3 + cmd := caCmd("sqlite3", corpDbPath, ".read "+tmpFile) + out, err := cmd.CombinedOutput() + if err != nil { + return []NodeResult{{Node: "sync-pops", Action: "sync", Error: fmt.Sprintf("sync failed: %v\n%s", err, string(out))}} + } + + // Verify + verifyCmd := caCmd("sqlite3", corpDbPath, "SELECT COUNT(*) FROM pops;") + verifyOut, _ := verifyCmd.CombinedOutput() + count := strings.TrimSpace(string(verifyOut)) + + log(cfg, "✓ Synced %s POPs to corporate.db", count) + return []NodeResult{{Node: "sync-pops", Action: "sync", OK: true, Message: fmt.Sprintf("synced %s POPs", count)}} +} + +// caCmd creates an exec.Command for CA operations +func caCmd(name string, args ...string) caExecCmd { + return caExecCmd{Cmd: execCommand(name, args...)} +} + +type caExecCmd struct { + Cmd *exec.Cmd +} + +func (c caExecCmd) CombinedOutput() ([]byte, error) { + return c.Cmd.CombinedOutput() +} + +// execCommand is a variable for testing +var execCommand = execCommandReal + +func execCommandReal(name string, args ...string) *exec.Cmd { + return exec.Command(name, args...) +}