#!/bin/bash # vault1984 POP Update Script — push latest binary + optional re-harden # Usage: ./update-pop.sh [--harden] [ ] # --harden Also re-apply hardening (fail2ban, firewall, sysctl, etc.) # no args Update all known POPs (binary + service only) # Example: ./update-pop.sh --harden virginia us-east-1 i-01613f301bc47418e # ./update-pop.sh --harden (all POPs) set -euo pipefail HQ_URL="http://185.218.204.47:8080" APPLY_HARDEN=false HARDEN_SCRIPT="$(dirname "$0")/harden-pop.sh" # Known POPs — update this list as new POPs are added declare -A POPS=( ["virginia"]="us-east-1:i-01613f301bc47418e" ["singapore"]="ap-southeast-1:i-03285633c3dcc64e1" ["zurich"]="eu-central-2:i-0b907cebe7978c7c3" ["saopaulo"]="sa-east-1:i-06485da3657e4a89b" ) # Parse args ARGS=() for arg in "$@"; do [[ "$arg" == "--harden" ]] && APPLY_HARDEN=true || ARGS+=("$arg") done set -- "${ARGS[@]+"${ARGS[@]}"}" ssm_run() { local region=$1 iid=$2 shift 2 local cmds=("$@") local json_cmds json_cmds=$(python3 -c "import json,sys; print(json.dumps(sys.argv[1:]))" "${cmds[@]}") aws --region "$region" ssm send-command \ --instance-ids "$iid" \ --document-name AWS-RunShellScript \ --parameters "commands=$json_cmds" \ --query "Command.CommandId" --output text } ssm_wait() { local region=$1 iid=$2 cmd_id=$3 label=$4 for i in $(seq 1 18); do R=$(aws --region "$region" ssm get-command-invocation \ --command-id "$cmd_id" --instance-id "$iid" \ --query "{S:Status,O:StandardOutputContent,E:StandardErrorContent}" --output json 2>/dev/null || echo '{"S":"Pending"}') S=$(echo "$R" | python3 -c "import json,sys; print(json.load(sys.stdin).get('S','?'))") if [[ "$S" == "Success" || "$S" == "Failed" || "$S" == "TimedOut" ]]; then echo "$R" | python3 -c " import json,sys d=json.load(sys.stdin) out=d.get('O','').strip() err=d.get('E','').strip() if out: print(out) if err: print('STDERR:', err) " [[ "$S" == "Success" ]] && return 0 || { echo "FAILED ($S)"; return 1; } fi echo " ($i) $S..." sleep 10 done echo "TIMEOUT"; return 1 } verify_sg() { local region=$1 iid=$2 node=$3 ASSIGNED=$(aws --region "$region" ec2 describe-instances \ --instance-ids "$iid" \ --query "Reservations[0].Instances[0].SecurityGroups[0].GroupName" --output text 2>/dev/null || echo "unknown") if [[ "$ASSIGNED" != "vault1984-pop" ]]; then echo " ⚠️ SG mismatch on $node: got '$ASSIGNED', expected 'vault1984-pop'" echo " Attempting to reconcile..." # Get the correct vault1984-pop SG ID for this region VPC_ID=$(aws --region "$region" ec2 describe-vpcs \ --filters "Name=isDefault,Values=true" --query "Vpcs[0].VpcId" --output text) SG_ID=$(aws --region "$region" ec2 describe-security-groups \ --filters "Name=group-name,Values=vault1984-pop" "Name=vpc-id,Values=$VPC_ID" \ --query "SecurityGroups[0].GroupId" --output text 2>/dev/null || echo "None") if [[ -z "$SG_ID" || "$SG_ID" == "None" ]]; then echo " vault1984-pop SG doesn't exist in $region — creating..." SG_ID=$(aws --region "$region" ec2 create-security-group \ --group-name vault1984-pop \ --description "Vault1984 POP - outbound only, no inbound" \ --vpc-id "$VPC_ID" --query "GroupId" --output text) aws --region "$region" ec2 revoke-security-group-ingress \ --group-id "$SG_ID" --protocol -1 --cidr 0.0.0.0/0 2>/dev/null || true aws --region "$region" ec2 revoke-security-group-ingress \ --group-id "$SG_ID" --protocol -1 --source-group "$SG_ID" 2>/dev/null || true echo " Created SG: $SG_ID" fi aws --region "$region" ec2 modify-instance-attribute \ --instance-id "$iid" --groups "$SG_ID" echo " ✓ SG reconciled → vault1984-pop ($SG_ID)" else echo " ✓ SG: vault1984-pop" fi } update_pop() { local NODE_ID="$1" REGION="$2" INSTANCE_ID="$3" echo "" echo "━━━ $NODE_ID ($REGION / $INSTANCE_ID) ━━━" # 1. SG verification + reconciliation echo " Verifying security group..." verify_sg "$REGION" "$INSTANCE_ID" "$NODE_ID" # 2. Binary + service update echo " Updating vault1984 binary..." CMD_ID=$(ssm_run "$REGION" "$INSTANCE_ID" \ "set -e" \ "curl -sfo /usr/local/bin/vault1984.new $HQ_URL/download/vault1984-arm64" \ "chmod +x /usr/local/bin/vault1984.new" \ "mv /usr/local/bin/vault1984.new /usr/local/bin/vault1984" \ "printf '[Unit]\nDescription=Vault1984\nAfter=network.target\n\n[Service]\nEnvironment=NODE_ID=$NODE_ID\nExecStart=/usr/local/bin/vault1984 --telemetry-freq=60 --telemetry-host=$HQ_URL/telemetry\nRestart=always\nRestartSec=10\nWorkingDirectory=/var/lib/vault1984\n\n[Install]\nWantedBy=multi-user.target\n' > /etc/systemd/system/vault1984.service" \ "mkdir -p /var/lib/vault1984" \ "systemctl stop v1984-agent 2>/dev/null || true" \ "systemctl disable v1984-agent 2>/dev/null || true" \ "systemctl daemon-reload && systemctl enable vault1984 && systemctl restart vault1984" \ "sleep 2" \ "systemctl is-active vault1984 && echo 'vault1984: OK' || echo 'vault1984: FAILED'") ssm_wait "$REGION" "$INSTANCE_ID" "$CMD_ID" "binary update" # 3. Hardening re-apply (optional, --harden flag) if $APPLY_HARDEN; then echo " Re-applying hardening..." HARDEN_CONTENT=$(cat "$HARDEN_SCRIPT") CMD_ID=$(aws --region "$REGION" ssm send-command \ --instance-ids "$INSTANCE_ID" \ --document-name AWS-RunShellScript \ --parameters "commands=[\"bash -s << 'HARDEN'\\n$(echo "$HARDEN_CONTENT" | python3 -c "import sys; print(sys.stdin.read().replace('\\\\', '\\\\\\\\').replace('\"', '\\\\\"'))")\\nHARDEN\"]" \ --query "Command.CommandId" --output text 2>/dev/null || true) # Simpler approach: write script then run it CMD_ID=$(aws --region "$REGION" ssm send-command \ --instance-ids "$INSTANCE_ID" \ --document-name AWS-RunShellScript \ --parameters commands="[ \"amazon-linux-extras install epel -y -q > /dev/null 2>&1 || true\", \"yum install -y -q fail2ban > /dev/null 2>&1 || true\", \"printf '[DEFAULT]\\nbantime = 86400\\nfindtime = 600\\nmaxretry = 3\\nignoreip = 127.0.0.1/8 ::1\\n\\n[sshd]\\nenabled = true\\nport = ssh\\nfilter = sshd\\nlogpath = /var/log/secure\\nmaxretry = 3\\nbantime = 86400\\n' > /etc/fail2ban/jail.local\", \"systemctl enable fail2ban && systemctl restart fail2ban\", \"sleep 5 && fail2ban-client status sshd\", \"for svc in postfix rpcbind sshd; do systemctl stop \$svc 2>/dev/null; systemctl disable \$svc 2>/dev/null; done; echo 'services disabled'\", \"echo 'Hardening applied'\" ]" \ --query "Command.CommandId" --output text) ssm_wait "$REGION" "$INSTANCE_ID" "$CMD_ID" "hardening" fi # 4. Verify health echo " Verifying node health..." CMD_ID=$(ssm_run "$REGION" "$INSTANCE_ID" \ "echo '--- vault1984 ---'" \ "systemctl is-active vault1984 && echo 'OK' || echo 'FAILED'" \ "echo '--- fail2ban ---'" \ "systemctl is-active fail2ban && echo 'OK' || echo 'FAILED'" \ "echo '--- fail2ban jails ---'" \ "fail2ban-client status 2>/dev/null || echo 'no jails'" \ "echo '--- sshd ---'" \ "systemctl is-active sshd 2>/dev/null && echo 'WARNING: sshd still active' || echo 'disabled OK'" \ "echo '--- rpcbind ---'" \ "systemctl is-active rpcbind 2>/dev/null && echo 'WARNING: still active' || echo 'disabled OK'" \ "echo '--- open ports ---'" \ "ss -tlnp | grep LISTEN") ssm_wait "$REGION" "$INSTANCE_ID" "$CMD_ID" "verify" echo " ✓ $NODE_ID done" } # Main if [ ${#ARGS[@]} -eq 3 ]; then update_pop "${ARGS[0]}" "${ARGS[1]}" "${ARGS[2]}" elif [ ${#ARGS[@]} -eq 0 ]; then echo "=== vault1984 POP Update — all nodes$(${APPLY_HARDEN} && echo ' (+harden)' || echo '') ===" for NODE_ID in "${!POPS[@]}"; do IFS=: read -r REGION INSTANCE_ID <<< "${POPS[$NODE_ID]}" update_pop "$NODE_ID" "$REGION" "$INSTANCE_ID" done echo "" echo "=== All POPs updated ===" else echo "Usage: $0 [--harden] [ ]" exit 1 fi