186 lines
8.1 KiB
Bash
Executable File
186 lines
8.1 KiB
Bash
Executable File
#!/bin/bash
|
|
# vault1984 POP Update Script — push latest binary + optional re-harden
|
|
# Usage: ./update-pop.sh [--harden] [<node-id> <region> <instance-id>]
|
|
# --harden Also re-apply hardening (fail2ban, firewall, sysctl, etc.)
|
|
# no args Update all known POPs (binary + service only)
|
|
# Example: ./update-pop.sh --harden virginia us-east-1 i-01613f301bc47418e
|
|
# ./update-pop.sh --harden (all POPs)
|
|
set -euo pipefail
|
|
|
|
HQ_URL="http://185.218.204.47:8080"
|
|
APPLY_HARDEN=false
|
|
HARDEN_SCRIPT="$(dirname "$0")/harden-pop.sh"
|
|
|
|
# Known POPs — update this list as new POPs are added
|
|
declare -A POPS=(
|
|
["virginia"]="us-east-1:i-01613f301bc47418e"
|
|
["singapore"]="ap-southeast-1:i-03285633c3dcc64e1"
|
|
["zurich"]="eu-central-2:i-0b907cebe7978c7c3"
|
|
["saopaulo"]="sa-east-1:i-06485da3657e4a89b"
|
|
)
|
|
|
|
# Parse args
|
|
ARGS=()
|
|
for arg in "$@"; do
|
|
[[ "$arg" == "--harden" ]] && APPLY_HARDEN=true || ARGS+=("$arg")
|
|
done
|
|
set -- "${ARGS[@]+"${ARGS[@]}"}"
|
|
|
|
ssm_run() {
|
|
local region=$1 iid=$2
|
|
shift 2
|
|
local cmds=("$@")
|
|
local json_cmds
|
|
json_cmds=$(python3 -c "import json,sys; print(json.dumps(sys.argv[1:]))" "${cmds[@]}")
|
|
aws --region "$region" ssm send-command \
|
|
--instance-ids "$iid" \
|
|
--document-name AWS-RunShellScript \
|
|
--parameters "commands=$json_cmds" \
|
|
--query "Command.CommandId" --output text
|
|
}
|
|
|
|
ssm_wait() {
|
|
local region=$1 iid=$2 cmd_id=$3 label=$4
|
|
for i in $(seq 1 18); do
|
|
R=$(aws --region "$region" ssm get-command-invocation \
|
|
--command-id "$cmd_id" --instance-id "$iid" \
|
|
--query "{S:Status,O:StandardOutputContent,E:StandardErrorContent}" --output json 2>/dev/null || echo '{"S":"Pending"}')
|
|
S=$(echo "$R" | python3 -c "import json,sys; print(json.load(sys.stdin).get('S','?'))")
|
|
if [[ "$S" == "Success" || "$S" == "Failed" || "$S" == "TimedOut" ]]; then
|
|
echo "$R" | python3 -c "
|
|
import json,sys
|
|
d=json.load(sys.stdin)
|
|
out=d.get('O','').strip()
|
|
err=d.get('E','').strip()
|
|
if out: print(out)
|
|
if err: print('STDERR:', err)
|
|
"
|
|
[[ "$S" == "Success" ]] && return 0 || { echo "FAILED ($S)"; return 1; }
|
|
fi
|
|
echo " ($i) $S..."
|
|
sleep 10
|
|
done
|
|
echo "TIMEOUT"; return 1
|
|
}
|
|
|
|
verify_sg() {
|
|
local region=$1 iid=$2 node=$3
|
|
ASSIGNED=$(aws --region "$region" ec2 describe-instances \
|
|
--instance-ids "$iid" \
|
|
--query "Reservations[0].Instances[0].SecurityGroups[0].GroupName" --output text 2>/dev/null || echo "unknown")
|
|
if [[ "$ASSIGNED" != "vault1984-pop" ]]; then
|
|
echo " ⚠️ SG mismatch on $node: got '$ASSIGNED', expected 'vault1984-pop'"
|
|
echo " Attempting to reconcile..."
|
|
# Get the correct vault1984-pop SG ID for this region
|
|
VPC_ID=$(aws --region "$region" ec2 describe-vpcs \
|
|
--filters "Name=isDefault,Values=true" --query "Vpcs[0].VpcId" --output text)
|
|
SG_ID=$(aws --region "$region" ec2 describe-security-groups \
|
|
--filters "Name=group-name,Values=vault1984-pop" "Name=vpc-id,Values=$VPC_ID" \
|
|
--query "SecurityGroups[0].GroupId" --output text 2>/dev/null || echo "None")
|
|
if [[ -z "$SG_ID" || "$SG_ID" == "None" ]]; then
|
|
echo " vault1984-pop SG doesn't exist in $region — creating..."
|
|
SG_ID=$(aws --region "$region" ec2 create-security-group \
|
|
--group-name vault1984-pop \
|
|
--description "Vault1984 POP - outbound only, no inbound" \
|
|
--vpc-id "$VPC_ID" --query "GroupId" --output text)
|
|
aws --region "$region" ec2 revoke-security-group-ingress \
|
|
--group-id "$SG_ID" --protocol -1 --cidr 0.0.0.0/0 2>/dev/null || true
|
|
aws --region "$region" ec2 revoke-security-group-ingress \
|
|
--group-id "$SG_ID" --protocol -1 --source-group "$SG_ID" 2>/dev/null || true
|
|
echo " Created SG: $SG_ID"
|
|
fi
|
|
aws --region "$region" ec2 modify-instance-attribute \
|
|
--instance-id "$iid" --groups "$SG_ID"
|
|
echo " ✓ SG reconciled → vault1984-pop ($SG_ID)"
|
|
else
|
|
echo " ✓ SG: vault1984-pop"
|
|
fi
|
|
}
|
|
|
|
update_pop() {
|
|
local NODE_ID="$1" REGION="$2" INSTANCE_ID="$3"
|
|
echo ""
|
|
echo "━━━ $NODE_ID ($REGION / $INSTANCE_ID) ━━━"
|
|
|
|
# 1. SG verification + reconciliation
|
|
echo " Verifying security group..."
|
|
verify_sg "$REGION" "$INSTANCE_ID" "$NODE_ID"
|
|
|
|
# 2. Binary + service update
|
|
echo " Updating vault1984 binary..."
|
|
CMD_ID=$(ssm_run "$REGION" "$INSTANCE_ID" \
|
|
"set -e" \
|
|
"curl -sfo /usr/local/bin/vault1984.new $HQ_URL/download/vault1984-arm64" \
|
|
"chmod +x /usr/local/bin/vault1984.new" \
|
|
"mv /usr/local/bin/vault1984.new /usr/local/bin/vault1984" \
|
|
"printf '[Unit]\nDescription=Vault1984\nAfter=network.target\n\n[Service]\nEnvironment=NODE_ID=$NODE_ID\nExecStart=/usr/local/bin/vault1984 --telemetry-freq=60 --telemetry-host=$HQ_URL/telemetry\nRestart=always\nRestartSec=10\nWorkingDirectory=/var/lib/vault1984\n\n[Install]\nWantedBy=multi-user.target\n' > /etc/systemd/system/vault1984.service" \
|
|
"mkdir -p /var/lib/vault1984" \
|
|
"systemctl stop v1984-agent 2>/dev/null || true" \
|
|
"systemctl disable v1984-agent 2>/dev/null || true" \
|
|
"systemctl daemon-reload && systemctl enable vault1984 && systemctl restart vault1984" \
|
|
"sleep 2" \
|
|
"systemctl is-active vault1984 && echo 'vault1984: OK' || echo 'vault1984: FAILED'")
|
|
ssm_wait "$REGION" "$INSTANCE_ID" "$CMD_ID" "binary update"
|
|
|
|
# 3. Hardening re-apply (optional, --harden flag)
|
|
if $APPLY_HARDEN; then
|
|
echo " Re-applying hardening..."
|
|
HARDEN_CONTENT=$(cat "$HARDEN_SCRIPT")
|
|
CMD_ID=$(aws --region "$REGION" ssm send-command \
|
|
--instance-ids "$INSTANCE_ID" \
|
|
--document-name AWS-RunShellScript \
|
|
--parameters "commands=[\"bash -s << 'HARDEN'\\n$(echo "$HARDEN_CONTENT" | python3 -c "import sys; print(sys.stdin.read().replace('\\\\', '\\\\\\\\').replace('\"', '\\\\\"'))")\\nHARDEN\"]" \
|
|
--query "Command.CommandId" --output text 2>/dev/null || true)
|
|
# Simpler approach: write script then run it
|
|
CMD_ID=$(aws --region "$REGION" ssm send-command \
|
|
--instance-ids "$INSTANCE_ID" \
|
|
--document-name AWS-RunShellScript \
|
|
--parameters commands="[
|
|
\"amazon-linux-extras install epel -y -q > /dev/null 2>&1 || true\",
|
|
\"yum install -y -q fail2ban > /dev/null 2>&1 || true\",
|
|
\"printf '[DEFAULT]\\nbantime = 86400\\nfindtime = 600\\nmaxretry = 3\\nignoreip = 127.0.0.1/8 ::1\\n\\n[sshd]\\nenabled = true\\nport = ssh\\nfilter = sshd\\nlogpath = /var/log/secure\\nmaxretry = 3\\nbantime = 86400\\n' > /etc/fail2ban/jail.local\",
|
|
\"systemctl enable fail2ban && systemctl restart fail2ban\",
|
|
\"sleep 5 && fail2ban-client status sshd\",
|
|
\"for svc in postfix rpcbind sshd; do systemctl stop \$svc 2>/dev/null; systemctl disable \$svc 2>/dev/null; done; echo 'services disabled'\",
|
|
\"echo 'Hardening applied'\"
|
|
]" \
|
|
--query "Command.CommandId" --output text)
|
|
ssm_wait "$REGION" "$INSTANCE_ID" "$CMD_ID" "hardening"
|
|
fi
|
|
|
|
# 4. Verify health
|
|
echo " Verifying node health..."
|
|
CMD_ID=$(ssm_run "$REGION" "$INSTANCE_ID" \
|
|
"echo '--- vault1984 ---'" \
|
|
"systemctl is-active vault1984 && echo 'OK' || echo 'FAILED'" \
|
|
"echo '--- fail2ban ---'" \
|
|
"systemctl is-active fail2ban && echo 'OK' || echo 'FAILED'" \
|
|
"echo '--- fail2ban jails ---'" \
|
|
"fail2ban-client status 2>/dev/null || echo 'no jails'" \
|
|
"echo '--- sshd ---'" \
|
|
"systemctl is-active sshd 2>/dev/null && echo 'WARNING: sshd still active' || echo 'disabled OK'" \
|
|
"echo '--- rpcbind ---'" \
|
|
"systemctl is-active rpcbind 2>/dev/null && echo 'WARNING: still active' || echo 'disabled OK'" \
|
|
"echo '--- open ports ---'" \
|
|
"ss -tlnp | grep LISTEN")
|
|
ssm_wait "$REGION" "$INSTANCE_ID" "$CMD_ID" "verify"
|
|
|
|
echo " ✓ $NODE_ID done"
|
|
}
|
|
|
|
# Main
|
|
if [ ${#ARGS[@]} -eq 3 ]; then
|
|
update_pop "${ARGS[0]}" "${ARGS[1]}" "${ARGS[2]}"
|
|
elif [ ${#ARGS[@]} -eq 0 ]; then
|
|
echo "=== vault1984 POP Update — all nodes$(${APPLY_HARDEN} && echo ' (+harden)' || echo '') ==="
|
|
for NODE_ID in "${!POPS[@]}"; do
|
|
IFS=: read -r REGION INSTANCE_ID <<< "${POPS[$NODE_ID]}"
|
|
update_pop "$NODE_ID" "$REGION" "$INSTANCE_ID"
|
|
done
|
|
echo ""
|
|
echo "=== All POPs updated ==="
|
|
else
|
|
echo "Usage: $0 [--harden] [<node-id> <region> <instance-id>]"
|
|
exit 1
|
|
fi
|