#!/usr/bin/env bash
# Production Health - Daily check, part run ON ATP2APP1 (ssh spadmin@atp2app1 -p 22 from ATP2JMP1).
# Covers runbook steps 3 (load/memory/disk/Redis), 4 (services, Redis clients, mail queue),
# 5 (journal errors, mail and SMTP AUTH failures, FTP failures by source, last upload per FTP account, fail2ban),
# 8 (apt, reboot-required, certbot).
# Output is plain text, one header per check, so it can be pasted into a chat as-is.
# At the end it prints the usp_Record statements for this host with the measured values.
# Needs sudo for journalctl, certbot and the mail/vsftpd logs; run as:  sudo ./check-app1.sh
# If the file was copied from Windows and bash complains about '\r', run:  sed -i 's/\r$//' check-app1.sh

# Redis runs on this host, so default to its primary address; override with REDIS_HOST=<ip> when it lives elsewhere.
REDIS_HOST="${REDIS_HOST:-$(hostname -I 2>/dev/null | awk '{print $1}')}"
REDIS_HOST="${REDIS_HOST:-127.0.0.1}"
H() { printf '\n### %s\n' "$1"; }
REC=()

echo "Production Health - APP1 - $(date '+%Y-%m-%d %H:%M') - run by $(whoami) on $(hostname) - $(. /etc/os-release; echo "$PRETTY_NAME")"

H "3. Load averages vs cores (expect 1-min load under core count) and last boot"
echo "cores: $(nproc)   $(uptime)"
echo "last boot: $(uptime -s)"

H "3. Memory and swap in MB (expect swap used 0 or unchanged)"
free -m
MEM_PCT=$(free | awk '/Mem:/ {printf "%d", ($2-$7)*100/$2}')
SWAP_MB=$(free -m | awk '/Swap:/ {print $3}')
echo "memory used (excluding available): ${MEM_PCT} %   swap used: ${SWAP_MB} MB"
REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','mem_used_pct',${MEM_PCT};")
REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','swap_used_mb',${SWAP_MB};")

H "3. Disk free (expect > 15 % on / and /var)"
df -h / /var /home 2>/dev/null
ROOT_FREE=$(df --output=pcent / | tail -1 | tr -dc '0-9')
ROOT_FREE=$((100 - ROOT_FREE))
REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','disk_free_pct',${ROOT_FREE};")

H "3. Top 6 processes by resident memory (expect redis-server and nothing surprising)"
ps -eo pid,rss,pcpu,comm --sort=-rss | head -7 | awk 'NR==1{print; next}{printf "%6s %8d MB %5s%% %s\n", $1, $2/1024, $3, $4}'

H "3. Redis memory (expect used < 80 % of maxmemory, or < 2 GB if maxmemory is 0; fragmentation < 1.5)"
if command -v redis-cli >/dev/null; then
  redis-cli -h "$REDIS_HOST" INFO memory 2>&1 | grep -E "^(used_memory_human|used_memory:|maxmemory_human|maxmemory:|mem_fragmentation_ratio)" | tr -d '\r'
  REDIS_MB=$(redis-cli -h "$REDIS_HOST" INFO memory 2>/dev/null | awk -F: '/^used_memory:/ {printf "%d", $2/1048576}' | tr -d '\r')
  [ -n "$REDIS_MB" ] && REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','redis_used_mb',${REDIS_MB};")
else
  echo "redis-cli not installed; install redis-tools or check from WFE1"
fi

H "4. Services (expect all three active; a recent ActiveEnterTimestamp means a restart since yesterday)"
for s in postfix vsftpd redis-server; do
  printf '%-14s %-9s since %s\n' "$s" "$(systemctl is-active "$s")" "$(systemctl show "$s" -p ActiveEnterTimestamp --value)"
done

H "4. Redis clients and health counters (expect ~8-20 clients, 0 rejected, 0 evicted)"
if command -v redis-cli >/dev/null; then
  redis-cli -h "$REDIS_HOST" PING 2>&1 | tr -d '\r'
  redis-cli -h "$REDIS_HOST" INFO clients 2>&1 | grep -E "^connected_clients" | tr -d '\r'
  redis-cli -h "$REDIS_HOST" INFO stats 2>&1 | grep -E "^(rejected_connections|expired_keys|evicted_keys|total_commands_processed)" | tr -d '\r'
  CLIENTS=$(redis-cli -h "$REDIS_HOST" INFO clients 2>/dev/null | awk -F: '/^connected_clients/ {print $2}' | tr -d '\r ')
  [ -n "$CLIENTS" ] && REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','redis_clients',${CLIENTS};")
fi

H "4. Mail queue (expect empty or under 10 requests)"
MQ=$(mailq 2>&1 | tail -1)
echo "$MQ"
QN=$(echo "$MQ" | grep -oE 'in [0-9]+ Request' | grep -oE '[0-9]+')
REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','mail_queue',${QN:-0};")
if [ -n "$QN" ] && [ "$QN" -gt 0 ]; then echo "--- first deferred reasons:"; mailq 2>/dev/null | grep -E '^\s*\(' | sort | uniq -c | sort -rn | head -5; fi

H "5. Error-level journal entries, last 24 h (expect none you cannot explain)"
journalctl -p err -S -24h --no-pager 2>&1 | tail -30

H "5. Mail, last 24 h: sent / bounced / deferred, top bouncing domains, SMTP AUTH failures by source (expect bounces in single digits, AUTH failures from no single address in the tens)"
MAILLOG="${MAILLOG:-/var/log/mail.log}"
if [ -r "$MAILLOG" ]; then
  ML=$(cat "$MAILLOG"); echo "(source: $MAILLOG, current file)"
else
  # Debian's postfix logs its delivery agents under postfix@-.service, not postfix.service: match every postfix unit
  ML=$(journalctl -u 'postfix*' -S -24h --no-pager -o cat 2>/dev/null); echo "(source: journal, all postfix units)"
fi
BOUNCED=$(grep -c 'status=bounced' <<<"$ML")
echo "sent=$(grep -c 'status=sent' <<<"$ML")  bounced=${BOUNCED}  deferred=$(grep -c 'status=deferred' <<<"$ML")"
if [ "$BOUNCED" -gt 0 ]; then echo "--- top bouncing domains:"; grep 'status=bounced' <<<"$ML" | sed -E 's/.*to=<[^@]*@([^>]*)>.*/\1/' | sort | uniq -c | sort -rn | head -5; fi
SASL=$(grep -E 'SASL [A-Z0-9-]+ authentication failed' <<<"$ML")
SASLN=$(grep -c . <<<"$SASL")
echo "SMTP AUTH failures: ${SASLN}"
if [ "$SASLN" -gt 0 ]; then echo "--- by source address:"; sed -E 's/.*\[([0-9a-fA-F.:]+)\]: SASL.*/\1/' <<<"$SASL" | sort | uniq -c | sort -rn | head -5; fi
REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','mail_bounced',${BOUNCED};")
REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','smtp_auth_failed',${SASLN};")

H "5. FTP, last 24 h: failed logins by source and uploads (expect failures in single digits; one address in the tens is an attack)"
FTPLOG="${FTPLOG:-/var/log/vsftpd.log}"
FTP_IGNORE_USERS="${FTP_IGNORE_USERS:-atp2deploy}"   # accounts whose uploads are ours (deploy packages), not customer feeds
# vsftpd lines start with their time stamp:  Wed Oct  7 11:30:31 2026 [pid 123] [user] FAIL LOGIN: Client "::ffff:1.2.3.4"
ftp_epoch() { date -d "$(awk '{print $2, $3, $4, $5}' <<<"$1")" +%s 2>/dev/null; }
if [ -r "$FTPLOG" ]; then
  CUTOFF=$(date -d '-24 hours' +%s)
  ftp_recent() { cat "$FTPLOG" "$FTPLOG.1" 2>/dev/null | grep -E "$1" | while IFS= read -r l; do
      t=$(ftp_epoch "$l") || continue; [ -n "$t" ] && [ "$t" -ge "$CUTOFF" ] && printf '%s\n' "$l"; done; }
  FAILS=$(ftp_recent 'FAIL LOGIN')
  FAILN=$(grep -c . <<<"$FAILS")
  echo "failed logins: ${FAILN}"
  if [ "$FAILN" -gt 0 ]; then
    echo "--- by source address (count, address, accounts tried):"
    sed -E 's/.*\[([^]]*)\] FAIL LOGIN: Client "(::ffff:)?([^"]*)".*/\3 \1/' <<<"$FAILS" |
      awk '{n[$1]++; if (index(" " u[$1] " ", " " $2 " ") == 0) u[$1] = u[$1] " " $2} END {for (a in n) printf "%6d  %-40s%s\n", n[a], a, u[a]}' | sort -rn | head -5
  fi
  REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','ftp_failed_logins',${FAILN};")
  UPS=$(ftp_recent 'OK UPLOAD')
  if [ -n "$UPS" ]; then echo "--- uploads:"; tail -5 <<<"$UPS"; else echo "no uploads"; fi
else
  echo "no $FTPLOG; check vsftpd log location in /etc/vsftpd.conf (xferlog_file / vsftpd_log_file)"
fi

FTP_STALE_HOURS="${FTP_STALE_HOURS:-336}"   # customer feeds are irregular: flag only after 14 days without an upload
H "5. FTP: last upload per account, from every kept vsftpd log (customer feeds such as SFC's CSV are irregular: STALE only after $((FTP_STALE_HOURS / 24)) days without an upload)"
if [ -r "$FTPLOG" ]; then
  OLDEST=""
  for f in "$FTPLOG"*; do t=$(ftp_epoch "$(zcat -f "$f" 2>/dev/null | head -1)"); [ -n "$t" ] && { [ -z "$OLDEST" ] || [ "$t" -lt "$OLDEST" ]; } && OLDEST=$t; done
  [ -n "$OLDEST" ] && echo "(logs reach back to $(date -d "@$OLDEST" '+%Y-%m-%d'); an account missing below has not uploaded since)"
  NOW=$(date +%s)
  if [ -n "$OLDEST" ] && [ $(( (NOW - OLDEST) / 3600 )) -lt "$FTP_STALE_HOURS" ]; then
    echo "WARNING: the logs cover only $(( (NOW - OLDEST) / 86400 )) days, less than the $((FTP_STALE_HOURS / 24))-day threshold: a feed that stopped earlier is missing from the list, not flagged. Keep more vsftpd log rotations (/etc/logrotate.d/vsftpd)."
  fi
  zcat -f "$FTPLOG"* 2>/dev/null | grep 'OK UPLOAD' | while IFS= read -r l; do
      t=$(ftp_epoch "$l") || continue
      u=$(sed -E 's/.*\[([^]]*)\] OK UPLOAD.*/\1/' <<<"$l"); p=$(sed -E 's/.*OK UPLOAD: Client "[^"]*", "([^"]*)".*/\1/' <<<"$l")
      printf '%s|%s|%s\n' "$t" "$u" "$p"; done |
    sort -t'|' -k1,1n | awk -F'|' '{last[$2] = $0} END {for (u in last) print last[u]}' | sort -t'|' -k2,2 |
    while IFS='|' read -r t u p; do
      age=$(( (NOW - t) / 3600 ))
      tag=""; case " $FTP_IGNORE_USERS " in *" $u "*) tag=" (ours, not recorded)";; esac
      stale=""; if [ -z "$tag" ] && [ "$age" -gt "$FTP_STALE_HOURS" ]; then stale="   STALE (over $((FTP_STALE_HOURS / 24)) days)"; fi
      printf '%-20s %s  %5d h ago  %s%s%s\n' "$u" "$(date -d "@$t" '+%Y-%m-%d %H:%M')" "$age" "$(basename "$p")" "$tag" "$stale"
      [ -z "$tag" ] && echo "EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','ftp_upload_age_h:${u}',${age};" >> "${TMPDIR:-/tmp}/app1-rec.$$"
    done
  # the loop above runs in a subshell (pipeline), so its usp_Record lines come back through a temp file
  if [ -f "${TMPDIR:-/tmp}/app1-rec.$$" ]; then while IFS= read -r r; do REC+=("$r"); done < "${TMPDIR:-/tmp}/app1-rec.$$"; rm -f "${TMPDIR:-/tmp}/app1-rec.$$"; fi
fi

H "5. fail2ban: banned addresses per jail (expect installed with sshd, vsftpd and postfix-sasl jails; bead ATTAIN7-gjn3.7)"
if command -v fail2ban-client >/dev/null; then
  for j in $(fail2ban-client status 2>/dev/null | sed -n 's/.*Jail list:[[:space:]]*//p' | tr ',' ' '); do
    printf '%-16s %s\n' "$j" "$(fail2ban-client status "$j" 2>/dev/null | grep -E 'Currently banned|Total banned' | sed -E 's/.*- //' | tr -s ' \t' ' ' | tr '\n' ' ')"
  done
else
  echo "fail2ban not installed"
fi

H "8. Pending apt upgrades and reboot-required (expect < 10 upgrades, no security updates older than a week, no reboot-required)"
apt list --upgradable 2>/dev/null | tail -n +2 | head -15
echo "pending upgrades: $(apt list --upgradable 2>/dev/null | tail -n +2 | wc -l)"
if [ -f /var/run/reboot-required ]; then echo "REBOOT REQUIRED:"; cat /var/run/reboot-required.pkgs 2>/dev/null; else echo "no reboot required"; fi

H "8. Recent apt activity (changes since last check)"
grep -E '^(Start-Date|Upgrade|Install|Remove)' /var/log/apt/history.log 2>/dev/null | tail -6 | cut -c1-200

H "8. Let's Encrypt: certificate expiry and renewal activity (expect > 30 days left; a renewal in the last day is a change to record)"
if command -v certbot >/dev/null; then
  certbot certificates 2>/dev/null | grep -E 'Certificate Name|Expiry Date|Domains'
  systemctl list-timers 'certbot*' --no-pager 2>/dev/null | head -3
  journalctl -u certbot.service -S yesterday --no-pager 2>/dev/null | grep -iE 'renew|cert' | tail -3
else
  echo "certbot not found"
fi

# ---- WEEKLY sections: Mondays, or force with  curl ... | sudo ATTAIN_WEEKLY=1 bash  (0 suppresses them) ----
case "${ATTAIN_WEEKLY:-}" in 1) WEEKLY=1 ;; 0) WEEKLY=0 ;; *) if [ "$(date +%u)" = 1 ]; then WEEKLY=1; else WEEKLY=0; fi ;; esac
if [ "$WEEKLY" = 1 ]; then
  H "WEEKLY. Patching: pending apt upgrades and how many from security, how long a reboot has been pending, unattended-upgrades (expect no security upgrade waiting, no reboot pending over a week)"
  UPG=$(apt list --upgradable 2>/dev/null | tail -n +2)
  APT_N=$(grep -c . <<<"$UPG"); APT_SEC=$(grep -c -- '-security' <<<"$UPG")
  echo "pending upgrades: ${APT_N}, from security: ${APT_SEC}"
  if [ "$APT_SEC" -gt 0 ]; then grep -- '-security' <<<"$UPG" | head -10; fi
  if [ -f /var/run/reboot-required ]; then
    RB_T=$(stat -c %Y /var/run/reboot-required); RB_D=$(( ($(date +%s) - RB_T) / 86400 ))
    echo "reboot required for ${RB_D} days (since $(date -d "@$RB_T" '+%Y-%m-%d'))"
  else RB_D=0; echo "no reboot required"; fi
  echo "last boot: $(uptime -s)   unattended-upgrades: $(systemctl is-active unattended-upgrades 2>/dev/null) / $(systemctl is-enabled unattended-upgrades 2>/dev/null)"
  REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','apt_security_pending',${APT_SEC};")
  REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','reboot_required_days',${RB_D};")

  H "WEEKLY. SSH and FTP failed logins over 7 days, top source addresses (expect background noise, no single address in the hundreds)"
  SSHF=$(journalctl -u ssh -u sshd -S -7d --no-pager -o cat 2>/dev/null | grep -E 'Failed password|Invalid user')
  SSHN=$(grep -c . <<<"$SSHF")
  echo "ssh failures: ${SSHN}"
  if [ "$SSHN" -gt 0 ]; then grep -oE 'from [0-9a-fA-F.:]+' <<<"$SSHF" | awk '{print $2}' | sort | uniq -c | sort -rn | head -5; fi
  FTPF7=""
  if [ -r "$FTPLOG" ]; then
    CUT7=$(date -d '-7 days' +%s)
    FTPF7=$(zcat -f "$FTPLOG"* 2>/dev/null | grep 'FAIL LOGIN' | while IFS= read -r l; do t=$(ftp_epoch "$l") || continue; [ -n "$t" ] && [ "$t" -ge "$CUT7" ] && printf '%s\n' "$l"; done)
  fi
  FTPN7=$(grep -c . <<<"$FTPF7")
  echo "ftp failures: ${FTPN7}"
  if [ "$FTPN7" -gt 0 ]; then sed -E 's/.*FAIL LOGIN: Client "(::ffff:)?([^"]*)".*/\2/' <<<"$FTPF7" | sort | uniq -c | sort -rn | head -5; fi
  REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','ssh_failed_7d',${SSHN};")
  REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','ftp_failed_7d',${FTPN7};")

  H "WEEKLY. Mail over 7 days: sent, bounced, deferred, bounce rate (expect bounces under 5 % of sent)"
  if [ -r "$MAILLOG" ]; then ML7=$(zcat -f "$MAILLOG"* 2>/dev/null); echo "(source: $MAILLOG and rotated copies, everything kept)"
  else ML7=$(journalctl -u 'postfix*' -S -7d --no-pager -o cat 2>/dev/null); echo "(source: journal, all postfix units, 7 days)"; fi
  S7=$(grep -c 'status=sent' <<<"$ML7"); B7=$(grep -c 'status=bounced' <<<"$ML7"); D7=$(grep -c 'status=deferred' <<<"$ML7")
  BPCT=$(awk -v b="$B7" -v s="$S7" 'BEGIN { if (s + b > 0) printf "%.1f", b * 100 / (s + b); else print 0 }')
  echo "sent=${S7}  bounced=${B7}  deferred=${D7}  bounce rate ${BPCT} %"
  if [ "$B7" -gt 0 ]; then grep 'status=bounced' <<<"$ML7" | sed -E 's/.*to=<[^@]*@([^>]*)>.*/\1/' | sort | uniq -c | sort -rn | head -5; fi
  REC+=("EXEC OPS_HealthCheck.dbo.usp_Record 'APP1','mail_bounce_pct_7d',${BPCT},'sent ${S7} bounced ${B7}';")
fi

H "RECORD lines for OPS_HealthCheck (copy these; check-dat1.ps1 asks you to paste them at its PASTE prompt)"
printf '%s\n' "${REC[@]}"