#!/bin/sh
# FailNote agent installer: https://itadminutils.com/systemd-alert-on-failure
#
#   Install:    curl -fsSL https://app.itadminutils.com/install.sh | sudo sh -s -- YOUR_KEY
#   Uninstall:  sudo failnote uninstall
#
# What it adds (all plain text you can read):
#   /usr/local/bin/failnote                           the agent, a shell script
#   /etc/failnote/failnote.conf                       settings
#   /etc/failnote/curl.conf                           your key (root only)
#   /etc/systemd/system/failnote-report@.service      runs when a service fails
#   /etc/systemd/system/failnote-heartbeat.{service,timer}   checks in every 60 seconds
#   /etc/systemd/system/service.d/10-failnote.conf    adds OnFailure= to every service (systemd 244+)
#
# When a service fails, FailNote sends its name, its `systemctl status` and its
# last 40 journal lines (set FAILNOTE_LOG_LINES=0 in failnote.conf to send none).
# Check-ins send the hostname, OS name, systemd version and failed unit names.

set -eu

# Everything runs from main, called on the last line, so a download cut short
# can't run half a script.
main() {
  API="https://app.itadminutils.com"
  API_SET=""
  KEY="${FAILNOTE_KEY:-}"
  while [ $# -gt 0 ]; do
    case "$1" in
      --api) API="$2"; API_SET=1; shift 2 ;;
      --api=*) API="${1#--api=}"; API_SET=1; shift ;;
      -*) fail "unknown option $1" ;;
      *) KEY="$1"; shift ;;
    esac
  done

  [ "$(id -u)" -eq 0 ] || fail "run this as root (with sudo)."
  printf '%s' "$KEY" | grep -Eq '^fn_[0-9a-f]{48}$' || fail "missing or invalid FailNote key. Copy the install command from $API"
  if [ ! -d /run/systemd/system ] || ! command -v systemctl >/dev/null 2>&1; then
    fail "this server doesn't run systemd."
  fi
  command -v curl >/dev/null 2>&1 || fail "curl is required. Install it, then run this again."

  SYSTEMD_VERSION=$(systemctl --version | awk 'NR == 1 { print $2 }' | sed 's/[^0-9].*//')
  [ -n "$SYSTEMD_VERSION" ] || SYSTEMD_VERSION=0

  mkdir -p /etc/failnote /var/lib/failnote /etc/systemd/system
  chmod 755 /etc/failnote /var/lib/failnote

  if [ ! -f /etc/failnote/failnote.conf ] || [ -n "$API_SET" ]; then
    cat > /etc/failnote/failnote.conf <<EOF
# FailNote settings. Run "sudo failnote test" after changing them.
FAILNOTE_API="$API"
# Journal lines sent with each failure. 0 sends none.
FAILNOTE_LOG_LINES=40
EOF
    chmod 644 /etc/failnote/failnote.conf
  fi

  umask 077
  printf 'header = "Authorization: Bearer %s"\n' "$KEY" > /etc/failnote/curl.conf
  chmod 600 /etc/failnote/curl.conf
  umask 022

  write_agent
  chmod 755 /usr/local/bin/failnote

  cat > /etc/systemd/system/failnote-report@.service <<'EOF'
[Unit]
Description=FailNote alert for %i.service
Documentation=https://itadminutils.com/systemd-alert-on-failure

[Service]
Type=oneshot
# "-" keeps this unit out of the failed list when FailNote can't be reached.
ExecStart=-/usr/local/bin/failnote report %i.service
TimeoutStartSec=150
EOF

  cat > /etc/systemd/system/failnote-heartbeat.service <<'EOF'
[Unit]
Description=FailNote check-in
Documentation=https://itadminutils.com/systemd-alert-on-failure

[Service]
Type=oneshot
ExecStart=-/usr/local/bin/failnote heartbeat
TimeoutStartSec=120
EOF

  cat > /etc/systemd/system/failnote-heartbeat.timer <<'EOF'
[Unit]
Description=FailNote check-in every minute

[Timer]
OnBootSec=30s
OnUnitActiveSec=60s
AccuracySec=5s

[Install]
WantedBy=timers.target
EOF

  # FailNote's own units must not report themselves.
  for unit in failnote-report@.service failnote-heartbeat.service; do
    mkdir -p "/etc/systemd/system/$unit.d"
    ln -sf /dev/null "/etc/systemd/system/$unit.d/10-failnote.conf"
  done

  HOOK="on"
  if [ "$SYSTEMD_VERSION" -ge 244 ]; then
    mkdir -p /etc/systemd/system/service.d
    cat > /etc/systemd/system/service.d/10-failnote.conf <<'EOF'
# Added by FailNote: report any service that fails. Remove with "sudo failnote uninstall".
[Unit]
OnFailure=failnote-report@%N.service
EOF
  else
    HOOK="off"
  fi

  systemctl daemon-reload
  systemctl enable --now failnote-heartbeat.timer >/dev/null 2>&1

  HOST=$(uname -n)
  if out=$(/usr/local/bin/failnote heartbeat 2>&1); then
    echo
    echo "FailNote is installed on $HOST and checking in."
  else
    echo
    echo "FailNote is installed on $HOST, but it couldn't check in yet:"
    printf '  %s\n' "$out"
  fi
  if [ "$HOOK" = "on" ]; then
    echo "  Any service that fails sends an alert within seconds."
  else
    echo "  systemd $SYSTEMD_VERSION is older than 244, so FailNote checks for failed units every minute instead."
  fi
  echo "  Send a test alert:  sudo failnote test"
  echo "  Check the agent:    failnote status"
  echo "  Remove FailNote:    sudo failnote uninstall"
}

fail() {
  echo "failnote: $*" >&2
  exit 1
}

write_agent() {
  cat > /usr/local/bin/failnote <<'AGENT'
#!/bin/sh
# FailNote agent: reports failed systemd units and checks in every minute.
# Settings: /etc/failnote/failnote.conf
# About: https://itadminutils.com/systemd-alert-on-failure

set -u

VERSION="1.0.0"
CONF=/etc/failnote/failnote.conf
CURL_CONF=/etc/failnote/curl.conf
STATE=/var/lib/failnote
FAILNOTE_API="https://app.itadminutils.com"
FAILNOTE_LOG_LINES=40
[ -r "$CONF" ] && . "$CONF"

log() { echo "failnote: $*" >&2; }

machine_id() {
  id=$(cat /etc/machine-id 2>/dev/null || uname -n)
  printf 'failnote:%s' "$id" | sha256sum | cut -c1-64
}

os_name() {
  ( . /etc/os-release 2>/dev/null && printf '%s' "${PRETTY_NAME:-${NAME:-Linux}}" ) || uname -sr
}

systemd_version() {
  systemctl --version 2>/dev/null | awk 'NR == 1 { print $2 }'
}

# post PATH [curl form args]: prints the reply. Returns 0 on success,
# 1 if FailNote couldn't be reached, 2 if it answered with an error.
post() {
  path=$1
  shift
  out=$(mktemp) || return 1
  code=$(curl -sS -m 30 -K "$CURL_CONF" -o "$out" -w '%{http_code}' \
    --form-string "server=$(machine_id)" \
    --form-string "hostname=$(uname -n)" \
    --form-string "os=$(os_name)" \
    --form-string "systemd=$(systemd_version)" \
    --form-string "agent=$VERSION" \
    "$@" "$FAILNOTE_API$path")
  status=$?
  body=$(cat "$out" 2>/dev/null)
  rm -f "$out"
  if [ "$status" -ne 0 ]; then
    log "couldn't reach $FAILNOTE_API (curl exit $status)"
    return 1
  fi
  case "$code" in
    2??) [ -n "$body" ] && printf '%s\n' "$body"; return 0 ;;
  esac
  log "${body:-error} (HTTP $code)"
  return 2
}

# report UNIT [hook|poll|retry]
report() {
  unit=$1
  source=${2:-hook}
  mkdir -p "$STATE/reported" "$STATE/pending"
  : > "$STATE/reported/$unit"
  status_file=$(mktemp) || return 1
  journal_file=$(mktemp) || { rm -f "$status_file"; return 1; }
  systemctl status --full --no-pager -n 0 "$unit" > "$status_file" 2>&1
  case "$FAILNOTE_LOG_LINES" in
    ''|*[!0-9]*|0) ;;
    *) journalctl -u "$unit" -n "$FAILNOTE_LOG_LINES" --no-pager -o short-iso > "$journal_file" 2>&1 ;;
  esac

  tries="0 5 20"
  [ "$source" = "hook" ] || tries="0"
  result=1
  for delay in $tries; do
    sleep "$delay"
    post /api/agent/report --form-string "unit=$unit" --form-string "source=$source" \
      -F "status=<$status_file" -F "journal=<$journal_file" > /dev/null
    result=$?
    [ "$result" -eq 1 ] || break
  done
  rm -f "$status_file" "$journal_file"
  # Couldn't reach FailNote (often the network is down too). The next check-in retries.
  [ "$result" -eq 1 ] && : > "$STATE/pending/$unit"
  return "$result"
}

# Unit names from systemctl list-units output, minus FailNote's own units.
unit_names() {
  awk '{ for (i = 1; i <= NF; i++) if ($i ~ /\.(service|socket|mount|timer|path|swap|target|automount|slice|scope|device)$/) { print $i; break } }' |
    grep -v '^failnote-'
}

heartbeat() {
  mkdir -p "$STATE/reported" "$STATE/pending"
  failed=$(systemctl list-units --state=failed --all --plain --no-legend --no-pager --full 2>/dev/null | unit_names)
  restarting=$(systemctl list-units --state=auto-restart --all --plain --no-legend --no-pager --full 2>/dev/null | unit_names)
  post /api/agent/heartbeat --form-string "failed=$failed" --form-string "restarting=$restarting" > /dev/null
  result=$?

  # Reports that couldn't be sent earlier.
  for file in "$STATE"/pending/*; do
    [ -e "$file" ] || continue
    unit=$(basename "$file")
    rm -f "$file"
    report "$unit" retry > /dev/null || true
  done

  # Failed units the OnFailure hook didn't report (older systemd, RestartMode=direct,
  # or a unit with its own hooks). Up to 5 per minute.
  handled=""
  count=0
  for unit in $failed; do
    if grep -qxF "$unit" "$STATE/known-failed" 2>/dev/null || [ -n "$(find "$STATE/reported/$unit" -mmin -5 2>/dev/null)" ]; then
      handled="$handled $unit"
      continue
    fi
    [ "$count" -lt 5 ] || continue
    count=$((count + 1))
    report "$unit" poll > /dev/null || true
    handled="$handled $unit"
  done
  printf '%s\n' "$handled" | tr ' ' '\n' | sed '/^$/d' > "$STATE/known-failed.new" && mv "$STATE/known-failed.new" "$STATE/known-failed"
  return "$result"
}

show_status() {
  echo "FailNote agent $VERSION"
  echo "Sends to: $FAILNOTE_API"
  echo "systemd: $(systemd_version)"
  if [ -e /etc/systemd/system/service.d/10-failnote.conf ]; then
    echo "Failure hook: on"
  else
    echo "Failure hook: off (failed units are found by the check-in every minute)"
  fi
  if systemctl is-active --quiet failnote-heartbeat.timer; then
    echo "Check-in: every minute"
  else
    echo "Check-in: stopped (start it with: sudo systemctl enable --now failnote-heartbeat.timer)"
  fi
}

uninstall() {
  [ "$(id -u)" -eq 0 ] || { log "run this as root (with sudo)."; exit 1; }
  post /api/agent/goodbye > /dev/null || log "couldn't tell FailNote. Remove this server in your dashboard."
  systemctl disable --now failnote-heartbeat.timer > /dev/null 2>&1 || true
  rm -f /etc/systemd/system/service.d/10-failnote.conf
  rmdir /etc/systemd/system/service.d 2> /dev/null || true
  rm -rf /etc/systemd/system/failnote-report@.service.d /etc/systemd/system/failnote-heartbeat.service.d
  rm -f /etc/systemd/system/failnote-report@.service /etc/systemd/system/failnote-heartbeat.service /etc/systemd/system/failnote-heartbeat.timer
  systemctl daemon-reload
  rm -rf /etc/failnote /var/lib/failnote
  rm -f /usr/local/bin/failnote
  echo "FailNote is removed from this server."
}

case "${1:-}" in
  report)
    [ -n "${2:-}" ] || { log "usage: failnote report UNIT"; exit 2; }
    report "$2" hook
    ;;
  heartbeat) heartbeat ;;
  test) post /api/agent/test ;;
  status) show_status ;;
  uninstall) uninstall ;;
  version | --version) echo "$VERSION" ;;
  *)
    echo "Usage: failnote status | test | uninstall"
    exit 2
    ;;
esac
AGENT
}

main "$@"