#!/bin/bash
# tests/test_selfheal.sh — coverage for oxpulse-partner-edge-selfheal.sh.
#
# Every case below is one whose failure would be SILENT in production:
#
#   containers
#     1. unhealthy + budget available  → restarts        (without this the script
#        is decoration and the 26h cheburator outage repeats)
#     2. attempts at MAX               → does NOT restart, raises given_up, alerts
#        ONCE (this is the failingstreak=19471 case from FOLLOWUPS.md — an
#        unbounded healer would restart such a box forever and call it healthy
#        maintenance; this bound is the reason the compose template refused a
#        restart loop in the first place)
#     3. inside start grace            → does NOT restart (otherwise a slow-
#        starting container is killed on every tick and never finishes starting)
#     4. no healthcheck                → never touched   (`naive` ships without
#        one on purpose; acting without evidence is the thing being avoided)
#     5. healthy                       → does nothing
#     6. pending attempt now healthy   → counts `healed`, clears pending
#     7. MIN_GAP not elapsed           → does NOT restart
#     8. a FOREIGN container carrying our compose project label is never touched
#        (measured: rvpn's `all-rvpn-gate` belongs to a neighbouring project and
#        has com.docker.compose.project=oxpulse-partner-edge)
#     9. stopped                       → started
#    10. stopped `compose run` one-off → never started
#    11. stopped seconds ago           → left alone (docker's own restart policy
#        and a debugging operator both need that window)
#
#   the operator escape hatch
#    12. global hold file              → nothing is touched at all
#    13. per-subject hold file         → that subject only
#
#   systemd
#    14. a failed oxpulse unit         → restarted, and recorded healed
#    15. the healer's OWN service      → never restarted by itself
#    16. a unit in the declared enable-set that is disabled → enabled
#    17. a unit NOT in the set         → never enabled (ru-subnets is disabled on
#        4 of 4 live nodes BY DESIGN; a healer keyed on "every oxpulse timer"
#        would switch on a deliberately-off feature fleet-wide)
#    18. the healer's OWN timer is absent from its enable-set (disabling that
#        timer is how an operator stops this script; a healer that re-enables
#        itself cannot be switched off)
#    19. per-tick action cap honoured
#
#   disk
#    20. below threshold               → never prunes
#    21. at threshold                  → prunes, verifies, records healed
#
#   metrics
#    22. EVERY container gets a series in the gauge file, not just the last one
#        (shipped-and-measured regression: a gauge textfile is truncated per
#        write, so five per-container writes left one surviving sample)
#
# `docker`, `systemctl`, `df` and `timeout` are PATH shims driven by fixture
# files, so no daemon, no init system and no real disk are needed.
set -uo pipefail
REPO_ROOT="${REPO_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
SCRIPT="$REPO_ROOT/oxpulse-partner-edge-selfheal.sh"
[[ -f "$SCRIPT" ]] || { echo "FAIL: $SCRIPT not found"; exit 1; }

PASS=0; FAIL=0
pass() { echo "PASS: $1"; PASS=$((PASS+1)); }
fail() { echo "FAIL: $1"; FAIL=$((FAIL+1)); }

# Build a sandbox. FIXTURES holds one file per container:
#   "<health> <started> <has_health> <state> <finished> <oneoff>"
setup() {
	TMP="$(mktemp -d)"; BIN="$TMP/bin"
	mkdir -p "$BIN" "$TMP/state" "$TMP/fixtures" "$TMP/units" "$TMP/textfile"

	cat > "$BIN/docker" <<'DOCKER'
#!/bin/bash
FX="$FIXTURES"
read_fx() {   # container -> sets health started hashc state finished oneoff
	local f="$FX/$1"
	[[ -r "$f" ]] || return 1
	read -r health started hashc state finished oneoff < "$f"
	state="${state:-running}"; finished="${finished:-0}"; oneoff="${oneoff:-False}"
	return 0
}
case "$1" in
  ps)
    shift
    all=0; want_image=0
    for a in "$@"; do
      [[ "$a" == "-a" ]] && all=1
      [[ "$a" == *".Image"* ]] && want_image=1
    done
    for f in "$FX"/*; do
      [[ -e "$f" ]] || continue
      n="$(basename "$f")"; read_fx "$n" || continue
      if [[ $all == 1 || "$state" == running ]]; then
        if [[ $want_image == 1 ]]; then cat "$CONTAINER_IMAGE/$n" 2>/dev/null; else echo "$n"; fi
      fi
    done ;;
  inspect)
    fmt="$3"; c="$4"
    case "$fmt" in
      # Image lookups: {{.Id}} resolves a repo:tag OR a short id to its full id.
      *.Id*)
        line=$(awk -v k="$c" '$1==k || $2==k {print $1; exit}' "$IMAGE_LIST" 2>/dev/null)
        [[ -n "$line" ]] && echo "sha256:$line" || exit 1 ;;
      *Config.Image*)
        cat "$CONTAINER_IMAGE/$c" 2>/dev/null || exit 1 ;;
    esac
    read_fx "$c" || exit 1
    case "$fmt" in
      *State.Health.Status*)  echo "$health" ;;
      *if\ .State.Health*)    [[ "$hashc" == y ]] && echo y ;;
      *State.StartedAt*)      date -d "@$started" -Is ;;
      *State.FinishedAt*)     date -d "@$finished" -Is ;;
      *compose.oneoff*)       echo "$oneoff" ;;
    esac ;;
  images)
    # --format '{{.ID}} {{.Repository}}:{{.Tag}}'
    awk '{print $1" "$2}' "$IMAGE_LIST" 2>/dev/null ;;
  rmi)
    echo "$2" >> "$RMI_LOG"
    grep -v " $2\$" "$IMAGE_LIST" > "$IMAGE_LIST.t" 2>/dev/null && mv -f "$IMAGE_LIST.t" "$IMAGE_LIST"
    [[ -n "${FAKE_DISK_AFTER_RMI:-}" ]] && echo "$FAKE_DISK_AFTER_RMI" > "$TMPDIR_DISK"
    ;;
  restart) echo "$2" >> "$RESTART_LOG"; exit "${DOCKER_RESTART_RC:-0}" ;;
  start)
    echo "$2" >> "$START_LOG"
    [[ "${DOCKER_START_RC:-0}" == 0 ]] || exit "${DOCKER_START_RC}"
    # A successful start makes it running, exactly as the daemon would.
    read_fx "$2" && echo "$health $started $hashc running $finished $oneoff" > "$FX/$2"
    ;;
  image|builder)
    echo "$1 $2" >> "$PRUNE_LOG"
    # A prune reclaims space; the fixture disk drops to the post-prune value.
    [[ -n "${FAKE_DISK_AFTER:-}" ]] && echo "$FAKE_DISK_AFTER" > "$TMPDIR_DISK"
    ;;
esac
exit 0
DOCKER

	cat > "$BIN/systemctl" <<'SYSTEMCTL'
#!/bin/bash
U="$FX_UNITS"
failed_file="$U/failed"
case "$1" in
  list-units)
    [[ -r "$failed_file" ]] || exit 0
    while read -r u; do [[ -n "$u" ]] && echo "$u loaded failed failed Some unit"; done < "$failed_file" ;;
  is-failed)
    for a in "$@"; do :; done
    unit="${!#}"
    grep -qxF "$unit" "$failed_file" 2>/dev/null && exit 0 || exit 1 ;;
  is-enabled)
    unit="${!#}"
    if [[ -r "$U/enabled.$unit" ]]; then cat "$U/enabled.$unit"; else echo enabled; fi ;;
  restart)
    unit="${!#}"; echo "restart $unit" >> "$UNIT_LOG"
    if grep -qxF "$unit" "$U/heals" 2>/dev/null; then
        grep -vxF "$unit" "$failed_file" > "$failed_file.t" 2>/dev/null; mv -f "$failed_file.t" "$failed_file"
    fi ;;
  enable)
    unit="${!#}"; echo "enable $unit" >> "$UNIT_LOG"
    grep -qxF "$unit" "$U/heals" 2>/dev/null && echo enabled > "$U/enabled.$unit" ;;
esac
exit 0
SYSTEMCTL

	# `timeout N cmd ...` — run cmd, ignore the bound. Keeps the suite identical
	# on a box whose coreutils has no `timeout` (macOS) and in CI.
	cat > "$BIN/timeout" <<'TIMEOUT'
#!/bin/bash
shift
exec "$@"
TIMEOUT

	cat > "$BIN/df" <<'DF'
#!/bin/bash
pct="$(cat "$TMPDIR_DISK" 2>/dev/null || echo 10)"
echo "Filesystem 1024-blocks Used Available Capacity Mounted"
echo "/dev/fake 100 ${pct} 10 ${pct}% /var/lib/docker"
DF

	chmod +x "$BIN/docker" "$BIN/systemctl" "$BIN/timeout" "$BIN/df"

	# tg_alert stand-in: the script sources a lib if present; here we prove the
	# alert path fires by shipping a lib that records the call.
	cat > "$TMP/telegram-alert-lib.sh" <<'TG'
tg_alert() { echo "$1" >> "$ALERT_LOG"; }
TG
	# The REAL metric sink, so the gauge assertions exercise the shipped writer.
	cp "$REPO_ROOT/lib/metric-sink-lib.sh" "$TMP/metric-sink-lib.sh"

	mkdir -p "$TMP/cimg"
	export IMAGE_LIST="$TMP/images" CONTAINER_IMAGE="$TMP/cimg" RMI_LOG="$TMP/rmi"
	: > "$IMAGE_LIST"; : > "$RMI_LOG"
	export FIXTURES="$TMP/fixtures" FX_UNITS="$TMP/units"
	export RESTART_LOG="$TMP/restarts" START_LOG="$TMP/starts"
	export ALERT_LOG="$TMP/alerts" UNIT_LOG="$TMP/units.log" PRUNE_LOG="$TMP/prunes"
	export TMPDIR_DISK="$TMP/diskpct"
	: > "$RESTART_LOG"; : > "$START_LOG"; : > "$ALERT_LOG"
	: > "$UNIT_LOG"; : > "$PRUNE_LOG"; : > "$FX_UNITS/failed"; : > "$FX_UNITS/heals"
	echo 10 > "$TMPDIR_DISK"
	export PATH="$BIN:$PATH"
	export OXPULSE_SELFHEAL_STATE_DIR="$TMP/state"
	export OXPULSE_SELFHEAL_LOCK="$TMP/lock"
	export OXPULSE_SELFHEAL_UPGRADE_LOCK="$TMP/nonexistent-upgrade.lock"
	export OXPULSE_SELFHEAL_HOLD="$TMP/selfheal.hold"
	export INSTALL_LIB_DIR="$TMP"
	export PARTNER_EDGE_TEXTFILE_DIR="$TMP/textfile"
	unset OXPULSE_SELFHEAL_DRY_RUN FAKE_DISK_AFTER DOCKER_START_RC DOCKER_RESTART_RC
}
teardown() { rm -rf "$TMP"; }

# name health started [hashc] [state] [finished] [oneoff]
fixture() { echo "$2 $3 ${4:-y} ${5:-running} ${6:-0} ${7:-False}" > "$FIXTURES/$1"; }
# container -> the image ref it runs
cimage()  { echo "$2" > "$CONTAINER_IMAGE/$1"; }
# id repo:tag  — the local image store
image()   { echo "$1 $2" >> "$IMAGE_LIST"; }
rmis()    { wc -l < "$RMI_LOG" | tr -d ' '; }
run_it()   { bash "$SCRIPT" >"$TMP/out" 2>&1; }
restarts() { wc -l < "$RESTART_LOG" | tr -d ' '; }
starts()   { wc -l < "$START_LOG"   | tr -d ' '; }
alerts()   { wc -l < "$ALERT_LOG"   | tr -d ' '; }
prunes()   { wc -l < "$PRUNE_LOG"   | tr -d ' '; }
state()    { awk -F= -v k="$2" '$1==k{print $2}' "$OXPULSE_SELFHEAL_STATE_DIR/${1//[^A-Za-z0-9._@-]/_}.state" 2>/dev/null; }
OLD=$(( $(date +%s) - 9999 ))

echo ""
echo "=== oxpulse-partner-edge-selfheal.sh ==="

# 1 — the healing case
setup; fixture oxpulse-partner-c1 unhealthy "$OLD"; run_it
[[ "$(restarts)" == 1 ]] && pass "unhealthy container is restarted" \
                         || fail "unhealthy container NOT restarted (got $(restarts))"
teardown

# 2 — the bound: this is the one that matters
setup; fixture oxpulse-partner-c1 unhealthy "$OLD"
mkdir -p "$OXPULSE_SELFHEAL_STATE_DIR"
printf 'win_start=%s\nattempts=3\nlast_attempt=%s\npending_since=0\n' \
  "$(( $(date +%s) - 100 ))" "$(( $(date +%s) - 999 ))" > "$OXPULSE_SELFHEAL_STATE_DIR/oxpulse-partner-c1.state"
run_it
r=$(restarts); a=$(alerts); g=$(state oxpulse-partner-c1 gave_up)
[[ "$r" == 0 ]] && pass "budget exhausted → NO fourth restart" || fail "restarted despite exhausted budget ($r)"
[[ "$a" == 1 ]] && pass "budget exhausted → alerts exactly once" || fail "alert count $a, expected 1"
[[ "$g" == 1 ]] && pass "budget exhausted → gave_up recorded" || fail "gave_up=$g, expected 1"
run_it
[[ "$(alerts)" == 1 ]] && pass "give-up alert is not repeated every tick" || fail "re-alerted: $(alerts)"
teardown

# 3 — start grace
setup; fixture oxpulse-partner-c1 unhealthy "$(( $(date +%s) - 5 ))"; run_it
[[ "$(restarts)" == 0 ]] && pass "inside start grace → not restarted" || fail "restarted a just-started container"
teardown

# 4 — no healthcheck is never touched
setup; fixture oxpulse-partner-c1 unhealthy "$OLD" n; run_it
[[ "$(restarts)" == 0 ]] && pass "container without a healthcheck is never touched" || fail "touched a container with no healthcheck"
teardown

# 5 — healthy is left alone
setup; fixture oxpulse-partner-c1 healthy "$OLD"; run_it
[[ "$(restarts)" == 0 ]] && pass "healthy container is left alone" || fail "restarted a healthy container"
teardown

# 6 — a pending attempt that came back healthy is counted as healed
setup; fixture oxpulse-partner-c1 healthy "$OLD"
mkdir -p "$OXPULSE_SELFHEAL_STATE_DIR"
printf 'pending_since=%s\nattempts=1\nwin_start=%s\n' "$(( $(date +%s) - 60 ))" "$(( $(date +%s) - 60 ))" \
  > "$OXPULSE_SELFHEAL_STATE_DIR/oxpulse-partner-c1.state"
run_it
[[ "$(state oxpulse-partner-c1 pending_since)" == 0 ]] && pass "verified heal clears the pending marker" \
                                                       || fail "pending marker not cleared"
teardown

# 7 — MIN_GAP
setup; fixture oxpulse-partner-c1 unhealthy "$OLD"
mkdir -p "$OXPULSE_SELFHEAL_STATE_DIR"
printf 'win_start=%s\nattempts=1\nlast_attempt=%s\npending_since=0\n' \
  "$(( $(date +%s) - 100 ))" "$(( $(date +%s) - 10 ))" > "$OXPULSE_SELFHEAL_STATE_DIR/oxpulse-partner-c1.state"
run_it
[[ "$(restarts)" == 0 ]] && pass "MIN_GAP not elapsed → not restarted" || fail "restarted inside MIN_GAP"
teardown

# 8 — a foreign container carrying our project label
setup; fixture all-rvpn-gate unhealthy "$OLD"; run_it
[[ "$(restarts)" == 0 ]] && pass "foreign container sharing our project label is never touched" \
                         || fail "restarted a container belonging to another project"
teardown

# 9 — stopped is started
setup; fixture oxpulse-partner-c1 none "$OLD" y exited "$OLD"; run_it
[[ "$(starts)" == 1 ]] && pass "stopped container is started" || fail "stopped container NOT started (got $(starts))"
teardown

# 10 — a compose one-off is never started
setup; fixture oxpulse-partner-c1 none "$OLD" y exited "$OLD" True; run_it
[[ "$(starts)" == 0 ]] && pass "compose one-off container is never started" || fail "started a compose one-off"
teardown

# 11 — stopped moments ago
setup; fixture oxpulse-partner-c1 none "$OLD" y exited "$(( $(date +%s) - 5 ))"; run_it
[[ "$(starts)" == 0 ]] && pass "just-stopped container is left alone" || fail "raced docker's restart policy"
teardown

# 12 — the global hold file stops everything
setup
fixture oxpulse-partner-c1 unhealthy "$OLD"
fixture oxpulse-partner-c2 none "$OLD" y exited "$OLD"
echo oxpulse-geoip-refresh.service > "$FX_UNITS/failed"
echo 99 > "$TMPDIR_DISK"
touch "$OXPULSE_SELFHEAL_HOLD"
run_it
tot=$(( $(restarts) + $(starts) + $(prunes) ))
[[ "$tot" == 0 && ! -s "$UNIT_LOG" ]] && pass "global hold file → no action of any kind" \
                                      || fail "acted despite the hold file (docker=$tot units=$(wc -l <"$UNIT_LOG"))"
# The early exit is a SECOND layer over the per-subject checks, and without its
# own assertion it is untested: every healer independently honours the global
# hold, so deleting the early exit changes no action and the case above stays
# green. What it uniquely buys is that a held node stops inspecting anything at
# all — a new healer added later that forgets its own _held call is covered by
# this and nothing else.
grep -q "taking no action" "$TMP/out" && pass "global hold exits BEFORE inspecting anything" \
                                      || fail "hold honoured only per-subject — a healer that forgets _held would act"
grep -q '^partner_edge_selfheal_hold 1$' "$PARTNER_EDGE_TEXTFILE_DIR/partner_edge_selfheal_hold.prom" 2>/dev/null \
	&& pass "a held node is VISIBLE as held in metrics" \
	|| fail "a held node is indistinguishable from a healthy one in metrics"
teardown

# 13 — a per-subject hold file stops exactly one subject
setup
fixture oxpulse-partner-c1 unhealthy "$OLD"
fixture oxpulse-partner-c2 unhealthy "$OLD"
touch "${OXPULSE_SELFHEAL_HOLD}.oxpulse-partner-c1"
run_it
[[ "$(restarts)" == 1 ]] && grep -qxF oxpulse-partner-c2 "$RESTART_LOG" \
	&& pass "per-subject hold file holds exactly that subject" \
	|| fail "per-subject hold wrong: restarted [$(tr '\n' ' ' <"$RESTART_LOG")]"
teardown

# 14 — a failed oxpulse unit is restarted and recorded healed
setup
echo oxpulse-geoip-refresh.service > "$FX_UNITS/failed"
echo oxpulse-geoip-refresh.service > "$FX_UNITS/heals"
run_it
grep -q "restart oxpulse-geoip-refresh.service" "$UNIT_LOG" \
	&& pass "a failed oxpulse unit is restarted" || fail "failed unit NOT restarted"
grep -q "oxpulse-geoip-refresh.service: recovered" "$TMP/out" \
	&& pass "a unit that comes back is recorded as healed" || fail "recovery not recorded"
teardown

# 15 — never restart itself
setup
printf 'oxpulse-partner-edge-selfheal.service\n' > "$FX_UNITS/failed"
run_it
[[ ! -s "$UNIT_LOG" ]] && pass "the healer never restarts its own service" \
                       || fail "restarted itself: $(cat "$UNIT_LOG")"
teardown

# 16 — enable-set drift is repaired
setup
echo disabled > "$FX_UNITS/enabled.oxpulse-xray-update.timer"
echo oxpulse-xray-update.timer > "$FX_UNITS/heals"
run_it
grep -q "enable oxpulse-xray-update.timer" "$UNIT_LOG" \
	&& pass "a declared unit found disabled is enabled" || fail "enable drift NOT repaired"
teardown

# 17 — a unit outside the declared set is never enabled
setup
echo disabled > "$FX_UNITS/enabled.oxpulse-partner-edge-ru-subnets-update.timer"
run_it
grep -q "ru-subnets" "$UNIT_LOG" \
	&& fail "enabled a unit that is deliberately disabled fleet-wide" \
	|| pass "a unit outside the declared enable-set is never enabled"
teardown

# 18 — the healer's own timer must not be in its enable-set
if grep -qE '^\s*oxpulse-partner-edge-selfheal\.timer\s*$' \
     <(awk '/^ENABLE_UNITS=\(/{f=1;next} f&&/^\)/{exit} f' "$SCRIPT"); then
	fail "the healer's own timer is in its enable-set — it would re-enable itself when switched off"
else
	pass "the healer's own timer is absent from its enable-set"
fi

# 19 — the per-tick action cap
setup
printf 'oxpulse-a.service\noxpulse-b.service\noxpulse-c.service\n' > "$FX_UNITS/failed"
export OXPULSE_SELFHEAL_ACTS_PER_TICK=2
run_it
n=$(grep -c '^restart ' "$UNIT_LOG")
[[ "$n" == 2 ]] && pass "per-tick action cap honoured ($n of 3 failed units)" \
               || fail "action cap ignored: $n actions, expected 2"
unset OXPULSE_SELFHEAL_ACTS_PER_TICK
teardown

# 20 — a disk below the threshold is never pruned
setup; echo 50 > "$TMPDIR_DISK"; run_it
[[ "$(prunes)" == 0 ]] && pass "disk below threshold → never pruned" || fail "pruned a disk that was fine"
teardown

# 21 — a disk at the threshold is reclaimed
setup; echo 92 > "$TMPDIR_DISK"; export FAKE_DISK_AFTER=40; run_it
[[ "$(prunes)" -ge 1 ]] && pass "disk over threshold → reclaimed" || fail "disk over threshold NOT reclaimed"
grep -q "disk reclaimed to 40%" "$TMP/out" && pass "reclaim is VERIFIED, not assumed" \
                                           || fail "reclaim outcome not verified"
teardown

# 23 — a prune that does NOT help is recorded as a failure, never as a heal.
# This is the assertion that keeps the disk healer honest: without it the script
# could report success on every run while the disk stayed full, and the give-up
# alert — the only thing that reaches a human before the node wedges — would
# never fire.
setup; echo 92 > "$TMPDIR_DISK"; export FAKE_DISK_AFTER=95; run_it
grep -q "disk still at 95% after reclaim" "$TMP/out" \
	&& pass "a prune that frees nothing is recorded as failed" \
	|| fail "an ineffective prune was not reported as such"
grep -q "disk reclaimed" "$TMP/out" \
	&& fail "claimed a heal while the disk was still over threshold" \
	|| pass "never claims a heal it cannot verify"
unset FAKE_DISK_AFTER
teardown

# 22 — every container gets a series, not just the last one
setup
fixture oxpulse-partner-aaa healthy "$OLD"
fixture oxpulse-partner-mmm unhealthy "$OLD"
fixture oxpulse-partner-zzz healthy "$OLD"
run_it
prom="$PARTNER_EDGE_TEXTFILE_DIR/partner_edge_container_unhealthy.prom"
n=$(grep -c '^partner_edge_container_unhealthy{' "$prom" 2>/dev/null || echo 0)
types=$(grep -c '^# TYPE' "$prom" 2>/dev/null || echo 0)
[[ "$n" == 3 ]] && pass "every container has a gauge series ($n of 3)" \
               || fail "only $n of 3 container series survived the gauge write"
[[ "$types" == 1 ]] && pass "exactly one # TYPE line" || fail "$types TYPE lines — node_exporter drops the file"
grep -q '^partner_edge_container_unhealthy{container="oxpulse-partner-mmm"} 1$' "$prom" \
	&& pass "the unhealthy container reports 1" || fail "unhealthy container's sample is wrong"
teardown

# 24 — a CRASHLOOPING long-running service is bounded and escalates.
#
# The subtle one. `systemctl restart` on a long-running service returns as soon
# as it STARTS, so `is-failed` immediately after is false even for a unit that
# dies three seconds later. If success on that reading cleared the budget, a
# crashlooper would get a fresh budget every tick: one restart per UNIT_GAP
# forever, no given_up, no alert, and nothing in any metric to tell it apart
# from a healthy node — the unbounded healer this whole file exists to prevent.
# oxpulse-awg-params-agent.service is long-running on every node.
setup
export OXPULSE_SELFHEAL_UNIT_GAP=0        # the gap is not what is under test here
echo oxpulse-crash.service > "$FX_UNITS/heals"   # "restart" always reports success
for i in 1 2 3 4 5; do
	echo oxpulse-crash.service > "$FX_UNITS/failed"   # ...and it is failed again next tick
	run_it
done
n=$(grep -c '^restart oxpulse-crash.service$' "$UNIT_LOG")
[[ "$n" == 3 ]] && pass "a crashlooping unit is bounded at 3 restarts ($n)" \
               || fail "crashloop restarted $n times, expected 3 — the budget is being reset on a false 'recovered'"
[[ "$(alerts)" == 1 ]] && pass "a crashlooping unit escalates exactly once" \
                       || fail "crashloop alert count $(alerts), expected 1"
unset OXPULSE_SELFHEAL_UNIT_GAP
teardown

# --- image garbage collection -------------------------------------------------
# An upgrade pulls the new images and never removes the ones it replaced, so every
# edge grows one tag per release forever. Measured 2026-08-11 before this existed:
# 90 stale tags on rvpn reaching back to v0.8.0, 286 across the five nodes, 12.2 GB
# recoverable on rvpn alone — and nothing had ever removed one.
img_setup() {   # a fleet-shaped image store: ours across 4 releases + a neighbour's
	setup
	echo 92 > "$TMPDIR_DISK"; export FAKE_DISK_AFTER_RMI=40
	fixture oxpulse-partner-xray healthy "$OLD"
	cimage  oxpulse-partner-xray ghcr.io/anatolykoptev/partner-edge-xray:v0.16.20
	image aaa1 ghcr.io/anatolykoptev/partner-edge-xray:v0.16.20
	image aaa2 ghcr.io/anatolykoptev/partner-edge-xray:v0.16.19
	image aaa3 ghcr.io/anatolykoptev/partner-edge-xray:v0.16.9
	image aaa4 ghcr.io/anatolykoptev/partner-edge-xray:v0.8.0
	image bbb1 rocketvpn-monitor-api:latest          # the neighbouring project's
	image bbb2 rust:1-bullseye                       # somebody's build base
}

# 25 — stale tags of OUR repos are removed
img_setup; run_it
[[ "$(rmis)" == 2 ]] && pass "image GC removes exactly the stale tags (2 of 4)" \
                     || fail "removed $(rmis) tags, expected 2 — [$(tr '\n' ' ' <"$RMI_LOG")]"
grep -q 'v0.8.0' "$RMI_LOG" && grep -q 'v0.16.9' "$RMI_LOG" \
	&& pass "the oldest releases are the ones removed" || fail "wrong tags removed"
teardown

# 26 — the newest two are kept, because upgrade.sh rolls back to the PREVIOUS image
img_setup; run_it
grep -qE 'v0\.16\.(20|19)' "$RMI_LOG" \
	&& fail "removed a release upgrade.sh would roll back to" \
	|| pass "the current and previous releases are kept"
teardown

# 27 — a NEIGHBOUR's image is never touched. rvpn shares this docker daemon with
# the rvpnm project; a prune that reached their images would be an outage we
# caused on somebody else's service.
img_setup; run_it
grep -qE 'rocketvpn|rust' "$RMI_LOG" \
	&& fail "removed an image belonging to another project" \
	|| pass "images outside our repos are never touched"
teardown

# 28 — version ordering is NUMERIC. A lexical sort puts v0.16.9 above v0.16.20 and
# would delete the running release.
img_setup; run_it
grep -q 'v0.16.9' "$RMI_LOG" && grep -qv 'v0.16.20' "$RMI_LOG" \
	&& pass "v0.16.20 outranks v0.16.9 (version sort, not lexical)" \
	|| fail "version ordering is lexical — the newest release would be deleted"
teardown

# 29 — below the threshold nothing is collected at all
img_setup; echo 50 > "$TMPDIR_DISK"; run_it
[[ "$(rmis)" == 0 ]] && pass "no image GC below the disk threshold" || fail "collected images on a healthy disk"
teardown

# 30 — ORDERING: our own garbage is taken BEFORE the shared build cache. On a box
# whose docker daemon is shared with another project, pruning their build cache
# when removing our own stale tags would have been enough is a cost we imposed
# for nothing.
img_setup; run_it
if grep -q 'builder' "$PRUNE_LOG"; then
	fail "pruned the SHARED build cache even though our own tags freed enough"
else
	pass "shared build cache is untouched when our own tags suffice"
fi
teardown

# 31 — THE DANGEROUS ONE. A container running an OLD release must keep its image
# even though that version is not among the newest kept. A node that skipped
# upgrades, or one mid-rollback, is exactly this shape — and deleting the image a
# running service was created from is an outage we caused ourselves.
# The keep-set is compared by image ID, not by tag string, because a container can
# reference an image by digest or under a different tag.
img_setup
cimage oxpulse-partner-xray ghcr.io/anatolykoptev/partner-edge-xray:v0.8.0
run_it
grep -q 'v0.8.0' "$RMI_LOG" \
	&& fail "deleted the image a RUNNING container was created from" \
	|| pass "an in-use image is kept however old its version"
teardown

# A managed container can run a THIRD-PARTY image whose version scheme has nothing
# to do with ours. Every edge runs hysteria2 as `tobyxdd/hysteria:v2.8.2`, and
# v2.8.2 version-sorts above every v0.16.x we have ever released. The fixtures
# above could not see this: they gave the neighbour's images repos we do NOT run,
# so the repo filter dropped them before they could reach the version pool. Here
# the third-party image IS one of our services, so it reaches it.
hy_setup() {
	img_setup
	fixture oxpulse-partner-hysteria2 healthy "$OLD"
	cimage  oxpulse-partner-hysteria2 tobyxdd/hysteria:v2.8.2
	image ccc1 tobyxdd/hysteria:v2.8.2
	image ccc2 tobyxdd/hysteria:v2
}

# 32 — THE ONE ONLY THE LIVE NODE SHOWED. Measured on zvonilka 2026-08-11 against
# the real daemon, before this had ever run under real disk pressure: a keep-set
# computed across all repos at once handed BOTH slots to v2.8.2 and v2, so every
# release we had fell outside it and only the in-use-by-ID guard saved the four
# that happened to be running.
# The assertion is on the PREVIOUS release, not the current one. The current one
# survives either way — the in-use guard catches it — so asserting on it passes
# under the bug and proves nothing. The previous release is the image upgrade.sh
# rolls back to, it is by definition not running, and it is what actually died.
hy_setup; run_it
grep -q 'v0.16.19' "$RMI_LOG" \
	&& fail "a third-party version scheme took both keep slots — the rollback target was deleted" \
	|| pass "keep-set is per repo: a neighbour's higher version cannot evict our releases"
teardown

# 33 — and the fix must not degrade into "keep everything": our own stale tags are
# still collected while the third-party repo is present.
hy_setup; run_it
[[ "$(rmis)" == 2 ]] && grep -q 'v0.8.0' "$RMI_LOG" && grep -q 'v0.16.9' "$RMI_LOG" \
	&& pass "our own stale tags are still collected alongside a third-party repo" \
	|| fail "collected $(rmis) tag(s) — [$(tr '\n' ' ' <"$RMI_LOG")]"
teardown

# 34 — the third-party repo gets the same rule applied to ITSELF, not an exemption:
# it keeps its newest KEEP_RELEASES and no more. With only two tags present both
# survive; the assertion is that neither is treated as ours to delete.
hy_setup; run_it
grep -q 'tobyxdd' "$RMI_LOG" \
	&& fail "deleted a tag of a service we run but do not build" \
	|| pass "a third-party repo we run is collected by the same per-repo rule"
teardown

# 35 — the keep-set is matched by FULL REF, not by version number. Two of our own
# services can sit on different releases: an upgrade that failed one service, or a
# node that skipped one, leaves a repo lagging. Then a version that is current for
# the laggard is stale for the others — and matching on the version alone lets the
# stale one ride the laggard's keep entry and never be collected. The cost is a
# leak rather than an outage, which is exactly why nothing else would report it.
setup
echo 92 > "$TMPDIR_DISK"; export FAKE_DISK_AFTER_RMI=40
fixture oxpulse-partner-xray healthy "$OLD"
fixture oxpulse-partner-sfu  healthy "$OLD"
cimage  oxpulse-partner-xray ghcr.io/anatolykoptev/partner-edge-xray:v0.16.21
cimage  oxpulse-partner-sfu  ghcr.io/anatolykoptev/partner-edge-sfu:v0.16.19
image xxx1 ghcr.io/anatolykoptev/partner-edge-xray:v0.16.21
image xxx2 ghcr.io/anatolykoptev/partner-edge-xray:v0.16.20
image xxx3 ghcr.io/anatolykoptev/partner-edge-xray:v0.16.19
image sss1 ghcr.io/anatolykoptev/partner-edge-sfu:v0.16.19
image sss2 ghcr.io/anatolykoptev/partner-edge-sfu:v0.16.18
run_it
[[ "$(rmis)" == 1 ]] && grep -q 'xray:v0.16.19' "$RMI_LOG" \
	&& pass "a stale tag is collected even while another repo still keeps that version" \
	|| fail "collected $(rmis) — [$(tr '\n' ' ' <"$RMI_LOG")], expected only xray:v0.16.19"
teardown

# --- --collect-images: the collector alone, called by upgrade.sh ---------------
# The disk healer opens at 85%, so between upgrades a node grows a tag per service
# per release with nothing reporting it. upgrade.sh calls this the moment it
# creates that garbage. The mode must do ONE thing — anything else would be a
# healer running inside an upgrade, on a node whose services are mid-restart.
run_collect() { bash "$SCRIPT" --collect-images >"$TMP/out" 2>&1; }

# 36 — it collects, at any disk level. The threshold is the healer's rule, not
# this one's: garbage is garbage whether or not the disk is under pressure.
img_setup; echo 12 > "$TMPDIR_DISK"; run_collect
[[ "$(rmis)" == 2 ]] && pass "--collect-images collects regardless of disk level" \
                     || fail "collected $(rmis) at 12% disk, expected 2"
teardown

# 37 — and ONLY that. A container restart or a unit heal here would fire during an
# upgrade, against services that are legitimately mid-restart.
img_setup
fixture oxpulse-partner-c1 unhealthy "$OLD"
echo "oxpulse-geoip-refresh.service" > "$FX_UNITS/failed"
run_collect
[[ "$(restarts)" == 0 && "$(starts)" == 0 ]] \
	&& pass "--collect-images restarts no container" \
	|| fail "it restarted $(restarts) container(s) and started $(starts) — that is the healer's job, not the upgrade's"
[[ "$(wc -l < "$UNIT_LOG" | tr -d ' ')" == 0 ]] \
	&& pass "--collect-images touches no systemd unit" \
	|| fail "it acted on a unit: [$(tr '\n' ' ' <"$UNIT_LOG")]"
[[ "$(prunes)" == 0 ]] \
	&& pass "--collect-images never prunes the shared build cache" \
	|| fail "it pruned [$(tr '\n' ' ' <"$PRUNE_LOG")] — the neighbour's cache is not ours to reclaim here"
teardown

# 38 — the hold file stops it too. An operator holding a node expects NOTHING to
# touch docker there, and an upgrade is not an exemption.
img_setup; : > "$OXPULSE_SELFHEAL_HOLD"; run_collect
[[ "$(rmis)" == 0 ]] && pass "--collect-images honours the hold file" \
                     || fail "it removed $(rmis) tag(s) on a held node"
teardown

# 39 — it spends no disk budget. That budget exists to stop a node thrashing under
# pressure and to give up loudly; charging an upgrade against it would exhaust the
# allowance and fire the give-up alert on a node that is not in trouble.
img_setup; run_collect
_att="$(state disk attempts)"
[[ -z "$_att" || "$_att" == 0 ]] \
	&& pass "--collect-images spends no disk budget (attempts=${_att:-unset})" \
	|| fail "it charged the disk budget: attempts=$_att — the give-up alert would fire on a healthy node"
teardown

echo ""
echo "PASS=$PASS FAIL=$FAIL"
[[ "$FAIL" -eq 0 ]]