Gladex Agent Logs

@@ -883,8 +883,24 @@
 run_mutation "M7 the first match wins instead of the newest" \
     'IPD_FILE="$IPD_CAND"' \
     'IPD_FILE="${IPD_FILE:-$IPD_CAND}"' \
     "$FX_NEWEST_DRIFT" "error" "1" "error" "1"
+# M8 — a CHECK-ERROR line stops being a verdict-shaped line at all: the writer
+# writes it, the ALERTS channel carries it, and this row answers "no verdict
+# line" — the (r1) defect this row was built to close, reachable again through
+# section 22's REPLAYED file rather than a fixture this suite typed.
+run_mutation "M8 a CHECK-ERROR line is no longer read as a verdict" \
+    '*"OK: No drift"*|*"DRIFT "*|*"CHECK-ERROR "*) IPD_VERDICT="$IPD_LINE" ;;' \
+    '*"OK: No drift"*|*"DRIFT "*) IPD_VERDICT="$IPD_LINE" ;;' \
+    "$WREP/no_a_records" "warning" "0" "ok" "0"
+# M9 — recognised, then rendered as something other than red: the CHECK-ERROR
+# verdict falls through to `*` and becomes `cannot verify: unrecognised
+# verdict`, i.e. a warning standing where a red belongs — the (109)(e)
+# anti-pattern, reached by a line the writer itself produced.
+run_mutation "M9 a CHECK-ERROR verdict rendered as a warning" \
+    '        *"DRIFT "*|*"CHECK-ERROR "*)' \
+    '        *"DRIFT "*)' \
+    "$WREP/dns_lookup_failed" "warning" "0" "ok" "0"
 
 MD5_AFTER=$(md5sum "$TOOL" | cut -d' ' -f1)
 [ "$MD5_BEFORE" = "$MD5_AFTER" ] \
     && ok "the tool is byte-identical after the battery (mutants are applied to a copy)" \

> build · mimo-v2.6-flash-free
$ cd /data/repo && bash -n tests/test_system_status_ip_drift.sh && time bash tests/test_system_status_ip_drift.sh 2>&1 | tail -25
ok   - M4 the sandbox guard defeated: the mutant parses
  ok   - M4 the sandbox guard defeated caught (status='ok' errors='0' overall='ok' rc=0; correct is warning/0/ok/0)
  ok   - M5 the want= strip defeated: plant applied exactly once
  ok   - M5 the want= strip defeated: the original pattern is gone
  ok   - M5 the want= strip defeated: the mutant parses
  ok   - M5 the want= strip defeated caught (status='ok' errors='0' overall='ok' rc=0; correct is ok/0/ok/0)
  ok   - M6 the printable-ASCII belt defeated: plant applied exactly once
  ok   - M6 the printable-ASCII belt defeated: the original pattern is gone
  ok   - M6 the printable-ASCII belt defeated: the mutant parses
  ok   - M6 the printable-ASCII belt defeated caught (status='ok' errors='0' overall='ok' rc=0; correct is ok/0/ok/0)
  ok   - M7 the first match wins instead of the newest: plant applied exactly once
  ok   - M7 the first match wins instead of the newest: the original pattern is gone
  ok   - M7 the first match wins instead of the newest: the mutant parses
  ok   - M7 the first match wins instead of the newest caught (status='ok' errors='0' overall='ok' rc=0; correct is error/1/error/1)
  ok   - M8 a CHECK-ERROR line is no longer read as a verdict: plant applied exactly once
  ok   - M8 a CHECK-ERROR line is no longer read as a verdict: the original pattern is gone
  ok   - M8 a CHECK-ERROR line is no longer read as a verdict: the mutant parses
  ok   - M8 a CHECK-ERROR line is no longer read as a verdict caught (status='warning' errors='0' overall='ok' rc=0; correct is warning/0/ok/0)
  ok   - M9 a CHECK-ERROR verdict rendered as a warning: plant applied exactly once
  ok   - M9 a CHECK-ERROR verdict rendered as a warning: the original pattern is gone
  ok   - M9 a CHECK-ERROR verdict rendered as a warning: the mutant parses
  ok   - M9 a CHECK-ERROR verdict rendered as a warning caught (status='warning' errors='0' overall='ok' rc=0; correct is warning/0/ok/0)
  ok   - the tool is byte-identical after the battery (mutants are applied to a copy)

=== Results: 288 passed, 0 failed ===

real	0m36.770s
user	0m24.410s
sys	0m21.118s
> build · mimo-v2.6-flash-free
The mutation expectations I wrote for M8/M9 were wrong (they must be the *correct* verdict on those dirs, or a mutant behaving like the real tool could report "caught"). Fixing:
$ cd /data/repo && grep -n 'WREP/no_a_records" \|WREP/dns_lookup_failed" ' tests/test_system_status_ip_drift.sh
894:    "$WREP/no_a_records" "warning" "0" "ok" "0"
902:    "$WREP/dns_lookup_failed" "warning" "0" "ok" "0"
> build · mimo-v2.6-flash-free
← Edit repo/tests/test_system_status_ip_drift.sh
Index: repo/tests/test_system_status_ip_drift.sh
===================================================================
--- repo/tests/test_system_status_ip_drift.sh
+++ repo/tests/test_system_status_ip_drift.sh
@@ -890,9 +890,9 @@
 # section 22's REPLAYED file rather than a fixture this suite typed.
 run_mutation "M8 a CHECK-ERROR line is no longer read as a verdict" \
     '*"OK: No drift"*|*"DRIFT "*|*"CHECK-ERROR "*) IPD_VERDICT="$IPD_LINE" ;;' \
     '*"OK: No drift"*|*"DRIFT "*) IPD_VERDICT="$IPD_LINE" ;;' \
-    "$WREP/no_a_records" "warning" "0" "ok" "0"
+    "$WREP/no_a_records" "error" "1" "error" "1"
 # M9 — recognised, then rendered as something other than red: the CHECK-ERROR
 # verdict falls through to `*` and becomes `cannot verify: unrecognised
 # verdict`, i.e. a warning standing where a red belongs — the (109)(e)
 # anti-pattern, reached by a line the writer itself produced.

> build · mimo-v2.6-flash-free
← Edit repo/tests/test_system_status_ip_drift.sh
Index: repo/tests/test_system_status_ip_drift.sh
===================================================================
--- repo/tests/test_system_status_ip_drift.sh
+++ repo/tests/test_system_status_ip_drift.sh
@@ -898,9 +898,9 @@
 # anti-pattern, reached by a line the writer itself produced.
 run_mutation "M9 a CHECK-ERROR verdict rendered as a warning" \
     '        *"DRIFT "*|*"CHECK-ERROR "*)' \
     '        *"DRIFT "*)' \
-    "$WREP/dns_lookup_failed" "warning" "0" "ok" "0"
+    "$WREP/dns_lookup_failed" "error" "1" "error" "1"
 
 MD5_AFTER=$(md5sum "$TOOL" | cut -d' ' -f1)
 [ "$MD5_BEFORE" = "$MD5_AFTER" ] \
     && ok "the tool is byte-identical after the battery (mutants are applied to a copy)" \

> build · mimo-v2.6-flash-free
$ cd /data/repo && bash tests/test_system_status_ip_drift.sh 2>&1 | grep -E "replay|M8|M9|Results" | sed -n '1,70p'
=== 22. the WRITER's own exit branches, replayed through tools/ip-drift-cron ===
  ok   - replay: the writer and both sandbox hooks it needs are present
  ok   - replay ok: the writer's no-drift branch exits 0 through this harness
  ok   - replay ok: the writer wrote its own no-drift verdict into the sandbox
  ok   - replay ok: a no-drift run raises no alert — this harness does not force red
  ok   - replay ok: tool emitted JSON
  ok   - replay ok: output is valid JSON and decodes as UTF-8 (the row's detail did not break it)
  ok   - replay ok: status is 'ok'
  ok   - replay ok: detail carries 'OK: No drift'
  ok   - replay ok: errors 0
  ok   - replay ok: overall ok
  ok   - replay ok: exit 0
  ok   - replay no_a_records: the writer exits 3 on this branch
  ok   - replay no_a_records: the writer's own classification is in its dated log (code=no_a_records)
  ok   - replay no_a_records: the same code reached the writer's escalation channel
  ok   - replay no_a_records: tool emitted JSON
  ok   - replay no_a_records: output is valid JSON and decodes as UTF-8 (the row's detail did not break it)
  ok   - replay no_a_records: status is 'error'
  ok   - replay no_a_records: detail carries 'CHECK-ERROR gladex.de'
  ok   - replay no_a_records: errors 1
  ok   - replay no_a_records: overall error
  ok   - replay no_a_records: exit 1
  ok   - replay no_a_records: the row carries the writer's own code
  ok   - replay no_a_records: the row carries the checker's exit
  ok   - replay no_a_records: the detail quotes the WRITER's line verbatim (typed by no fixture here)
  ok   - replay dns_lookup_failed: the writer exits 3 on this branch
  ok   - replay dns_lookup_failed: the writer's own classification is in its dated log (code=dns_lookup_failed)
  ok   - replay dns_lookup_failed: the same code reached the writer's escalation channel
  ok   - replay dns_lookup_failed: tool emitted JSON
  ok   - replay dns_lookup_failed: output is valid JSON and decodes as UTF-8 (the row's detail did not break it)
  ok   - replay dns_lookup_failed: status is 'error'
  ok   - replay dns_lookup_failed: detail carries 'CHECK-ERROR gladex.de'
  ok   - replay dns_lookup_failed: errors 1
  ok   - replay dns_lookup_failed: overall error
  ok   - replay dns_lookup_failed: exit 1
  ok   - replay dns_lookup_failed: the row carries the writer's own code
  ok   - replay dns_lookup_failed: the row carries the checker's exit
  ok   - replay dns_lookup_failed: the detail quotes the WRITER's line verbatim (typed by no fixture here)
  ok   - replay public_ip_unavailable: the writer exits 3 on this branch
  ok   - replay public_ip_unavailable: the writer's own classification is in its dated log (code=public_ip_unavailable)
  ok   - replay public_ip_unavailable: the same code reached the writer's escalation channel
  ok   - replay public_ip_unavailable: tool emitted JSON
  ok   - replay public_ip_unavailable: output is valid JSON and decodes as UTF-8 (the row's detail did not break it)
  ok   - replay public_ip_unavailable: status is 'error'
  ok   - replay public_ip_unavailable: detail carries 'CHECK-ERROR gladex.de'
  ok   - replay public_ip_unavailable: errors 1
  ok   - replay public_ip_unavailable: overall error
  ok   - replay public_ip_unavailable: exit 1
  ok   - replay public_ip_unavailable: the row carries the writer's own code
  ok   - replay public_ip_unavailable: the row carries the checker's exit
  ok   - replay public_ip_unavailable: the detail quotes the WRITER's line verbatim (typed by no fixture here)
  ok   - M8 a CHECK-ERROR line is no longer read as a verdict: plant applied exactly once
  ok   - M8 a CHECK-ERROR line is no longer read as a verdict: the original pattern is gone
  ok   - M8 a CHECK-ERROR line is no longer read as a verdict: the mutant parses
  ok   - M8 a CHECK-ERROR line is no longer read as a verdict caught (status='warning' errors='0' overall='ok' rc=0; correct is error/1/error/1)
  ok   - M9 a CHECK-ERROR verdict rendered as a warning: plant applied exactly once
  ok   - M9 a CHECK-ERROR verdict rendered as a warning: the original pattern is gone
  ok   - M9 a CHECK-ERROR verdict rendered as a warning: the mutant parses
  ok   - M9 a CHECK-ERROR verdict rendered as a warning caught (status='warning' errors='0' overall='ok' rc=0; correct is error/1/error/1)
=== Results: 288 passed, 0 failed ===
> build · mimo-v2.6-flash-free
$ cd /data/repo && S=$(mktemp -d /tmp/ipdreplay.XXXX) && mkdir -p "$S" && cat > "$S/check" <<'EOF'
#!/bin/sh
case "${WREP_SCEN:-}" in
  no_a_records) echo '{"public_ip":null,"dns_records":[],"drift":null,"error":"No DNS A records found","error_code":"no_a_records","exit_code":3}'; exit 3 ;;
esac
EOF
chmod +x "$S/check" && WREP_SCEN=no_a_records IPDRIFT_CHECK_BIN="$S/check" IPDRIFT_LOG_DIR="$S/logs" ./tools/ip-drift-cron gladex.de; echo "cron rc=$?"; echo '--- dated log'; cat "$S/logs"/ip-drift-*.log; echo '--- alerts'; cat "$S/logs/ip-drift-ALERTS.log"; echo '--- row'; GLADEX_IPDRIFT_LOG_DIR="$S/logs" ./tools/system-status --format human 2>/dev/null | grep -E 'ip-drift|Overall'; rm -rf "$S"
cron rc=3
--- dated log
[2026-10-05T00:17:15Z] Starting IP drift check for gladex.de
{"public_ip":null,"dns_records":[],"drift":null,"error":"No DNS A records found","error_code":"no_a_records","exit_code":3}
[2026-10-05T00:17:15Z] CHECK-ERROR gladex.de: code=no_a_records detail=No DNS A records found checker_exit=3
[2026-10-05T00:17:15Z] DIAGNOSIS: the DNS A record for gladex.de is GONE (query completed, no answer).
[2026-10-05T00:17:15Z] ACTION: inspect the zone with 'pdns-api.py records'; do NOT rewrite DNS from here.
[2026-10-05T00:17:15Z] DNS NOT MODIFIED (deliberate). A VPS IP change needs investor/operator sign-off.
[2026-10-05T00:17:15Z] Escalated via /tmp/ipdreplay.CDCd/logs/ip-drift-ALERTS.log + NEEDS-INVESTOR in PROGRESS.md
[2026-10-05T00:17:15Z] Drift check complete
[2026-10-05T00:17:15Z] CHECK-ERROR gladex.de: code=no_a_records detail=No DNS A records found checker_exit=3
--- alerts
[2026-10-05T00:17:15Z] CHECK-ERROR gladex.de: code=no_a_records detail=No DNS A records found checker_exit=3
--- row
  ip-drift                  [FAIL] [2026-10-05T00:17:15Z] CHECK-ERROR gladex.de: code=no_a_records detail=No DNS A records found checker_exit=3 (3s old)
Overall: 2 CHECK(S) FAILED
> build · mimo-v2.6-flash-free
$ cd /data/repo && bash tests/test_ip_drift_cron.sh 2>&1 | tail -3; echo "cron-suite rc=$?"; bash tests/test_ip_drift_cron.sh --mutations 2>&1 | tail -3; echo "mutations rc=$?"
ok   - C8 guard mode keeps only the two UNGUARDED opens

=== Results: 103 passed, 0 failed ===
cron-suite rc=0
  ok   - M5 → wrong verdict lands on the intended assertion: 'err_no_a → exit 3'

=== Results: 124 passed, 0 failed ===
mutations rc=0
> build · mimo-v2.6-flash-free
$ cd /data/repo && bash tests/test_detached_children.sh 2>&1 | tail -4; echo "detached rc=$?"
Command moved to the background (shell ID: sh_1096d4d6f0016kwt8MTSh7SNe6).
Output is streaming to: /root/.local/share/opencode/shell/fa3e85b2145a60c127b5c9261e76a7ba6d520299/sh_1096d4d6f0016kwt8MTSh7SNe6.out
> build · mimo-v2.6-flash-free

Select another run log from the list above. Only files matching run-YYYYMMDD-HHMMSS-N.log are readable.

[Mon Oct  5 02:15:09 2026] 127.0.0.1:54242 Accepted
[Mon Oct  5 02:15:10 2026] 127.0.0.1:54242 Closing
[Mon Oct  5 02:15:10 2026] 127.0.0.1:54250 Accepted
[Mon Oct  5 02:15:10 2026] 127.0.0.1:54250 Closing
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40502 Accepted
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40502 Closing
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40514 Accepted
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40514 Closing
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40528 Accepted
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40528 Closing
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40544 Accepted
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40544 Closing
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40560 Accepted
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40560 Closing
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40576 Accepted
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40576 Closing
[Mon Oct  5 02:15:55 2026] 127.0.0.1:40582 Accepted
[Mon Oct  5 02:15:56 2026] 127.0.0.1:40582 Closing
[Mon Oct  5 02:15:56 2026] 127.0.0.1:40596 Accepted
[Mon Oct  5 02:15:56 2026] 127.0.0.1:40596 Closing
[Mon Oct  5 02:15:56 2026] 127.0.0.1:40612 Accepted
[Mon Oct  5 02:15:56 2026] 127.0.0.1:40612 Closing
[Mon Oct  5 02:15:56 2026] 127.0.0.1:40614 Accepted
[Mon Oct  5 02:15:56 2026] 127.0.0.1:40614 Closing
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53040 Accepted
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53040 Closing
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53048 Accepted
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53048 Closing
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53050 Accepted
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53050 Closing
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53058 Accepted
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53058 Closing
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53070 Accepted
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53070 Closing
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53078 Accepted
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53078 Closing
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53090 Accepted
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53090 Closing
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53092 Accepted
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53092 Closing
[Mon Oct  5 02:17:15 2026] 127.0.0.1:53106 Accepted
[Mon Oct  5 02:17:16 2026] 127.0.0.1:53106 Closing
[Mon Oct  5 02:17:16 2026] 127.0.0.1:53116 Accepted
[Mon Oct  5 02:17:16 2026] 127.0.0.1:53116 Closing
[Mon Oct  5 02:21:27 2026] 127.0.0.1:51362 Accepted
[Mon Oct  5 02:21:27 2026] 127.0.0.1:51362 Closed without sending a request; it was probably just an unused speculative preconnection
[Mon Oct  5 02:21:27 2026] 127.0.0.1:51362 Closing
[Mon Oct  5 02:21:29 2026] 127.0.0.1:51372 Accepted
[Mon Oct  5 02:21:29 2026] 127.0.0.1:51372 Closed without sending a request; it was probably just an unused speculative preconnection
[Mon Oct  5 02:21:29 2026] 127.0.0.1:51372 Closing
[Mon Oct  5 02:22:04 2026] 127.0.0.1:53428 Accepted
[Mon Oct  5 02:22:04 2026] 127.0.0.1:53428 Closed without sending a request; it was probably just an unused speculative preconnection
[Mon Oct  5 02:22:04 2026] 127.0.0.1:53428 Closing
[Mon Oct  5 02:22:05 2026] 127.0.0.1:36310 Accepted
[Mon Oct  5 02:22:05 2026] 127.0.0.1:36310 Closed without sending a request; it was probably just an unused speculative preconnection
[Mon Oct  5 02:22:05 2026] 127.0.0.1:36310 Closing
[Mon Oct  5 02:24:45 2026] 127.0.0.1:42952 Accepted
[Mon Oct  5 02:24:45 2026] 127.0.0.1:42952 Closing
[Mon Oct  5 02:24:45 2026] 127.0.0.1:42966 Accepted