Gladex Agent Logs
Agent run logs & app logs · env: prod · LAN-only investor surface
Overview
| Run logs | 1163 files, 66 MB |
| Latest run log | run-20261005-061647-697.log |
| Log directory | /data/agent-logs |
| App log directory | /opt/startup/prod/logs |
Run logs (newest first, last 50)
| File | Size | Modified (UTC) |
|---|---|---|
| run-20261005-061647-697.log | 142 KB | 2026-10-05 04:30:52 |
| run-20261005-052959-696.log | 226 KB | 2026-10-05 04:06:39 |
| run-20261005-044026-695.log | 177 KB | 2026-10-05 03:19:52 |
| run-20261005-034927-694.log | 254 KB | 2026-10-05 02:30:18 |
| run-20261005-030059-693.log | 312 KB | 2026-10-05 01:39:20 |
| run-20261004-233305-692.log | 1010 KB | 2026-10-05 00:50:51 |
| run-20261004-230052-691.log | 130 KB | 2026-10-04 21:22:57 |
| run-20261004-221607-690.log | 258 KB | 2026-10-04 20:50:44 |
| run-20261004-210204-689.log | 310 KB | 2026-10-04 20:05:59 |
| run-20261004-184239-688.log | 505 KB | 2026-10-04 18:51:56 |
| run-20261004-172538-687.log | 486 KB | 2026-10-04 16:32:32 |
| run-20261004-165539-686.log | 142 KB | 2026-10-04 15:15:29 |
| run-20261004-161338-685.log | 176 KB | 2026-10-04 14:45:31 |
| run-20261004-160330-684.log | 153 B | 2026-10-04 14:03:31 |
| run-20261004-155322-683.log | 153 B | 2026-10-04 13:53:23 |
| run-20261004-154314-682.log | 190 B | 2026-10-04 13:43:15 |
| run-20261004-153306-681.log | 153 B | 2026-10-04 13:33:07 |
| run-20261004-152258-680.log | 153 B | 2026-10-04 13:22:58 |
| run-20261004-151250-679.log | 190 B | 2026-10-04 13:12:50 |
| run-20261004-150241-678.log | 153 B | 2026-10-04 13:02:42 |
| run-20261004-145233-677.log | 153 B | 2026-10-04 12:52:34 |
| run-20261004-144225-676.log | 190 B | 2026-10-04 12:42:26 |
| run-20261004-143217-675.log | 153 B | 2026-10-04 12:32:17 |
| run-20261004-142209-674.log | 153 B | 2026-10-04 12:22:09 |
| run-20261004-141201-673.log | 153 B | 2026-10-04 12:12:01 |
| run-20261004-140152-672.log | 153 B | 2026-10-04 12:01:53 |
| run-20261004-135144-671.log | 153 B | 2026-10-04 11:51:44 |
| run-20261004-134136-670.log | 153 B | 2026-10-04 11:41:36 |
| run-20261004-133127-669.log | 153 B | 2026-10-04 11:31:27 |
| run-20261004-132119-668.log | 153 B | 2026-10-04 11:21:19 |
| run-20261004-131110-667.log | 153 B | 2026-10-04 11:11:10 |
| run-20261004-130101-666.log | 190 B | 2026-10-04 11:01:02 |
| run-20261004-125053-665.log | 153 B | 2026-10-04 10:50:54 |
| run-20261004-124045-664.log | 153 B | 2026-10-04 10:40:45 |
| run-20261004-123036-663.log | 153 B | 2026-10-04 10:30:37 |
| run-20261004-122028-662.log | 153 B | 2026-10-04 10:20:29 |
| run-20261004-121020-661.log | 153 B | 2026-10-04 10:10:20 |
| run-20261004-120011-660.log | 153 B | 2026-10-04 10:00:11 |
| run-20261004-115003-659.log | 153 B | 2026-10-04 09:50:03 |
| run-20261004-113954-658.log | 153 B | 2026-10-04 09:39:55 |
| run-20261004-112946-657.log | 153 B | 2026-10-04 09:29:47 |
| run-20261004-111938-656.log | 153 B | 2026-10-04 09:19:39 |
| run-20261004-110930-655.log | 153 B | 2026-10-04 09:09:31 |
| run-20261004-105922-654.log | 153 B | 2026-10-04 08:59:23 |
| run-20261004-104914-653.log | 153 B | 2026-10-04 08:49:14 |
| run-20261004-103906-652.log | 153 B | 2026-10-04 08:39:06 |
| run-20261004-102858-651.log | 153 B | 2026-10-04 08:28:58 |
| run-20261004-101850-650.log | 153 B | 2026-10-04 08:18:50 |
| run-20261004-100839-649.log | 153 B | 2026-10-04 08:08:42 |
| run-20261004-095831-648.log | 190 B | 2026-10-04 07:58:32 |
Tail — run-20261005-061647-697.log (last 200 lines)
+assert_lacks "err_ip → does not borrow the drift path's public-IP diagnosis" \
+ "egress is a public IP but differs from DNS" "$NP"
+
# an error must never be mistaken for drift, and vice versa
run_cron err_no_a
assert_lacks "check error is not reported as DRIFT" "DRIFT gladex.de" "$(alerts)"
run_cron drift_pub
> build · mimo-v2.6-flash-free
[0m← [0mEdit repo/tests/test_ip_drift_cron.sh
Index: repo/tests/test_ip_drift_cron.sh
===================================================================
--- repo/tests/test_ip_drift_cron.sh
+++ repo/tests/test_ip_drift_cron.sh
@@ -719,8 +719,22 @@
"M7": (' DIAGNOSIS="checker failed without a classified cause (exit ${EXIT_CODE})."\n'
' ACTION="run \'ip-drift-check --format json\' by hand and read its output." ;;',
' DIAGNOSIS="the DNS A record for ${DOMAIN} is GONE (query completed, no answer)."\n'
' ACTION="inspect the zone with \'pdns-api.py records\'; do NOT rewrite DNS from here." ;;'),
+ # M8 — a NAMED arm made to quote a DIFFERENT named arm (queue item (f6),
+ # 2026-10-05): `no_a_records` — the A record vanished — diagnoses a dead
+ # lookup and orders the resolver chain re-checked. Nothing in this suite
+ # asked what a named arm says before the block above it: the same plant
+ # scored 131/0 here and 471/0 in the reader suite (both measured), so the
+ # sentence was asserted nowhere in the tree. The default arm's twin, M7,
+ # guards the fourth branch; this one guards the three the case statement
+ # names outright.
+ "M8": (' no_a_records)\n'
+ ' DIAGNOSIS="the DNS A record for ${DOMAIN} is GONE (query completed, no answer)."\n'
+ ' ACTION="inspect the zone with \'pdns-api.py records\'; do NOT rewrite DNS from here." ;;',
+ ' no_a_records)\n'
+ ' DIAGNOSIS="the DNS lookup never completed (dig missing, unreachable or timed out)."\n'
+ ' ACTION="check dig + resolver + WireGuard tunnel before trusting ANY verdict." ;;'),
}
needle, repl = M[os.environ["MUT_ID"]]
path = os.environ["MUT_TARGET"]
src = open(path).read()
> build · mimo-v2.6-flash-free
[0m← [0mEdit repo/tests/test_ip_drift_cron.sh
Index: repo/tests/test_ip_drift_cron.sh
===================================================================
--- repo/tests/test_ip_drift_cron.sh
+++ repo/tests/test_ip_drift_cron.sh
@@ -771,8 +771,9 @@
run_mutation M4 'drift → exit 1'
run_mutation M5 'err_no_a → exit 3'
run_mutation M6 'unparseable → diagnosis says the address did not parse'
run_mutation M7 'err_int → default arm diagnosis names an unclassified cause'
+ run_mutation M8 'err_no_a → named arm diagnosis says the A record is gone'
fi
# =============================================================================
echo ""
> build · mimo-v2.6-flash-free
[0m$ [0mcd /data/repo && bash -n tests/test_ip_drift_cron.sh && echo 'parse OK' && bash tests/test_ip_drift_cron.sh > /tmp/opencode/f6/after.out 2>&1; echo "rc=$?"; tail -2 /tmp/opencode/f6/after.out; echo '--- plant re-run (after block) ---'; IPDRIFT_CRON=/tmp/opencode/f6/ip-drift-cron IPDRIFT_NO_LIVE=1 bash tests/test_ip_drift_cron.sh > /tmp/opencode/f6/q1-after.out 2>&1; echo "rc=$?"; tail -2 /tmp/opencode/f6/q1-after.out; grep '^ FAIL' /tmp/opencode/f6/q1-after.out
parse OK
rc=0
=== Results: 161 passed, 0 failed ===
--- plant re-run (after block) ---
rc=1
=== Results: 154 passed, 4 failed ===
FAIL - err_no_a → named arm diagnosis says the A record is gone (missing: DIAGNOSIS: the DNS A record for gladex.de is GONE (query completed, no answer).)
FAIL - err_no_a → named arm action points at the zone, never at a rewrite (missing: ACTION: inspect the zone with 'pdns-api.py records'; do NOT rewrite DNS from here.)
FAIL - err_no_a → does not borrow the dead-lookup diagnosis (unexpected: the DNS lookup never completed)
FAIL - err_no_a → does not borrow the dead-lookup action (unexpected: check dig + resolver + WireGuard tunnel before trusting ANY verdict.)
> build · mimo-v2.6-flash-free
[0m$ [0mcd /data/repo && bash tests/test_ip_drift_cron.sh --mutations > /tmp/opencode/f6/mut.out 2>&1; echo "rc=$?"; tail -2 /tmp/opencode/f6/mut.out; echo '--- M8 lines ---'; grep -n 'M8' /tmp/opencode/f6/mut.out; echo '--- control ---'; grep -n 'control:' /tmp/opencode/f6/mut.out
rc=0
=== Results: 194 passed, 0 failed ===
--- M8 lines ---
205: ok - M8 applied (its needle occurs exactly once)
206: ok - M8 is syntactically valid (red comes from behaviour, not a parse error)
207: ok - M8 → suite goes red (rc=1), the mutant did not survive
208: ok - M8 → wrong verdict lands on the intended assertion: 'err_no_a → named arm diagnosis says the A record is gone'
--- control ---
176: ok - control: unmutated cron → suite green (rc=0)
> build · mimo-v2.6-flash-free
Block lands **161/0** (was 134), battery **194/0** (was 163), M8 caught on its own assertion. Now neighbour suites + byte-identity checks:
[0m$ [0mcd /data/repo && echo "worktree: $(md5sum tools/ip-drift-cron | cut -d' ' -f1)"; echo "HEAD : $(git show HEAD:tools/ip-drift-cron | md5sum | cut -d' ' -f1)"; echo "worktree: $(md5sum tools/system-status | cut -d' ' -f1)"; echo "HEAD : $(git show HEAD:tools/system-status | md5sum | cut -d' ' -f1)"; git status --porcelain
worktree: 2816126cb8bad48aabd03be621c2a60c
HEAD : 2816126cb8bad48aabd03be621c2a60c
worktree: 50e704f809f7507cfd0b03a5b35f8e04
HEAD : 50e704f809f7507cfd0b03a5b35f8e04
M tests/test_ip_drift_cron.sh
[0m$ [0mcd /data/repo && bash tests/test_system_status_ip_drift.sh 2>&1 | tail -2; php tests/test_ip_drift_safety.php 2>&1 | tail -2; bash tests/test_detached_children.sh 2>&1 | tail -2
=== Results: 471 passed, 0 failed ===
=== Results: 16 passed, 0 failed ===
Command exceeded timeout of 120000 ms. Retry with a larger timeout if the command is expected to take longer.
> build · mimo-v2.6-flash-free
[0m$ [0mcd /data/repo && sed -n '1053,1058p' tools/REGISTRY.md; echo '=== 1100-1115 ==='; sed -n '1100,1115p' tools/REGISTRY.md; echo '=== detached in bg ===';
**Tests**:
- `bash tests/test_ip_drift_cron.sh` → **134 assertions** (measured 2026-10-05; was **114**
before `[0.4.210]` added the default-arm block, **103** before `[0.4.208]` added the
`drift_unparse` block, **93** before `[0.4.206]` added section **7, the channel sweep**),
hermetic: a scenario-driven
=== 1100-1115 ===
measured before it was written: the same plant left this suite **114/0 green** while
`tests/test_system_status_ip_drift.sh` reddened **21** for it, i.e. the sentence was
guarded repo-wide and *this* file — the one a maintainer opens when the writer
misbehaves — was the blind spot. Mutant **M7** swaps the pair and must redden here.
- **Pre-fix replay**: `IPDRIFT_CRON=<pre-fix blob> IPDRIFT_CRON_LEGACY=1 bash
tests/test_ip_drift_cron.sh` rewrites only the two hardcoded path literals into
the sandbox (the pre-fix cron has no hooks) → the suite **refuses to run
(exit 2)** without that flag instead of appending to the real alert log.
- `bash tests/test_ip_drift_cron.sh --mutations` — mutation battery, **M1–M7** (each
precondition-asserted to occur exactly once, must RUN and give the wrong
verdict, with a clean control); **163 assertions** measured 2026-10-05.
- `php tests/test_ip_drift_safety.php` (16 assertions — the older static guard: no
DNS-write call in executable code, CGNAT classified, alert written, `--help` green)
**Status**: Built ✅ (hardened 2026-09-24: CHECK-ERROR alerting + hooks + first behavioural suite)
---
=== detached in bg ===
[0m$ [0mcd /data/repo && bash tests/test_detached_children.sh > /tmp/opencode/f6/detached.out 2>&1; echo "rc=$?"; tail -2 /tmp/opencode/f6/detached.out
rc=0
detached-children suite: 155 passed, 0 failed
=================================
> build · mimo-v2.6-flash-free
[0m← [0mEdit repo/tools/REGISTRY.md
Index: repo/tools/REGISTRY.md
===================================================================
--- repo/tools/REGISTRY.md
+++ repo/tools/REGISTRY.md
@@ -1051,10 +1051,11 @@
- `IPDRIFT_LOG_DIR` — where logs/alerts go (the suite points it at a sandbox)
- `IPDRIFT_TOOLS_DIR` — tools directory (implies the default checker path)
**Tests**:
-- `bash tests/test_ip_drift_cron.sh` → **134 assertions** (measured 2026-10-05; was **114**
- before `[0.4.210]` added the default-arm block, **103** before `[0.4.208]` added the
+- `bash tests/test_ip_drift_cron.sh` → **161 assertions** (measured 2026-10-05; was **134**
+ before `[0.4.211]` added the named-arms block, **114** before `[0.4.210]` added the
+ default-arm block, **103** before `[0.4.208]` added the
`drift_unparse` block, **93** before `[0.4.206]` added section **7, the channel sweep**),
hermetic: a scenario-driven
`ip-drift-check` **stub** (`ok` / `drift_pub` / `drift_cgnat` / `err_no_a` /
`err_dns` / `err_ip` / `err_int` / `garbage` / `empty` / `rc2` / `rc99`) plus
> build · mimo-v2.6-flash-free
[0m← [0mEdit repo/tools/REGISTRY.md
Index: repo/tools/REGISTRY.md
===================================================================
--- repo/tools/REGISTRY.md
+++ repo/tools/REGISTRY.md
@@ -1101,8 +1101,24 @@
measured before it was written: the same plant left this suite **114/0 green** while
`tests/test_system_status_ip_drift.sh` reddened **21** for it, i.e. the sentence was
guarded repo-wide and *this* file — the one a maintainer opens when the writer
misbehaves — was the blind spot. Mutant **M7** swaps the pair and must redden here.
+- **The three named arms' own `DIAGNOSIS`/`ACTION`** (`[0.4.211]`, queue item **(f6)** —
+ the half of the same gap `[0.4.210]` left): closing the default arm asserted only what
+ the *default* says, so every reading of a named arm in this file was an ABSENCE, and
+ the queue's measurement held here too — `grep -rn -F "check dig + resolver + WireGuard
+ tunnel" tests/` → **0**, `"check egress; no drift verdict"` → **0**, `"inspect the
+ zone with"` → 1 and that hit is M7's *replacement* string, never an assertion. Planted
+ before the block existed: a writer whose `no_a_records` arm quotes the
+ `dns_lookup_failed` pair (A record gone, log says "the DNS lookup never completed")
+ scored **131/0** here and **471/0** in `tests/test_system_status_ip_drift.sh` — both
+ green over a sentence naming the wrong fault, the defect class `[0.4.208]` closed on
+ the drift side. Per arm: its own `DIAGNOSIS` and its own `ACTION` **present**, the
+ other two named pairs **absent**, and the three drift-path diagnoses absent (they
+ belong to a branch a check-error run never reaches). The same plant after the block:
+ **154 / 4 failed**, every red inside it. Mutant **M8** swaps `no_a_records`' pair for
+ `dns_lookup_failed`'s and must redden on `err_no_a → named arm diagnosis says the A
+ record is gone`.
- **Pre-fix replay**: `IPDRIFT_CRON=<pre-fix blob> IPDRIFT_CRON_LEGACY=1 bash
tests/test_ip_drift_cron.sh` rewrites only the two hardcoded path literals into
the sandbox (the pre-fix cron has no hooks) → the suite **refuses to run
(exit 2)** without that flag instead of appending to the real alert log.
[0m← [0mEdit repo/tools/REGISTRY.md
Index: repo/tools/REGISTRY.md
===================================================================
--- repo/tools/REGISTRY.md
+++ repo/tools/REGISTRY.md
@@ -1121,11 +1121,11 @@
- **Pre-fix replay**: `IPDRIFT_CRON=<pre-fix blob> IPDRIFT_CRON_LEGACY=1 bash
tests/test_ip_drift_cron.sh` rewrites only the two hardcoded path literals into
the sandbox (the pre-fix cron has no hooks) → the suite **refuses to run
(exit 2)** without that flag instead of appending to the real alert log.
-- `bash tests/test_ip_drift_cron.sh --mutations` — mutation battery, **M1–M7** (each
+- `bash tests/test_ip_drift_cron.sh --mutations` — mutation battery, **M1–M8** (each
precondition-asserted to occur exactly once, must RUN and give the wrong
- verdict, with a clean control); **163 assertions** measured 2026-10-05.
+ verdict, with a clean control); **194 assertions** measured 2026-10-05.
- `php tests/test_ip_drift_safety.php` (16 assertions — the older static guard: no
DNS-write call in executable code, CGNAT classified, alert written, `--help` green)
**Status**: Built ✅ (hardened 2026-09-24: CHECK-ERROR alerting + hooks + first behavioural suite)
> build · mimo-v2.6-flash-free
Select another run log from the list above. Only files matching run-YYYYMMDD-HHMMSS-N.log are readable.
App log tail — prod-8001.log (last 60 lines)
[Mon Oct 5 06:15:57 2026] 127.0.0.1:48608 Accepted [Mon Oct 5 06:15:57 2026] 127.0.0.1:48608 Closing [Mon Oct 5 06:15:57 2026] 127.0.0.1:48622 Accepted [Mon Oct 5 06:15:57 2026] 127.0.0.1:48622 Closing [Mon Oct 5 06:15:57 2026] 127.0.0.1:48626 Accepted [Mon Oct 5 06:15:57 2026] 127.0.0.1:48626 Closing [Mon Oct 5 06:15:57 2026] 127.0.0.1:48630 Accepted [Mon Oct 5 06:15:58 2026] 127.0.0.1:48630 Closing [Mon Oct 5 06:15:58 2026] 127.0.0.1:48640 Accepted [Mon Oct 5 06:15:58 2026] 127.0.0.1:48640 Closing [Mon Oct 5 06:17:41 2026] 127.0.0.1:60898 Accepted [Mon Oct 5 06:17:41 2026] 127.0.0.1:60898 Closing [Mon Oct 5 06:29:28 2026] 127.0.0.1:40382 Accepted [Mon Oct 5 06:29:28 2026] 127.0.0.1:40382 Closing [Mon Oct 5 06:29:29 2026] 127.0.0.1:40398 Accepted [Mon Oct 5 06:29:29 2026] 127.0.0.1:40398 Closing [Mon Oct 5 06:30:00 2026] 127.0.0.1:60618 Accepted [Mon Oct 5 06:30:00 2026] 127.0.0.1:60618 Closing [Mon Oct 5 06:30:09 2026] 127.0.0.1:39590 Accepted [Mon Oct 5 06:30:09 2026] 127.0.0.1:39590 Closing [Mon Oct 5 06:30:09 2026] 127.0.0.1:39594 Accepted [Mon Oct 5 06:30:09 2026] 127.0.0.1:39594 Closing [Mon Oct 5 06:30:09 2026] 127.0.0.1:39596 Accepted [Mon Oct 5 06:30:09 2026] 127.0.0.1:39596 Closing [Mon Oct 5 06:30:09 2026] 127.0.0.1:39608 Accepted [Mon Oct 5 06:30:09 2026] 127.0.0.1:39608 Closing [Mon Oct 5 06:30:09 2026] 127.0.0.1:39614 Accepted [Mon Oct 5 06:30:09 2026] 127.0.0.1:39614 Closing [Mon Oct 5 06:30:09 2026] 127.0.0.1:39624 Accepted [Mon Oct 5 06:30:09 2026] 127.0.0.1:39624 Closing [Mon Oct 5 06:30:09 2026] 127.0.0.1:39638 Accepted [Mon Oct 5 06:30:10 2026] 127.0.0.1:39638 Closing [Mon Oct 5 06:30:10 2026] 127.0.0.1:39650 Accepted [Mon Oct 5 06:30:10 2026] 127.0.0.1:39650 Closing [Mon Oct 5 06:30:10 2026] 127.0.0.1:39666 Accepted [Mon Oct 5 06:30:10 2026] 127.0.0.1:39666 Closing [Mon Oct 5 06:30:10 2026] 127.0.0.1:39672 Accepted [Mon Oct 5 06:30:10 2026] 127.0.0.1:39672 Closing [Mon Oct 5 06:30:57 2026] 127.0.0.1:48454 Accepted [Mon Oct 5 06:30:57 2026] 127.0.0.1:48454 Closing [Mon Oct 5 06:30:57 2026] 127.0.0.1:48458 Accepted [Mon Oct 5 06:30:57 2026] 127.0.0.1:48458 Closing [Mon Oct 5 06:30:57 2026] 127.0.0.1:48462 Accepted [Mon Oct 5 06:30:57 2026] 127.0.0.1:48462 Closing [Mon Oct 5 06:30:57 2026] 127.0.0.1:48468 Accepted [Mon Oct 5 06:30:57 2026] 127.0.0.1:48468 Closing [Mon Oct 5 06:30:57 2026] 127.0.0.1:48478 Accepted [Mon Oct 5 06:30:57 2026] 127.0.0.1:48478 Closing [Mon Oct 5 06:30:57 2026] 127.0.0.1:48492 Accepted [Mon Oct 5 06:30:57 2026] 127.0.0.1:48492 Closing [Mon Oct 5 06:30:57 2026] 127.0.0.1:48500 Accepted [Mon Oct 5 06:30:57 2026] 127.0.0.1:48500 Closing [Mon Oct 5 06:30:57 2026] 127.0.0.1:48508 Accepted [Mon Oct 5 06:30:57 2026] 127.0.0.1:48508 Closing [Mon Oct 5 06:30:57 2026] 127.0.0.1:48518 Accepted [Mon Oct 5 06:30:57 2026] 127.0.0.1:48518 Closing [Mon Oct 5 06:30:57 2026] 127.0.0.1:48526 Accepted [Mon Oct 5 06:30:57 2026] 127.0.0.1:48526 Closing [Mon Oct 5 06:31:19 2026] 127.0.0.1:55004 Accepted
Generated 2026-10-05 04:31:19 UTC · Gladex.de