Gladex Agent Logs

Agent run logs & app logs · env: prod · LAN-only investor surface

Overview
Run logs1163 files, 66 MB
Latest run logrun-20261005-061647-697.log
Log directory/data/agent-logs
App log directory/opt/startup/prod/logs
Run logs (newest first, last 50)
FileSizeModified (UTC)
run-20261005-061647-697.log 142 KB 2026-10-05 04:30:52
run-20261005-052959-696.log 226 KB 2026-10-05 04:06:39
run-20261005-044026-695.log 177 KB 2026-10-05 03:19:52
run-20261005-034927-694.log 254 KB 2026-10-05 02:30:18
run-20261005-030059-693.log 312 KB 2026-10-05 01:39:20
run-20261004-233305-692.log 1010 KB 2026-10-05 00:50:51
run-20261004-230052-691.log 130 KB 2026-10-04 21:22:57
run-20261004-221607-690.log 258 KB 2026-10-04 20:50:44
run-20261004-210204-689.log 310 KB 2026-10-04 20:05:59
run-20261004-184239-688.log 505 KB 2026-10-04 18:51:56
run-20261004-172538-687.log 486 KB 2026-10-04 16:32:32
run-20261004-165539-686.log 142 KB 2026-10-04 15:15:29
run-20261004-161338-685.log 176 KB 2026-10-04 14:45:31
run-20261004-160330-684.log 153 B 2026-10-04 14:03:31
run-20261004-155322-683.log 153 B 2026-10-04 13:53:23
run-20261004-154314-682.log 190 B 2026-10-04 13:43:15
run-20261004-153306-681.log 153 B 2026-10-04 13:33:07
run-20261004-152258-680.log 153 B 2026-10-04 13:22:58
run-20261004-151250-679.log 190 B 2026-10-04 13:12:50
run-20261004-150241-678.log 153 B 2026-10-04 13:02:42
run-20261004-145233-677.log 153 B 2026-10-04 12:52:34
run-20261004-144225-676.log 190 B 2026-10-04 12:42:26
run-20261004-143217-675.log 153 B 2026-10-04 12:32:17
run-20261004-142209-674.log 153 B 2026-10-04 12:22:09
run-20261004-141201-673.log 153 B 2026-10-04 12:12:01
run-20261004-140152-672.log 153 B 2026-10-04 12:01:53
run-20261004-135144-671.log 153 B 2026-10-04 11:51:44
run-20261004-134136-670.log 153 B 2026-10-04 11:41:36
run-20261004-133127-669.log 153 B 2026-10-04 11:31:27
run-20261004-132119-668.log 153 B 2026-10-04 11:21:19
run-20261004-131110-667.log 153 B 2026-10-04 11:11:10
run-20261004-130101-666.log 190 B 2026-10-04 11:01:02
run-20261004-125053-665.log 153 B 2026-10-04 10:50:54
run-20261004-124045-664.log 153 B 2026-10-04 10:40:45
run-20261004-123036-663.log 153 B 2026-10-04 10:30:37
run-20261004-122028-662.log 153 B 2026-10-04 10:20:29
run-20261004-121020-661.log 153 B 2026-10-04 10:10:20
run-20261004-120011-660.log 153 B 2026-10-04 10:00:11
run-20261004-115003-659.log 153 B 2026-10-04 09:50:03
run-20261004-113954-658.log 153 B 2026-10-04 09:39:55
run-20261004-112946-657.log 153 B 2026-10-04 09:29:47
run-20261004-111938-656.log 153 B 2026-10-04 09:19:39
run-20261004-110930-655.log 153 B 2026-10-04 09:09:31
run-20261004-105922-654.log 153 B 2026-10-04 08:59:23
run-20261004-104914-653.log 153 B 2026-10-04 08:49:14
run-20261004-103906-652.log 153 B 2026-10-04 08:39:06
run-20261004-102858-651.log 153 B 2026-10-04 08:28:58
run-20261004-101850-650.log 153 B 2026-10-04 08:18:50
run-20261004-100839-649.log 153 B 2026-10-04 08:08:42
run-20261004-095831-648.log 190 B 2026-10-04 07:58:32
Tail — run-20261005-061647-697.log (last 200 lines)
+assert_lacks "err_ip → does not borrow the drift path's public-IP diagnosis" \
+    "egress is a public IP but differs from DNS" "$NP"
+
 # an error must never be mistaken for drift, and vice versa
 run_cron err_no_a
 assert_lacks "check error is not reported as DRIFT" "DRIFT gladex.de" "$(alerts)"
 run_cron drift_pub

> build · mimo-v2.6-flash-free
← Edit repo/tests/test_ip_drift_cron.sh
Index: repo/tests/test_ip_drift_cron.sh
===================================================================
--- repo/tests/test_ip_drift_cron.sh
+++ repo/tests/test_ip_drift_cron.sh
@@ -719,8 +719,22 @@
     "M7": ('        DIAGNOSIS="checker failed without a classified cause (exit ${EXIT_CODE})."\n'
            '        ACTION="run \'ip-drift-check --format json\' by hand and read its output." ;;',
            '        DIAGNOSIS="the DNS A record for ${DOMAIN} is GONE (query completed, no answer)."\n'
            '        ACTION="inspect the zone with \'pdns-api.py records\'; do NOT rewrite DNS from here." ;;'),
+    # M8 — a NAMED arm made to quote a DIFFERENT named arm (queue item (f6),
+    # 2026-10-05): `no_a_records` — the A record vanished — diagnoses a dead
+    # lookup and orders the resolver chain re-checked. Nothing in this suite
+    # asked what a named arm says before the block above it: the same plant
+    # scored 131/0 here and 471/0 in the reader suite (both measured), so the
+    # sentence was asserted nowhere in the tree. The default arm's twin, M7,
+    # guards the fourth branch; this one guards the three the case statement
+    # names outright.
+    "M8": ('    no_a_records)\n'
+           '        DIAGNOSIS="the DNS A record for ${DOMAIN} is GONE (query completed, no answer)."\n'
+           '        ACTION="inspect the zone with \'pdns-api.py records\'; do NOT rewrite DNS from here." ;;',
+           '    no_a_records)\n'
+           '        DIAGNOSIS="the DNS lookup never completed (dig missing, unreachable or timed out)."\n'
+           '        ACTION="check dig + resolver + WireGuard tunnel before trusting ANY verdict." ;;'),
 }
 needle, repl = M[os.environ["MUT_ID"]]
 path = os.environ["MUT_TARGET"]
 src = open(path).read()

> build · mimo-v2.6-flash-free
← Edit repo/tests/test_ip_drift_cron.sh
Index: repo/tests/test_ip_drift_cron.sh
===================================================================
--- repo/tests/test_ip_drift_cron.sh
+++ repo/tests/test_ip_drift_cron.sh
@@ -771,8 +771,9 @@
     run_mutation M4 'drift → exit 1'
     run_mutation M5 'err_no_a → exit 3'
     run_mutation M6 'unparseable → diagnosis says the address did not parse'
     run_mutation M7 'err_int → default arm diagnosis names an unclassified cause'
+    run_mutation M8 'err_no_a → named arm diagnosis says the A record is gone'
 fi
 
 # =============================================================================
 echo ""

> build · mimo-v2.6-flash-free
$ cd /data/repo && bash -n tests/test_ip_drift_cron.sh && echo 'parse OK' && bash tests/test_ip_drift_cron.sh > /tmp/opencode/f6/after.out 2>&1; echo "rc=$?"; tail -2 /tmp/opencode/f6/after.out; echo '--- plant re-run (after block) ---'; IPDRIFT_CRON=/tmp/opencode/f6/ip-drift-cron IPDRIFT_NO_LIVE=1 bash tests/test_ip_drift_cron.sh > /tmp/opencode/f6/q1-after.out 2>&1; echo "rc=$?"; tail -2 /tmp/opencode/f6/q1-after.out; grep '^  FAIL' /tmp/opencode/f6/q1-after.out
parse OK
rc=0

=== Results: 161 passed, 0 failed ===
--- plant re-run (after block) ---
rc=1

=== Results: 154 passed, 4 failed ===
  FAIL - err_no_a → named arm diagnosis says the A record is gone (missing: DIAGNOSIS: the DNS A record for gladex.de is GONE (query completed, no answer).)
  FAIL - err_no_a → named arm action points at the zone, never at a rewrite (missing: ACTION: inspect the zone with 'pdns-api.py records'; do NOT rewrite DNS from here.)
  FAIL - err_no_a → does not borrow the dead-lookup diagnosis (unexpected: the DNS lookup never completed)
  FAIL - err_no_a → does not borrow the dead-lookup action (unexpected: check dig + resolver + WireGuard tunnel before trusting ANY verdict.)
> build · mimo-v2.6-flash-free
$ cd /data/repo && bash tests/test_ip_drift_cron.sh --mutations > /tmp/opencode/f6/mut.out 2>&1; echo "rc=$?"; tail -2 /tmp/opencode/f6/mut.out; echo '--- M8 lines ---'; grep -n 'M8' /tmp/opencode/f6/mut.out; echo '--- control ---'; grep -n 'control:' /tmp/opencode/f6/mut.out
rc=0

=== Results: 194 passed, 0 failed ===
--- M8 lines ---
205:  ok   - M8 applied (its needle occurs exactly once)
206:  ok   - M8 is syntactically valid (red comes from behaviour, not a parse error)
207:  ok   - M8 → suite goes red (rc=1), the mutant did not survive
208:  ok   - M8 → wrong verdict lands on the intended assertion: 'err_no_a → named arm diagnosis says the A record is gone'
--- control ---
176:  ok   - control: unmutated cron → suite green (rc=0)
> build · mimo-v2.6-flash-free
Block lands **161/0** (was 134), battery **194/0** (was 163), M8 caught on its own assertion. Now neighbour suites + byte-identity checks:
$ cd /data/repo && echo "worktree: $(md5sum tools/ip-drift-cron | cut -d' ' -f1)"; echo "HEAD    : $(git show HEAD:tools/ip-drift-cron | md5sum | cut -d' ' -f1)"; echo "worktree: $(md5sum tools/system-status | cut -d' ' -f1)"; echo "HEAD    : $(git show HEAD:tools/system-status | md5sum | cut -d' ' -f1)"; git status --porcelain
worktree: 2816126cb8bad48aabd03be621c2a60c
HEAD    : 2816126cb8bad48aabd03be621c2a60c
worktree: 50e704f809f7507cfd0b03a5b35f8e04
HEAD    : 50e704f809f7507cfd0b03a5b35f8e04
 M tests/test_ip_drift_cron.sh
$ cd /data/repo && bash tests/test_system_status_ip_drift.sh 2>&1 | tail -2; php tests/test_ip_drift_safety.php 2>&1 | tail -2; bash tests/test_detached_children.sh 2>&1 | tail -2
=== Results: 471 passed, 0 failed ===

=== Results: 16 passed, 0 failed ===


Command exceeded timeout of 120000 ms. Retry with a larger timeout if the command is expected to take longer.
> build · mimo-v2.6-flash-free
$ cd /data/repo && sed -n '1053,1058p' tools/REGISTRY.md; echo '=== 1100-1115 ==='; sed -n '1100,1115p' tools/REGISTRY.md; echo '=== detached in bg ==='; 
**Tests**:
- `bash tests/test_ip_drift_cron.sh` → **134 assertions** (measured 2026-10-05; was **114**
  before `[0.4.210]` added the default-arm block, **103** before `[0.4.208]` added the
  `drift_unparse` block, **93** before `[0.4.206]` added section **7, the channel sweep**),
  hermetic: a scenario-driven
=== 1100-1115 ===
  measured before it was written: the same plant left this suite **114/0 green** while
  `tests/test_system_status_ip_drift.sh` reddened **21** for it, i.e. the sentence was
  guarded repo-wide and *this* file — the one a maintainer opens when the writer
  misbehaves — was the blind spot. Mutant **M7** swaps the pair and must redden here.
- **Pre-fix replay**: `IPDRIFT_CRON=<pre-fix blob> IPDRIFT_CRON_LEGACY=1 bash
  tests/test_ip_drift_cron.sh` rewrites only the two hardcoded path literals into
  the sandbox (the pre-fix cron has no hooks) → the suite **refuses to run
  (exit 2)** without that flag instead of appending to the real alert log.
- `bash tests/test_ip_drift_cron.sh --mutations` — mutation battery, **M1–M7** (each
  precondition-asserted to occur exactly once, must RUN and give the wrong
  verdict, with a clean control); **163 assertions** measured 2026-10-05.
- `php tests/test_ip_drift_safety.php` (16 assertions — the older static guard: no
  DNS-write call in executable code, CGNAT classified, alert written, `--help` green)

**Status**: Built ✅ (hardened 2026-09-24: CHECK-ERROR alerting + hooks + first behavioural suite)
---
=== detached in bg ===
$ cd /data/repo && bash tests/test_detached_children.sh > /tmp/opencode/f6/detached.out 2>&1; echo "rc=$?"; tail -2 /tmp/opencode/f6/detached.out
rc=0
detached-children suite: 155 passed, 0 failed
=================================
> build · mimo-v2.6-flash-free
← Edit repo/tools/REGISTRY.md
Index: repo/tools/REGISTRY.md
===================================================================
--- repo/tools/REGISTRY.md
+++ repo/tools/REGISTRY.md
@@ -1051,10 +1051,11 @@
 - `IPDRIFT_LOG_DIR` — where logs/alerts go (the suite points it at a sandbox)
 - `IPDRIFT_TOOLS_DIR` — tools directory (implies the default checker path)
 
 **Tests**:
-- `bash tests/test_ip_drift_cron.sh` → **134 assertions** (measured 2026-10-05; was **114**
-  before `[0.4.210]` added the default-arm block, **103** before `[0.4.208]` added the
+- `bash tests/test_ip_drift_cron.sh` → **161 assertions** (measured 2026-10-05; was **134**
+  before `[0.4.211]` added the named-arms block, **114** before `[0.4.210]` added the
+  default-arm block, **103** before `[0.4.208]` added the
   `drift_unparse` block, **93** before `[0.4.206]` added section **7, the channel sweep**),
   hermetic: a scenario-driven
   `ip-drift-check` **stub** (`ok` / `drift_pub` / `drift_cgnat` / `err_no_a` /
   `err_dns` / `err_ip` / `err_int` / `garbage` / `empty` / `rc2` / `rc99`) plus

> build · mimo-v2.6-flash-free
← Edit repo/tools/REGISTRY.md
Index: repo/tools/REGISTRY.md
===================================================================
--- repo/tools/REGISTRY.md
+++ repo/tools/REGISTRY.md
@@ -1101,8 +1101,24 @@
   measured before it was written: the same plant left this suite **114/0 green** while
   `tests/test_system_status_ip_drift.sh` reddened **21** for it, i.e. the sentence was
   guarded repo-wide and *this* file — the one a maintainer opens when the writer
   misbehaves — was the blind spot. Mutant **M7** swaps the pair and must redden here.
+- **The three named arms' own `DIAGNOSIS`/`ACTION`** (`[0.4.211]`, queue item **(f6)** —
+  the half of the same gap `[0.4.210]` left): closing the default arm asserted only what
+  the *default* says, so every reading of a named arm in this file was an ABSENCE, and
+  the queue's measurement held here too — `grep -rn -F "check dig + resolver + WireGuard
+  tunnel" tests/` → **0**, `"check egress; no drift verdict"` → **0**, `"inspect the
+  zone with"` → 1 and that hit is M7's *replacement* string, never an assertion. Planted
+  before the block existed: a writer whose `no_a_records` arm quotes the
+  `dns_lookup_failed` pair (A record gone, log says "the DNS lookup never completed")
+  scored **131/0** here and **471/0** in `tests/test_system_status_ip_drift.sh` — both
+  green over a sentence naming the wrong fault, the defect class `[0.4.208]` closed on
+  the drift side. Per arm: its own `DIAGNOSIS` and its own `ACTION` **present**, the
+  other two named pairs **absent**, and the three drift-path diagnoses absent (they
+  belong to a branch a check-error run never reaches). The same plant after the block:
+  **154 / 4 failed**, every red inside it. Mutant **M8** swaps `no_a_records`' pair for
+  `dns_lookup_failed`'s and must redden on `err_no_a → named arm diagnosis says the A
+  record is gone`.
 - **Pre-fix replay**: `IPDRIFT_CRON=<pre-fix blob> IPDRIFT_CRON_LEGACY=1 bash
   tests/test_ip_drift_cron.sh` rewrites only the two hardcoded path literals into
   the sandbox (the pre-fix cron has no hooks) → the suite **refuses to run
   (exit 2)** without that flag instead of appending to the real alert log.

← Edit repo/tools/REGISTRY.md
Index: repo/tools/REGISTRY.md
===================================================================
--- repo/tools/REGISTRY.md
+++ repo/tools/REGISTRY.md
@@ -1121,11 +1121,11 @@
 - **Pre-fix replay**: `IPDRIFT_CRON=<pre-fix blob> IPDRIFT_CRON_LEGACY=1 bash
   tests/test_ip_drift_cron.sh` rewrites only the two hardcoded path literals into
   the sandbox (the pre-fix cron has no hooks) → the suite **refuses to run
   (exit 2)** without that flag instead of appending to the real alert log.
-- `bash tests/test_ip_drift_cron.sh --mutations` — mutation battery, **M1–M7** (each
+- `bash tests/test_ip_drift_cron.sh --mutations` — mutation battery, **M1–M8** (each
   precondition-asserted to occur exactly once, must RUN and give the wrong
-  verdict, with a clean control); **163 assertions** measured 2026-10-05.
+  verdict, with a clean control); **194 assertions** measured 2026-10-05.
 - `php tests/test_ip_drift_safety.php` (16 assertions — the older static guard: no
   DNS-write call in executable code, CGNAT classified, alert written, `--help` green)
 
 **Status**: Built ✅ (hardened 2026-09-24: CHECK-ERROR alerting + hooks + first behavioural suite)

> build · mimo-v2.6-flash-free

Select another run log from the list above. Only files matching run-YYYYMMDD-HHMMSS-N.log are readable.

App log tail — prod-8001.log (last 60 lines)
[Mon Oct  5 06:15:57 2026] 127.0.0.1:48608 Accepted
[Mon Oct  5 06:15:57 2026] 127.0.0.1:48608 Closing
[Mon Oct  5 06:15:57 2026] 127.0.0.1:48622 Accepted
[Mon Oct  5 06:15:57 2026] 127.0.0.1:48622 Closing
[Mon Oct  5 06:15:57 2026] 127.0.0.1:48626 Accepted
[Mon Oct  5 06:15:57 2026] 127.0.0.1:48626 Closing
[Mon Oct  5 06:15:57 2026] 127.0.0.1:48630 Accepted
[Mon Oct  5 06:15:58 2026] 127.0.0.1:48630 Closing
[Mon Oct  5 06:15:58 2026] 127.0.0.1:48640 Accepted
[Mon Oct  5 06:15:58 2026] 127.0.0.1:48640 Closing
[Mon Oct  5 06:17:41 2026] 127.0.0.1:60898 Accepted
[Mon Oct  5 06:17:41 2026] 127.0.0.1:60898 Closing
[Mon Oct  5 06:29:28 2026] 127.0.0.1:40382 Accepted
[Mon Oct  5 06:29:28 2026] 127.0.0.1:40382 Closing
[Mon Oct  5 06:29:29 2026] 127.0.0.1:40398 Accepted
[Mon Oct  5 06:29:29 2026] 127.0.0.1:40398 Closing
[Mon Oct  5 06:30:00 2026] 127.0.0.1:60618 Accepted
[Mon Oct  5 06:30:00 2026] 127.0.0.1:60618 Closing
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39590 Accepted
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39590 Closing
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39594 Accepted
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39594 Closing
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39596 Accepted
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39596 Closing
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39608 Accepted
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39608 Closing
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39614 Accepted
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39614 Closing
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39624 Accepted
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39624 Closing
[Mon Oct  5 06:30:09 2026] 127.0.0.1:39638 Accepted
[Mon Oct  5 06:30:10 2026] 127.0.0.1:39638 Closing
[Mon Oct  5 06:30:10 2026] 127.0.0.1:39650 Accepted
[Mon Oct  5 06:30:10 2026] 127.0.0.1:39650 Closing
[Mon Oct  5 06:30:10 2026] 127.0.0.1:39666 Accepted
[Mon Oct  5 06:30:10 2026] 127.0.0.1:39666 Closing
[Mon Oct  5 06:30:10 2026] 127.0.0.1:39672 Accepted
[Mon Oct  5 06:30:10 2026] 127.0.0.1:39672 Closing
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48454 Accepted
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48454 Closing
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48458 Accepted
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48458 Closing
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48462 Accepted
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48462 Closing
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48468 Accepted
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48468 Closing
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48478 Accepted
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48478 Closing
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48492 Accepted
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48492 Closing
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48500 Accepted
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48500 Closing
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48508 Accepted
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48508 Closing
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48518 Accepted
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48518 Closing
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48526 Accepted
[Mon Oct  5 06:30:57 2026] 127.0.0.1:48526 Closing
[Mon Oct  5 06:31:19 2026] 127.0.0.1:55004 Accepted

Generated 2026-10-05 04:31:19 UTC · Gladex.de