Gladex Agent Logs

Agent run logs & app logs · env: prod · LAN-only investor surface

Overview
Run logs1628 files, 118.7 MB
Latest run logrun-20261015-070843-1162.log
Log directory/data/agent-logs
App log directory/opt/startup/prod/logs
Run logs (newest first, last 50)
FileSizeModified (UTC)
run-20261015-070843-1162.log 452 KB 2026-10-15 05:36:50
run-20261015-054521-1161.log 416 KB 2026-10-15 04:58:33
run-20261015-033336-1160.log 684 KB 2026-10-15 03:35:11
run-20261015-020624-1159.log 212 KB 2026-10-15 01:23:26
run-20261015-015612-1158.log 154 B 2026-10-14 23:56:13
run-20261015-014600-1157.log 191 B 2026-10-14 23:46:01
run-20261015-013549-1156.log 191 B 2026-10-14 23:35:50
run-20261015-012537-1155.log 154 B 2026-10-14 23:25:38
run-20261015-011525-1154.log 191 B 2026-10-14 23:15:26
run-20261015-010513-1153.log 154 B 2026-10-14 23:05:14
run-20261015-005502-1152.log 154 B 2026-10-14 22:55:03
run-20261015-004450-1151.log 154 B 2026-10-14 22:44:51
run-20261015-003439-1150.log 154 B 2026-10-14 22:34:40
run-20261015-002427-1149.log 154 B 2026-10-14 22:24:28
run-20261015-001416-1148.log 154 B 2026-10-14 22:14:17
run-20261015-000405-1147.log 191 B 2026-10-14 22:04:06
run-20261014-235353-1146.log 191 B 2026-10-14 21:53:54
run-20261014-234342-1145.log 191 B 2026-10-14 21:43:43
run-20261014-233330-1144.log 154 B 2026-10-14 21:33:31
run-20261014-232318-1143.log 154 B 2026-10-14 21:23:19
run-20261014-231307-1142.log 154 B 2026-10-14 21:13:08
run-20261014-230255-1141.log 154 B 2026-10-14 21:02:56
run-20261014-225244-1140.log 154 B 2026-10-14 20:52:45
run-20261014-222900-1139.log 142 KB 2026-10-14 20:42:33
run-20261014-212813-1138.log 571 KB 2026-10-14 20:18:50
run-20261014-201639-1137.log 370 KB 2026-10-14 19:18:02
run-20261014-174154-1136.log 835 KB 2026-10-14 18:06:28
run-20261014-170319-1135.log 320 KB 2026-10-14 15:31:41
run-20261014-161418-1134.log 270 KB 2026-10-14 14:53:09
run-20261014-150444-1133.log 323 KB 2026-10-14 14:04:07
run-20261014-133301-1132.log 340 KB 2026-10-14 12:54:34
run-20261014-124552-1131.log 194 KB 2026-10-14 11:22:49
run-20261014-103913-1130.log 525 KB 2026-10-14 10:35:42
run-20261014-082946-1129.log 630 KB 2026-10-14 08:29:02
run-20261014-061713-1128.log 454 KB 2026-10-14 06:19:36
run-20261014-052042-1127.log 96 KB 2026-10-14 04:07:02
run-20261014-044319-1126.log 200 KB 2026-10-14 03:10:30
run-20261014-032502-1125.log 531 KB 2026-10-14 02:33:08
run-20261014-024815-1124.log 214 KB 2026-10-14 01:14:52
run-20261014-020537-1123.log 283 KB 2026-10-14 00:38:05
run-20261014-015526-1122.log 154 B 2026-10-13 23:55:27
run-20261014-014514-1121.log 191 B 2026-10-13 23:45:15
run-20261014-013503-1120.log 191 B 2026-10-13 23:35:04
run-20261014-012452-1119.log 154 B 2026-10-13 23:24:53
run-20261014-011441-1118.log 154 B 2026-10-13 23:14:42
run-20261014-010430-1117.log 154 B 2026-10-13 23:04:31
run-20261014-005419-1116.log 154 B 2026-10-13 22:54:20
run-20261014-004409-1115.log 154 B 2026-10-13 22:44:09
run-20261014-003358-1114.log 154 B 2026-10-13 22:33:58
run-20261014-002347-1113.log 154 B 2026-10-13 22:23:48
Tail — run-20261015-070843-1162.log (last 200 lines)
+        # tab or newline can survive into the JSON row), anything unreadable
+        # -> `unreadable`.
+        BS_PARSE=$(printf '%s' "$BS_OUT" | python3 -c '
+import json, sys
+rc = int(sys.argv[1])
+try:
+    d = json.load(sys.stdin)
+    if not isinstance(d, dict):
+        raise ValueError("not an object")
+    if rc in (0, 1):
+        n = (d.get("counts") or {}).get("stale")
+        if type(n) is int and n >= 0:
+            print("stale\t%d" % n)
+        else:
+            print("unreadable\t")
+    else:
+        e = d.get("errors") or []
+        msg = " ".join(e[0].split()) if (e and isinstance(e[0], str)) else ""
+        print("refusal\t%s" % (msg or ("baseline-stale refused (exit %d)" % rc)))
+except Exception:
+    print("unreadable\t")
+' "$BS_RC" 2>/dev/null || echo "unreadable")
+        BS_KIND="${BS_PARSE%%$'\t'*}"
+        BS_ARG="${BS_PARSE#*$'\t'}"
+        case "$BS_KIND" in
+            stale)
+                if [[ "$BS_ARG" == "0" ]]; then
+                    BS_STATUS="ok"
+                    BS_DETAIL="baseline in step: 0 transcript(s) above their recorded value"
+                else
+                    BS_STATUS="warning"
+                    BS_DETAIL="baseline stale: $BS_ARG transcript(s) above their recorded value - run tools/coverage-guard --record (green only)"
+                fi
+                ;;
+            refusal)
+                BS_STATUS="warning"
+                BS_DETAIL="cannot verify: $BS_ARG"
+                ;;
+            *)
+                BS_STATUS="warning"
+                BS_DETAIL="cannot verify: baseline-stale output unreadable (exit $BS_RC)"
+                ;;
+        esac
+    fi
+fi
+# Belt to the braces above: whatever the branch built, this row's detail may
+# never carry a quote (hand-built JSON + the renderer's own grep) or any
+# control character (a stored error string is not this row's to format).
+BS_DETAIL=$(printf '%s' "$BS_DETAIL" | sed 's/["\\]//g' | LC_ALL=C tr -d '\000-\037' | cut -c1-600)
+STATUS_ITEMS+=("{\"name\":\"baseline-stale\",\"status\":\"$BS_STATUS\",\"detail\":\"$BS_DETAIL\"}")
+if [[ "$BS_STATUS" == "error" ]]; then
+    # Never set today - every branch above is ok or warning by the contract
+    # this comment pins - and kept so a future branch that WANTS to be an
+    # error cannot land as a warning by omission.
+    ((ERRORS++)) || true
+fi
+
 if [[ "$FORMAT" == "json" ]]; then
     ITEMS=$(IFS=,; echo "${STATUS_ITEMS[*]}")
     echo "{\"overall\":\"$([ $ERRORS -eq 0 ] && echo ok || echo error)\",\"errors\":$ERRORS,\"checks\":[$ITEMS]}"
 else

> build · mimo-v2.6-flash-free
Now the help block needs the `GLADEX_BASELINE_STALE_BASELINE` hook documented (the row honors it):
← Edit repo/tools/system-status
Index: repo/tools/system-status
===================================================================
--- repo/tools/system-status
+++ repo/tools/system-status
@@ -526,13 +526,17 @@
             echo "                          this script); point it at an absent path to make this"
             echo "                          row say cannot verify without opening any live input"
             echo "  GLADEX_BASELINE_STALE_TIMEOUT  seconds the child is given (default 20; a"
             echo "                          non-numeric value falls back to it)"
-            echo "  The child's own baseline is set HERE to this row's reported repo"
-            echo "                          (GLADEX_REPO_DIR), so a sandbox suite that points that"
-            echo "                          hook at a throwaway tree never reads the LIVE baseline;"
-            echo "                          an exported GLADEX_BASELINE_STALE_COVERAGE reaches the"
-            echo "                          child unchanged, which is the hermetic fixture path."
+            echo "  GLADEX_BASELINE_STALE_BASELINE the baseline this row asks about (default:"
+            echo "                          the reported repo's own agent-logs/coverage-baseline.json)"
+            echo "  By default the child's own baseline is set HERE to this row's reported repo"
+            echo "                          (GLADEX_REPO_DIR) BEFORE any child runs, so a sandbox"
+            echo "                          suite that points that hook at a throwaway tree gets the"
+            echo "                          absent-baseline refusal and never reads the LIVE"
+            echo "                          baseline; an exported GLADEX_BASELINE_STALE_COVERAGE"
+            echo "                          reaches the child unchanged, which is the hermetic"
+            echo "                          fixture path."
             echo ""
             echo "Exit codes:"
             echo "  0  All systems healthy"
             echo "  1  One or more systems unhealthy"

> build · mimo-v2.6-flash-free
$ cd /data/repo && bash -n tools/system-status && echo "syntax ok" && ./tools/system-status | tail -30; echo "rc=$?"
syntax ok
  api-changelog             [OK] 200
  api-stats                 [OK] 200
  page-info                 [OK] 200
  page-stats                [OK] 200
  page-team                 [OK] 200
  cloud                     [OK] nextcloud 200 installed=true; immich 200 pong; 5/5 containers up
  DNS:gladex.de             [OK] 77.90.15.49 on 1.1.1.1+8.8.8.8
  DNS:dev.gladex.de         [OK] 77.90.15.49 on 1.1.1.1+8.8.8.8
  DNS:git.gladex.de         [OK] 77.90.15.49 on 1.1.1.1+8.8.8.8
  DNS:info.gladex.de        [OK] 77.90.15.49 on 1.1.1.1+8.8.8.8
  DNS:stats.gladex.de       [OK] 77.90.15.49 on 1.1.1.1+8.8.8.8
  DNS:team.gladex.de        [OK] 77.90.15.49 on 1.1.1.1+8.8.8.8
  DNS:log.gladex.de         [OK] 77.90.15.49 on 1.1.1.1+8.8.8.8
  MX:gladex.de              [OK] 10 gladex.de. on 1.1.1.1+8.8.8.8
  SOA:gladex.de             [WARN] serial=2026092602 consistent on 1.1.1.1+8.8.8.8; mname=placeholder (NEEDS-INVESTOR open)
  ip-drift                  [OK] [2026-10-11T01:00:02Z] OK: No drift (egress matches DNS A record) (4d4h old)
  tls-cert-expiry           [OK] 68d left
  tls-cert-expiry-photos    [OK] 70d left
  investor-messages         [OK] 0 unread dev=0 prod=0
  investor-duty             [OK] owed=0 unread=0 unreplied=0 open=0
  git-tree                  [WARN] 1 uncommitted changes
  red-watch                 [WARN] red, failed=6, since 2026-10-13T12:17:20Z, 2m old; FAIL A3 tree clean AND in sync with origin/main (kind=is key=git) ;; FAIL A12 system-status verdict - json channel unreadable (kind=is key=sysverdict) ;; FAIL A15 git-tree reports a clean tree (kind=contains key=sys) ;; FAIL A30 no unit is left in systemctl --failed (kind=contains key=sys) ;; FAIL A31 source-sync-check exits 0 (kind=rc key=sync) ;; FAIL A32 the committed tree reached dev AND prod 
  queue-source              [OK] one queue: [0.4.354] pointer-only, 111 item line(s) frozen across 308 section(s), 203 PROGRESS bullet(s), 2 path token(s)
  go-tests                  [OK] passing (worktree)
  go-compile                [OK] 47 module file(s) compile clean (1.89s) (commit 6b8988c)
  promote-gates             [WARN] not promotable since 2026-10-13T20:09:38Z (1d 9h ago): verdict REFUSED: verdict VERDICT-20261013T192305Z-work-attribution.md is STALE (promoted source [app/src/php, examples/workflows] changed 72516s after it \u2014 re-review required) | dev-sync REFUSED: dev tree differs from the repo in 1 file(s): src/php/download.php (differs) \u2014 this promotion copies REPO -> prod, so that work would be silently drop
  deploy-path               [OK] ledger OK - 48 file(s) across dev+prod attested (/data/agent-logs/promote-ledger.jsonl; source: baseline+promote)
  baseline-stale            [OK] baseline in step: 0 transcript(s) above their recorded value

Overall: 1 CHECK(S) FAILED
rc=0
> build · mimo-v2.6-flash-free
$ cd /data/repo && ./tools/system-status --format json > /tmp/opencode/ss-new.json; echo "rc=$?"; python3 -c "
import json
d=json.load(open('/tmp/opencode/ss-new.json'))
print('overall',d['overall'],'errors',d['errors'])
print([ (c['name'],c['status']) for c in d['checks'] if c['status']!='ok' ])
print('last', d['checks'][-1])"
git show HEAD:tools/system-status > /tmp/opencode/ss-head.sh && bash /tmp/opencode/ss-head.sh --format json > /tmp/opencode/ss-old.json; echo "oldrc=$?"; python3 -c "
import json
d=json.load(open('/tmp/opencode/ss-old.json'))
print('OLD overall',d['overall'],'errors',d['errors'],'checks',len(d['checks']))
print([ (c['name'],c['status']) for c in d['checks'] if c['status']!='ok' ])"
rc=1
overall error errors 1
[('failed-units', 'error'), ('log-truncation', 'warning'), ('loop-outcomes', 'warning'), ('SOA:gladex.de', 'warning'), ('git-tree', 'warning'), ('red-watch', 'warning'), ('promote-gates', 'warning')]
last {'name': 'baseline-stale', 'status': 'ok', 'detail': 'baseline in step: 0 transcript(s) above their recorded value'}
oldrc=1
OLD overall error errors 1 checks 42
[('failed-units', 'error'), ('log-truncation', 'warning'), ('loop-outcomes', 'warning'), ('SOA:gladex.de', 'warning'), ('investor-duty', 'warning'), ('git-tree', 'warning'), ('red-watch', 'warning'), ('queue-source', 'warning'), ('go-tests', 'warning'), ('go-compile', 'warning'), ('promote-gates', 'warning'), ('deploy-path', 'warning')]
> build · mimo-v2.6-flash-free
Live row is `ok`, `errors` unchanged (standing `failed-units` only). Now reading the template suite:
→ Read repo/tests/test_system_status_red_watch.sh 
> build · mimo-v2.6-flash-free
Template understood. Checking for suites that pin `system-status` help/output shapes, and timing a stubbed run:
$ cd /data/repo && grep -rln "system-status" tests/ | head -30; echo ---; grep -rn "help.*lines\|wc -l.*help\|42 checks\|checks\":" tests/*.sh tests/*.php 2>/dev/null | grep -i "system-status\|status" | head
tests/test_system_status_promote_gates.sh
tests/test_system_status_mx_soa.sh
tests/test_system_status_go_tests.sh
tests/test_tls_check.sh
tests/test_system_status_dig_single_point.sh
tests/test_system_status_ip_drift.sh
tests/test_system_status_red_watch.sh
tests/test_shift_deploy.sh
tests/test_healthcheck.sh
tests/test_commit_gate.sh
tests/test_promote_json.sh
tests/test_system_status_failed_units.sh
tests/test_system_status_unread.sh
tests/test_go_compile_drift_verify.sh
tests/test_system_status_staged.sh
tests/test_system_status_investor_duty.sh
tests/test_system_status_cloud.sh
tests/test_ip_drift_cron.sh
tests/test_registry_coverage.sh
tests/test_domain_availability.sh
tests/test_system_status_go_compile.sh
tests/test_dns_verify.sh
tests/test_dig_tools_transport.sh
tests/test_system_status_tls_expiry.sh
tests/test_deploy_ledger.sh
tests/test_red_watch.sh
tests/test_system_status_mx_soa_transport.sh
tests/test_monitor_cascade.sh
tests/test_queue_source.sh
tests/test_system_status_dns.sh
---
tests/test_system_status_dig_single_point.sh:20:# multi-line `echo "..."` help string (lines 94-102) and in eleven comment
tests/test_system_status_go_tests.sh:558:    || bad  "11b: the --help copy states exactly 4 cases (got $(n_lines "$H_MAP"))"
tests/test_system_status_ip_drift.sh:923:BAD_DOC=$(printf '{"checks":[{"name":"ip-drift","status":"ok","detail":"x\377y"}]}')
$ cd /data/repo && S=$(mktemp -d /tmp/opencode/timing.XXXX) && mkdir -p $S/bin && printf '#!/bin/sh\necho active\n' > $S/bin/systemctl && printf '#!/bin/sh\nprintf 200\n' > $S/bin/curl && printf '#!/bin/sh\necho "notAfter=Dec 22 12:00:00 2026 GMT"\n' > $S/bin/openssl && printf '#!/bin/sh\nexit 0\n' > $S/bin/go && chmod +x $S/bin/* && git init -q $S/repo && time env PATH="$S/bin:$PATH" GLADEX_REPO_DIR="$S/repo" GLADEX_DEV_DB=$S/no.db GLADEX_PROD_DB=$S/no.db GLADEX_REPO_LINT_BIN=$S/bin/nope GLADEX_PROMOTE_BIN=$S/bin/nope GLADEX_CLOUD_DOCKER_BIN=$S/bin/nope GLADEX_REDWATCH_STATE=$S/no.state ./tools/system-status --format json > $S/out.json 2>/dev/null; echo "rc=$?"; python3 -c "
import json;d=json.load(open('$S/out.json'))
print('errors',d['errors'],'overall',d['overall'])
print([ (c['name'],c['status'],c['detail'][:60]) for c in d['checks'] if c['status']!='ok'][:12])
print([c for c in d['checks'] if c['name']=='baseline-stale'])"; rm -rf $S
real	0m2.374s
user	0m0.648s
sys	0m0.864s
rc=0
errors 0 overall ok
[('failed-units', 'warning', "cannot verify: unparseable systemctl line 'active'"), ('log-truncation', 'warning', 'cannot verify: 0 identity-*.md file(s) and none with a wrapp'), ('loop-outcomes', 'warning', 'cannot verify: sandboxed run (no GLADEX_RUNLOG_DIR, no syste'), ('cloud', 'warning', 'nextcloud 200, status.php unreadable; immich 200, ping unrea'), ('SOA:gladex.de', 'warning', 'serial=2026092602 consistent on 1.1.1.1+8.8.8.8; mname=place'), ('ip-drift', 'warning', 'cannot verify: sandboxed run (no GLADEX_IPDRIFT_LOG_DIR, no '), ('investor-messages', 'warning', 'unreadable dev=? prod=?'), ('investor-duty', 'warning', 'cannot verify: inbox-status could not read its inputs (exit '), ('red-watch', 'warning', 'cannot verify: no red-watch state at /tmp/opencode/timing.7n'), ('queue-source', 'warning', 'cannot verify: /tmp/opencode/timing.7njr/repo/CHANGELOG.md i'), ('go-tests', 'warning', 'cannot verify: no go.mod at app/src/go (worktree)'), ('go-compile', 'warning', 'cannot verify: repo-lint not found or not executable (commit')]
[{'name': 'baseline-stale', 'status': 'warning', 'detail': 'cannot verify: no baseline at /tmp/opencode/timing.7njr/repo/agent-logs/coverage-baseline.json (coverage-guard --record has never run here)'}]
> build · mimo-v2.6-flash-free

Select another run log from the list above. Only files matching run-YYYYMMDD-HHMMSS-N.log are readable.

App log tail — prod-8001.log (last 60 lines)
[Thu Oct 15 07:32:39 2026] 127.0.0.1:55874 Accepted
[Thu Oct 15 07:32:39 2026] 127.0.0.1:55874 Closing
[Thu Oct 15 07:32:39 2026] 127.0.0.1:55888 Accepted
[Thu Oct 15 07:32:39 2026] 127.0.0.1:55888 Closing
[Thu Oct 15 07:32:39 2026] 127.0.0.1:55898 Accepted
[Thu Oct 15 07:32:39 2026] 127.0.0.1:55898 Closing
[Thu Oct 15 07:32:39 2026] 127.0.0.1:55912 Accepted
[Thu Oct 15 07:32:39 2026] 127.0.0.1:55912 Closing
[Thu Oct 15 07:32:39 2026] 127.0.0.1:55922 Accepted
[Thu Oct 15 07:32:39 2026] 127.0.0.1:55922 Closing
[Thu Oct 15 07:32:39 2026] 127.0.0.1:55928 Accepted
[Thu Oct 15 07:32:40 2026] 127.0.0.1:55928 Closing
[Thu Oct 15 07:32:40 2026] 127.0.0.1:55932 Accepted
[Thu Oct 15 07:32:40 2026] 127.0.0.1:55932 Closing
[Thu Oct 15 07:32:40 2026] 127.0.0.1:55940 Accepted
[Thu Oct 15 07:32:41 2026] 127.0.0.1:55940 Closing
[Thu Oct 15 07:32:41 2026] 127.0.0.1:55952 Accepted
[Thu Oct 15 07:32:41 2026] 127.0.0.1:55952 Closing
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39298 Accepted
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39298 Closing
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39302 Accepted
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39302 Closing
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39304 Accepted
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39304 Closing
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39310 Accepted
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39310 Closing
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39312 Accepted
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39312 Closing
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39326 Accepted
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39326 Closing
[Thu Oct 15 07:33:56 2026] 127.0.0.1:39338 Accepted
[Thu Oct 15 07:33:57 2026] 127.0.0.1:39338 Closing
[Thu Oct 15 07:33:57 2026] 127.0.0.1:39340 Accepted
[Thu Oct 15 07:33:57 2026] 127.0.0.1:39340 Closing
[Thu Oct 15 07:33:57 2026] 127.0.0.1:39352 Accepted
[Thu Oct 15 07:33:57 2026] 127.0.0.1:39352 Closing
[Thu Oct 15 07:33:57 2026] 127.0.0.1:39368 Accepted
[Thu Oct 15 07:33:57 2026] 127.0.0.1:39368 Closing
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41386 Accepted
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41386 Closing
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41396 Accepted
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41396 Closing
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41410 Accepted
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41410 Closing
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41418 Accepted
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41418 Closing
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41424 Accepted
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41424 Closing
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41430 Accepted
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41430 Closing
[Thu Oct 15 07:34:54 2026] 127.0.0.1:41436 Accepted
[Thu Oct 15 07:34:55 2026] 127.0.0.1:41436 Closing
[Thu Oct 15 07:34:55 2026] 127.0.0.1:42070 Accepted
[Thu Oct 15 07:34:55 2026] 127.0.0.1:42070 Closing
[Thu Oct 15 07:34:55 2026] 127.0.0.1:42080 Accepted
[Thu Oct 15 07:34:55 2026] 127.0.0.1:42080 Closing
[Thu Oct 15 07:34:55 2026] 127.0.0.1:42084 Accepted
[Thu Oct 15 07:34:55 2026] 127.0.0.1:42084 Closing
[Thu Oct 15 07:37:15 2026] 127.0.0.1:58896 Accepted

Other app logs: downloads.log

Generated 2026-10-15 05:37:15 UTC · Gladex.de