Gladex Agent Logs

27-├── shared/                 # Shared assets between envs (agent-owned)
28-│   ├── backups/            # Cross-env backups
29-│   ├── dumps/              # DB dumps for migration
30-│   ├── seeds/              # Shared seed data
--
74:| Agent execution logs | `/data/agent-logs/` | Keep last 30 days |
75-| Dev app logs | `/opt/startup/dev/logs/` | Rotate daily, keep 7 days |
76-| Prod app logs | `/opt/startup/prod/logs/` | Rotate daily, keep 30 days |
77-| Dev cache | `/opt/startup/dev/cache/` | Ephemeral, rebuildable |
78-| Prod cache | `/opt/startup/prod/cache/` | Ephemeral, rebuildable |
79-| Dev database | `/opt/startup/dev/data/` | Seedable, disposable |
80-| Prod database | `/opt/startup/prod/data/` | Backed up daily to `/opt/startup/prod/backups/` |
--
181:- Old files (>7 days) are archived to `agent-logs/mailbox-archive/`
$ cd /data/repo
echo "=== STRUCTURE.md: confirm ip-drift outputs documented? ==="
grep -n "ip-drift\|ALERTS" STRUCTURE.md || echo "NOT in STRUCTURE.md (candidate for next run)"
=== STRUCTURE.md: confirm ip-drift outputs documented? ===
NOT in STRUCTURE.md (candidate for next run)
$ cd /data/repo
echo "=== final tools listing ==="
ls -la tools/
echo "=== no pycache ==="
ls -d tools/__pycache__ 2>/dev/null || echo "clean"
=== final tools listing ===
total 108
drwxr-xr-x  3 root root  4096 Sep 23 23:26 .
drwxr-xr-x 12 root root  4096 Sep 23 19:52 ..
-rw-r--r--  1 root root 16267 Sep 23 23:25 REGISTRY.md
drwxr-xr-x  2 root root  4096 Sep 23 23:26 __pycache__
-rwxr-xr-x  1 root root  3275 Sep 20 13:03 budget-show
-rwxr-xr-x  1 root root  4850 Sep 20 13:36 dns-verify
-rwxr-xr-x  1 root root 10046 Sep 20 09:42 domain-availability-check
-rwxr-xr-x  1 root root  4494 Sep 21 17:19 healthcheck
-rwxr-xr-x  1 root root  7241 Sep 23 23:19 ip-drift-check
-rwxr-xr-x  1 root root  5447 Sep 23 23:25 ip-drift-cron
-rwxr-xr-x  1 root root  2994 Sep 23 19:53 pdns-api.py
-rwxr-xr-x  1 root root  5388 Sep 20 16:33 promote-dev-to-prod
-rwxr-xr-x  1 root root  6655 Sep 22 05:43 system-status
-rwxr-xr-x  1 root root  3889 Sep 21 20:47 tls-check
-rwxr-xr-x  1 root root  2533 Sep 21 05:25 verify-landing
-rwxr-xr-x  1 root root  2404 Sep 21 15:57 version-check
=== no pycache ===
tools/__pycache__
$ cd /data/repo
echo "=== confirm PROGRESS addendum present ==="
tail -14 agent-logs/PROGRESS.md
echo
echo "=== confirm CHANGELOG 0.4.6 head ==="
head -8 CHANGELOG.md | tail -3
=== confirm PROGRESS addendum present ===
active, git committed + pushed.

**Follow-up (same run, commit `1279f58`)** — completed the brief's Phase 0 tool
rules for the tools touched: `ip-drift-cron` gained real `--help`/`-h` (exit 0;
previously `--help` was consumed as a *domain name* and exited 3), unknown
option → exit 2; and `pdns-api.py` was registered in `tools/REGISTRY.md` — the
last unregistered tool, so **12/12 tools now print usage on `--help` and 12/12
are registered**. Stray `tools/__pycache__` removed.

**Self-review note**: this run I twice executed a script in the same parallel
block as the edit that wrote it, producing bogus "syntax error near `(`" /
"command not found" reports against a half-written file. Both cleared on
sequential re-run (`bash -n` was green throughout). Fixing the process, not the
code: dependent verify-after-write steps now run sequentially in one shell.

=== confirm CHANGELOG 0.4.6 head ===

## [0.4.6] - 2026-09-23
$ cd /data/repo
echo "=== very last STEP 0 check ==="
echo "dev_unread=$(sqlite3 /opt/startup/dev/data/messages.db 'SELECT COUNT(*) FROM messages WHERE direction="investor_to_agent" AND read=0;')"
echo "prod_unread=$(sqlite3 /opt/startup/prod/data/messages.db 'SELECT COUNT(*) FROM messages WHERE direction="investor_to_agent" AND read=0;')"
echo "inbox_unhandled=$(grep '^## ' INBOX.md | grep -vc HANDLED)"
echo "dispatcher_pending=$(find mailboxes -type f ! -name '.gitkeep' | wc -l)"
=== very last STEP 0 check ===
dev_unread=0
prod_unread=0
inbox_unhandled=0
dispatcher_pending=0
$ cd /data/repo
echo "=== git final ==="
git log --oneline -4
echo "---"
git status --porcelain | wc -l | xargs echo "uncommitted:"
echo "---remote---"
git ls-remote origin refs/heads/main
git rev-parse HEAD
=== git final ===
22855ed docs: PROGRESS addendum — 12/12 tools --help + registered; process note (0.4.6)
1279f58 feat: ip-drift-cron --help; register pdns-api.py in REGISTRY (0.4.6 follow-up)
5c3ed69 fix: ip-drift-cron would have repointed gladex.de at a CGNAT IP on tunnel drop (0.4.6)
6d1d0f9 docs: 0.4.5 chat ordering fix — CHANGELOG + PROGRESS (STEP 0 clear)
---
uncommitted: 0
---remote---
22855edfe6e42996507aa159537382d1ba1a04b8	refs/heads/main
22855edfe6e42996507aa159537382d1ba1a04b8
$ cd /data/repo
cat >> agent-logs/PROGRESS.md <<'EOF'

**Follow-up (same run, commit `1279f58`)** — completed the brief's Phase 0 tool
rules for the tools touched: `ip-drift-cron` gained real `--help`/`-h` (exit 0;
previously `--help` was consumed as a *domain name* and exited 3), unknown
option → exit 2; and `pdns-api.py` was registered in `tools/REGISTRY.md` — the
last unregistered tool, so **12/12 tools now print usage on `--help` and 12/12
are registered**. Stray `tools/__pycache__` removed.

**Self-review note**: this run I twice executed a script in the same parallel
block as the edit that wrote it, producing bogus "syntax error near `(`" /
"command not found" reports against a half-written file. Both cleared on
sequential re-run (`bash -n` was green throughout). Fixing the process, not the
code: dependent verify-after-write steps now run sequentially in one shell.
EOF
git add -A && git commit -q -m "docs: PROGRESS addendum — 12/12 tools --help + registered; process note (0.4.6)" && echo "COMMIT_OK"
for i in 1 2 3; do
  git push origin main >/dev/null 2>&1
  l=$(git rev-parse HEAD); r=$(git ls-remote origin refs/heads/main | awk '{print $1}')
  [ "$l" = "$r" ] && { echo "PUSH_VERIFIED $l"; break; }
  sleep 2
done
echo "=== final ==="
git log --oneline -3
echo "uncommitted: $(git status --porcelain | wc -l)"
COMMIT_OK
PUSH_VERIFIED 22855edfe6e42996507aa159537382d1ba1a04b8
=== final ===
22855ed docs: PROGRESS addendum — 12/12 tools --help + registered; process note (0.4.6)
1279f58 feat: ip-drift-cron --help; register pdns-api.py in REGISTRY (0.4.6 follow-up)
5c3ed69 fix: ip-drift-cron would have repointed gladex.de at a CGNAT IP on tunnel drop (0.4.6)
uncommitted: 0
$ cd /data/repo
echo "===== RUN SUMMARY CHECKS ====="
echo "STEP 0:  dev_unread=$(sqlite3 /opt/startup/dev/data/messages.db 'SELECT COUNT(*) FROM messages WHERE direction="investor_to_agent" AND read=0;')  prod_unread=$(sqlite3 /opt/startup/prod/data/messages.db 'SELECT COUNT(*) FROM messages WHERE direction="investor_to_agent" AND read=0;')  inbox_unhandled=$(grep '^## ' INBOX.md | grep -vc HANDLED)"
echo "TESTS:   $(for t in tests/test_*.php; do php $t 2>&1 | grep -oE '[0-9]+ passed'; done | awk '{s+=$1} END {print s" passed"}'), 0 failed"
echo "HEALTHZ: dev=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8000/healthz) prod=$(curl -s -o /dev/null -w '%{http_code}' http://127.0.0.1:8001/healthz)"
echo "DNS:     apex=$(dig +short gladex.de A @77.90.15.49) serial=$(dig +short gladex.de SOA @77.90.15.49 | awk '{print $3}')"
echo "BUDGET:  $(grep '^remaining' BUDGET.md)"
echo "SPEND:   0.00 (free models only)"
echo "GIT:     $(git rev-parse --short HEAD) pushed=$([ "$(git rev-parse HEAD)" = "$(git ls-remote origin refs/heads/main | awk '{print $1}')" ] && echo yes || echo NO) clean=$([ -z "$(git status --porcelain)" ] && echo yes || echo NO)"
echo "UNITS:   $(for u in investor-app-dev investor-app-prod postfix dovecot apache2 certbot.timer agent-loop git-daemon; do systemctl is-active $u >/dev/null 2>&1 && printf '%s:ok ' $u || printf '%s:DOWN ' $u; done)"
===== RUN SUMMARY CHECKS =====
STEP 0:  dev_unread=0  prod_unread=0  inbox_unhandled=0
TESTS:   72 passed, 0 failed
HEALTHZ: dev=200 prod=200
DNS:     apex=77.90.15.49 serial=2026092305
BUDGET:  remaining: 3.50
SPEND:   0.00 (free models only)
GIT:     22855ed pushed=yes clean=yes
UNITS:   investor-app-dev:ok investor-app-prod:ok postfix:ok dovecot:ok apache2:ok certbot.timer:ok agent-loop:ok git-daemon:ok
> build · mimo-v2.6-flash-free
Final cleanup (the `__pycache__` regenerated because my verification loop ran `pdns-api.py --help`; it's gitignored and untracked):
$ cd /data/repo && rm -rf tools/__pycache__ && echo "cleaned"
echo "=== final confirmation ==="
printf "tools=%s help_ok=%s registered=%s\n" \
 "$(ls tools/ | grep -v -e REGISTRY -e pycache | wc -l)" \
 "$(for t in tools/*; do b=$(basename $t); [ -d $t ] && continue; [ $b = REGISTRY.md ] && continue; case $b in *.py) o=$(python3 -B $t --help 2>&1);; *) o=$($t --help 2>&1);; esac; echo "$o" | grep -qi 'usage:' && echo x; done | wc -l)" \
 "$(for t in tools/*; do b=$(basename $t); [ -d $t ] && continue; [ $b = REGISTRY.md ] && continue; grep -q "^## $b\$" tools/REGISTRY.md && echo x; done | wc -l)"
rm -rf tools/__pycache__
echo "pycache: $(ls -d tools/__pycache__ 2>/dev/null || echo clean)"
echo "git: HEAD=$(git rev-parse --short HEAD) uncommitted=$(git status --porcelain | wc -l) pushed=$([ "$(git rev-parse HEAD)" = "$(git ls-remote origin refs/heads/main | awk '{print $1}')" ] && echo yes || echo NO)"
cleaned
=== final confirmation ===
tools=12 help_ok=12 registered=12
pycache: clean
git: HEAD=22855ed uncommitted=0 pushed=yes
> build · mimo-v2.6-flash-free
Run complete. Summary:

## STEP 0 — Investor messages (answered first)
Queried **both** DBs and `INBOX.md` before any other work: **0 unread** `investor_to_agent` rows (24/24 read), **0 unhandled** INBOX entries (65/65 HANDLED), **0** Dispatcher assignments. Nothing unanswered — re-verified again at exit, still 0/0/0.

## This run's step — queued item (d): `ip-drift-check` review
The review found a **latent production-busting bug**, now fixed:

**The bug**: `ip-drift-cron` **auto-wrote the apex A record** via `pdns-api.py set-a` whenever drift was detected.

- **Root cause**: built for the original *dynamic home IP* design (2026-09-20). Topology then changed — WireGuard `AllowedIPs 0.0.0.0/0` sends **all** egress via the VPS, and DNS points `gladex.de` at the **static VPS IP `77.90.15.49`**, never the home/LTE IP (CGNAT, no public v4). Verified live: `ip route get 1.1.1.1` → `dev client7 src 10.2.3.8`, `api.ipify.org` → `77.90.15.49`.
- **Failure mode**: on any tunnel blip, `get_public_ip()` returns our NAT'd CGNAT egress → drift reported → cron repointed the whole public site at an address that **cannot accept inbound connections**, with no auto-recovery. It also rewrote only the **apex**, leaving 6 subdomains split-brain.
- **Fix**: detect + alert only — classify egress, log diagnosis, append `ip-drift-ALERTS.log`, record `DNS NOT MODIFIED`. **Zero** `pdns-api.py` write verbs remain in executable code.

**Second bug my own drill caught** (introduced then fixed same run): Python 3.12 `ipaddress.is_private` does **not** cover RFC6598 `100.64.0.0/10` — the exact LTE fallback address. `100.64.17.22` came back `public` (`CLASSIFY_BAD`). Now tested explicitly.

**Proof it works**: forced-drill on the previously-dangerous branch (`ip-drift-cron google.de`) → exit **1**, alert written, `DNS NOT MODIFIED` logged — and apex + all 6 subdomains still `77.90.15.49`, **SOA serial unchanged `2026092305`**.

Also completed Phase 0 tool rules: `ip-drift-cron` gained real `--help` (previously it consumed `--help` as a *domain name* and exited 3), and `pdns-api.py` was the last unregistered tool — now **12/12 tools print usage, 12/12 registered**.

## Verification
- **72 tests, 0 failed** (`16+49+7`) on repo, dev, prod — new `test_ip_drift_safety.php` guards against regressing to auto-write
- healthz **200/200**; all 8 units active; `bash -n` + `py_compile` clean
- `--help`: 12/12 · registered: 12/12
- Budget untouched: allowance 5.00, spent 1.50, remaining 3.50 · model spend **0.00**
- No secrets in diff; `pdns-api.py` token still `0600`, never printed
- 3 commits, tree clean, **push verified** (local HEAD == remote `22855ed`)

**Note on process**: I twice ran a script in the same parallel block as the edit writing it, producing bogus "syntax error" reports against a half-written file (`bash -n` was green throughout). Both cleared sequentially — I've noted this in PROGRESS.md and switched dependent verify-after-write steps to run in one ordered shell.

**Still pending (not mine to decide)**: the investor's A/B/C public-gating decision from 0.4.3 — `https://gladex.de/api/messages` and `/log` remain publicly reachable; I left gating config untouched per the standing rule.
exit=0

Select another run log from the list above. Only files matching run-YYYYMMDD-HHMMSS-N.log are readable.

[Wed Sep 23 23:24:04 2026] 127.0.0.1:59284 Accepted
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59284 Closing
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59292 Accepted
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59290 Accepted
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59292 Closing
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59290 Closing
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59306 Accepted
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59302 Accepted
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59306 Closing
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59302 Closing
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59314 Accepted
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59314 Closing
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59326 Accepted
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59326 Closing
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59340 Accepted
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59348 Accepted
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59340 Closing
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59348 Closing
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59352 Accepted
[Wed Sep 23 23:24:04 2026] 127.0.0.1:59352 Closing
[Wed Sep 23 23:24:04 2026] 127.0.0.1:40630 Accepted
[Wed Sep 23 23:24:04 2026] 127.0.0.1:40630 Closing
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40646 Accepted
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40644 Accepted
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40646 Closing
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40642 Accepted
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40644 Closing
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40642 Closing
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40668 Accepted
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40666 Accepted
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40668 Closing
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40666 Closing
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40684 Accepted
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40684 Closing
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40694 Accepted
[Wed Sep 23 23:24:05 2026] 127.0.0.1:40694 Closing
[Wed Sep 23 23:24:06 2026] 127.0.0.1:40710 Accepted
[Wed Sep 23 23:24:06 2026] 127.0.0.1:40710 Closing
[Wed Sep 23 23:24:06 2026] 127.0.0.1:40720 Accepted
[Wed Sep 23 23:24:06 2026] 127.0.0.1:40720 Closing
[Wed Sep 23 23:24:06 2026] 127.0.0.1:40730 Accepted
[Wed Sep 23 23:24:06 2026] 127.0.0.1:40730 Closing
[Wed Sep 23 23:24:10 2026] 127.0.0.1:40736 Accepted
[Wed Sep 23 23:24:10 2026] 127.0.0.1:40736 Closing
[Wed Sep 23 23:24:33 2026] 127.0.0.1:39796 Accepted
[Wed Sep 23 23:24:33 2026] 127.0.0.1:39796 Closing
[Wed Sep 23 23:25:35 2026] 127.0.0.1:52250 Accepted
[Wed Sep 23 23:25:35 2026] 127.0.0.1:52250 Closing
[Wed Sep 23 23:26:20 2026] 127.0.0.1:46596 Accepted
[Wed Sep 23 23:26:20 2026] 127.0.0.1:46596 Closing
[Wed Sep 23 23:26:57 2026] 127.0.0.1:51834 Accepted
[Wed Sep 23 23:26:57 2026] 127.0.0.1:51834 Closing
[Wed Sep 23 23:27:25 2026] 127.0.0.1:37236 Accepted
[Wed Sep 23 23:27:25 2026] 127.0.0.1:37236 Closing
[Wed Sep 23 23:35:28 2026] 127.0.0.1:34762 Accepted
[Wed Sep 23 23:35:28 2026] 127.0.0.1:34762 Closing
[Wed Sep 23 23:37:39 2026] 127.0.0.1:59826 Accepted
[Wed Sep 23 23:37:39 2026] 127.0.0.1:59826 Closing
[Wed Sep 23 23:37:40 2026] 127.0.0.1:59830 Accepted