Gladex Agent Logs

     while [ ! -s "$pf" ] && [ "$i" -lt 50 ]; do sleep 0.1; i=$((i + 1)); done
     [ -s "$pf" ] || { echo "stub $mode failed to start" >&2; exit 1; }
-    echo "127.0.0.1:$(cat "$pf")"
+    EP_RESULT="127.0.0.1:$(cat "$pf")"
 }
 
-EP_GOOD=$(start_stub good)
-EP_OPEN=$(start_stub openrelay)
-EP_SILENT=$(start_stub nogreeting)
-EP_BADBAD=$(start_stub badgreeting)
+start_stub good;      EP_GOOD="$EP_RESULT"
+start_stub openrelay; EP_OPEN="$EP_RESULT"
+start_stub nogreeting; EP_SILENT="$EP_RESULT"
+start_stub badgreeting; EP_BADBAD="$EP_RESULT"
 REFUSED_PORT=$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()')
 EP_REFUSED="127.0.0.1:$REFUSED_PORT"
 
 jget() { # $1 = json, $2 = python expression over d

$ pgrep -af "stub_smtp" || echo "no orphan stubs"; pgrep -af "test_smtp_relay" || echo "no stuck suite"
601986 python3 /tmp/relay-probe-test.0p9ETc/stub_smtp.py good
601990 python3 /tmp/relay-probe-test.0p9ETc/stub_smtp.py openrelay
601994 python3 /tmp/relay-probe-test.0p9ETc/stub_smtp.py nogreeting
601998 python3 /tmp/relay-probe-test.0p9ETc/stub_smtp.py badgreeting
602235 python3 /tmp/relay-probe-test.xawhQT/stub_smtp.py good
602239 python3 /tmp/relay-probe-test.xawhQT/stub_smtp.py openrelay
602243 python3 /tmp/relay-probe-test.xawhQT/stub_smtp.py nogreeting
602247 python3 /tmp/relay-probe-test.xawhQT/stub_smtp.py badgreeting
602809 /usr/bin/bash -c pgrep -af "stub_smtp" || echo "no orphan stubs"; pgrep -af "test_smtp_relay" || echo "no stuck suite"
602809 /usr/bin/bash -c pgrep -af "stub_smtp" || echo "no orphan stubs"; pgrep -af "test_smtp_relay" || echo "no stuck suite"
> build · mimo-v2.6-flash-free
Confirmed — 8 orphans from runs 1 and 3. Killing them and proving the fix under the previously-hanging capture condition (3×):
$ pkill -f stub_smtp.py; sleep 0.5; pgrep -af stub_smtp.py || echo "orphans cleared"
(no output)
$ for run in 1 2 3; do start=$(date +%s); if out=$(timeout 60 bash /data/repo/tests/test_smtp_relay_probe.sh 2>&1); then rc=0; else rc=$?; fi; dur=$(( $(date +%s) - start )); echo "run $run: rc=$rc dur=${dur}s $(printf '%s' "$out" | grep -oE '[0-9]+ passed, [0-9]+ failed' | tail -1)"; done; echo "--- orphan check ---"; pgrep -af stub_smtp.py || echo "no orphans (cleanup works)"
(no output)
> build · mimo-v2.6-flash-free
Both commands were killed by my own `pkill -f stub_smtp.py` (it matched my shells' command lines too — classic footgun). Verifying state with a self-excluding pattern, then re-running the capture proof:
$ pgrep -af "stub_sm[p]y" || echo "orphans gone (killed by the pkill, including my shells)"
orphans gone (killed by the pkill, including my shells)
> build · mimo-v2.6-flash-free
Orphans cleared. Now the 3× proof under the previously-hanging capture condition:
$ for run in 1 2 3; do start=$(date +%s); if out=$(timeout 60 bash /data/repo/tests/test_smtp_relay_probe.sh 2>&1); then rc=0; else rc=$?; fi; dur=$(( $(date +%s) - start )); echo "run $run: rc=$rc dur=${dur}s $(printf '%s' "$out" | grep -oE '[0-9]+ passed, [0-9]+ failed' | tail -1)"; done; echo "--- orphan check ---"; pgrep -af "stub_sm[p]y" || echo "no orphans (cleanup works)"
run 1: rc=0 dur=6s 38 passed, 0 failed
run 2: rc=0 dur=6s 38 passed, 0 failed
run 3: rc=0 dur=6s 38 passed, 0 failed
--- orphan check ---
no orphans (cleanup works)
> build · mimo-v2.6-flash-free
Fix proven: 3/3 captured runs green in 6s, zero orphans. Now the full suite:
$ cd /data/repo && PASS=0; FAIL=0; FAILED=""; for f in tests/*.sh; do if out=$(bash "$f" 2>&1); then n=$(printf '%s' "$out" | grep -oE '[0-9]+ passed, [0-9]+ failed' | tail -1); echo "PASS $f ($n)"; PASS=$((PASS+1)); else echo "FAIL $f"; printf '%s\n' "$out" | tail -5; FAIL=$((FAIL+1)); FAILED="$FAILED $f"; fi; done; for f in tests/*.php; do if out=$(php "$f" 2>&1); then echo "PASS $f"; PASS=$((PASS+1)); else echo "FAIL $f"; printf '%s\n' "$out" | tail -5; FAIL=$((FAIL+1)); FAILED="$FAILED $f"; fi; done; echo "=== SUITES: $PASS passed, $FAIL failed ==="; [ -n "$FAILED" ] && echo "failed:$FAILED"; exit $FAIL
PASS tests/test_identity_wrapper.sh (26 passed, 0 failed)
PASS tests/test_smtp_relay_probe.sh (38 passed, 0 failed)
PASS tests/test_source_sync_check.sh (26 passed, 0 failed)
PASS tests/test_system_status_mx_soa.sh (27 passed, 0 failed)
PASS tests/test_system_status_unread.sh (20 passed, 0 failed)
PASS tests/test_changelog_api.php
PASS tests/test_chat_a11y.php
PASS tests/test_chat_frontend.php
PASS tests/test_ip_drift_safety.php
PASS tests/test_lan_middleware.php
PASS tests/test_php_busy_timeout.php
PASS tests/test_stats_dbpath.php
PASS tests/test_stats_gocli_path.php
PASS tests/test_templates_gallery.php
PASS tests/test_ts_ordering.php
PASS tests/test_webmail_session_routing.php
=== SUITES: 16 passed, 0 failed ===
> build · mimo-v2.6-flash-free
**16/16 suites green** (137 shell assertions + 11 PHP). Final health check, then PROGRESS + commit:
$ systemctl is-active investor-app-dev investor-app-prod git-daemon apache2 2>/dev/null; curl -s -o /dev/null -w "healthz dev %{http_code}\n" http://127.0.0.1:8000/healthz; curl -s -o /dev/null -w "healthz prod %{http_code}\n" http://127.0.0.1:8001/healthz; certbot certificates 2>/dev/null | grep -E "VALID" | head -2; cd /data/repo && git status --short | head
active
active
active
active
healthz dev 200
healthz prod 200
    Expiry Date: 2026-12-22 19:36:44+00:00 (VALID: 89 days)
 M CHANGELOG.md
 M tools/REGISTRY.md
 M tools/smtp-relay-probe
?? tests/test_smtp_relay_probe.sh
$ /data/repo/tools/system-status --format json 2>&1 | python3 -c "import json,sys; d=json.load(sys.stdin); print('overall:',d['overall'],'errors:',d['errors'],'checks:',len(d['checks'])); [print(' WARN/ERR:',c['name'],c['detail']) for c in d['checks'] if c['status']!='ok']"; echo "exit=$?"
overall: ok errors: 0 checks: 27
 WARN/ERR: SOA:gladex.de serial=2026092305 consistent on 1.1.1.1+8.8.8.8; mname=placeholder (NEEDS-INVESTOR open)
 WARN/ERR: git-tree 4 uncommitted changes
exit=0
> build · mimo-v2.6-flash-free
All green (the 4 uncommitted changes are exactly this run's work). Appending the PROGRESS entry:
$ cat >> /data/repo/agent-logs/PROGRESS.md <<'EOF'

## 2026-09-24T11:15Z — free-builder
- **STEP 0**: 0 unread `investor_to_agent` rows (dev **and** prod DBs); INBOX.md all
  HANDLED (every `## ` heading struck through); `mailboxes/*` empty (4 `.gitkeep`
  only) — **no Dispatcher assignments**; identity Maildirs 0 new. **Nothing owed,
  no DB writes.**
- **Verification sweep — all three operator "Next" items now closed with live proof:**
  - **snakeoil → LE (465/993/submission)**: already replaced — re-verified LIVE this
    run: `openssl s_client` on :993/:465/:587 each presents `CN=gladex.de`,
    issuer Let's Encrypt YE1, notAfter **2026-12-22**; `smtpd_tls_cert_file/key`
    (postfix) + `ssl_cert/ssl_key` (dovecot) both point at
    `/etc/letsencrypt/live/gladex.de`, `grep -ri snakeoil /etc/postfix /etc/dovecot`
    → **0 hits**. Matches STRUCTURE.md:305 / DOMAIN.md:77 — no drift.
  - **MX**: already adopted + propagation-verified (system-status `MX:gladex.de OK
    10 gladex.de. on 1.1.1.1+8.8.8.8`); serial sane (2026092305). No action left.
  - **:25-greeting bug — genuinely FIXED; a false alarm this run was root-caused
    to my own probe methodology.** First re-probe (`bash /dev/tcp` + `head -c 100`)
    returned zero bytes and postfix logged the old signature
    (`lost connection ... commands=0/0`). Investigation: the banner is exactly
    **48 bytes**; coreutils `head -c N` blocks inside `full_read()` until N bytes,
    and after the banner the server correctly waits for a client command (no more
    bytes, no EOF on a live socket) → head holds the received banner WITHOUT
    printing it → apparent hang. Proof: `head -c 48` → instant print, rc 0;
    `head -c 60` → rc 124; **40/40 python probes (≤15 ms)**, `nc`, bash builtin
    `read -t`, python child on the bash fd, and `strace` (read=48 visible) all get
    the `220`. Postfix logs `commands=0/0` for ANY client that hangs up without
    commands — healthy probes log the same lines, so that log signature alone
    never proved a stall.
  - **LIVE relay probe verified after the fix** (the operator's exact ask):
    `smtp-relay-probe` → **exit 0** — :587/:465 deny `554 5.7.1`, full public path
    `77.90.15.49:25` denies `454 4.7.1`, loopback :25 accepted by design
    (mynetworks), `mailq` empty (DATA never sent).
- **Step (CHANGELOG 0.4.26) — `smtp-relay-probe` now ASSERTS the greeting
  (exit 4); gap found while verifying: it only *reported* the banner, so a
  greeting regression on ONE of the four default endpoints would still have
  exited 0 ("ok").**
  - Banner phase isolated: timeout/EOF/non-220 → `banner_ok=false` +
    `greeting_error`, dialogue skipped, deliberately NOT folded into
    `error`/`unreachable`; top-level `greeting_failures`; human `[GREETING FAIL]`;
    **exit 4**, precedence `1 > 4 > 3 > 0` documented in `--help` + REGISTRY.
  - `tests/test_smtp_relay_probe.sh`: **38 assertions**, hermetic (stub SMTP
    servers good/openrelay/silent/bad-greeting on ephemeral ports — live postfix
    + VPS path never touched): relay exit 0/1/3 contracts, `--expect-open`, both
    greeting-failure shapes → 4, the regression guard (1 silent among healthy →
    4, previously silently 0), JSON/human/CLI contracts, static never-send-DATA,
    **2 mutations** (banner_ok flag defeated / 220 assertion defeated — both must
    RUN and wrongly exit 0 vs real 4).
  - **Two self-inflicted bugs caught & fixed before commit** (both would have
    shipped a vacuous guard): (1) first-draft mutants weren't `chmod +x` →
    rc=126 faked "caught"; suite now requires the mutant to execute and exit 0;
    (2) `EP=$(start_stub)` ran the launcher in a SUBSHELL → `STUBS+=($!)` lost →
    cleanup killed nothing → 8 orphaned stub processes (confirmed via pgrep)
    held the capture-pipe write end, hanging `$(...)`/`| tail` runs while direct
    runs masked it (harness reaps on main-process exit). Fixed: launcher no
    longer uses command substitution + stub stderr/stdout redirected away from
    the suite streams; proven **3/3 captured runs green (6s each), 0 orphans**.
- **Verified**: new suite 38/38 (3× under capture); **full run 16/16 suites, 0
  failures** (137 shell assertions + 11 PHP suites); live
  `smtp-relay-probe --format human` exit 0 with `greeting_failures=0` and all 4
  endpoints `banner_ok=true`; `system-status` → **ok, errors 0, 27 checks**.
- **Health**: units active (dev/prod/git/apache), healthz dev+prod **200**, TLS
  cert **89d** (2026-12-22), git-tree clean after commit, pushed to
  `git://git.gladex.de/gladex.git`. BUDGET 2026-09 unchanged **5.00 / 1.50 /
  3.50**, **model spend 0.00** (`*-free` only), no DNS write, no spend, no secret
  read.
- **NEEDS-INVESTOR (unchanged, still open)**: (a) **A/B/C public-gating decision**
  (msg **#57**, unopened) — investor surface world-reachable via Apache `:443`
  ProxyPass collapsing peers to 127.0.0.1; recommend **A**, or A+B; no config
  change made until the call. (b) **SOA MNAME placeholder** — provider panel only;
  visible each run as the `SOA:gladex.de` warning above.
EOF
echo "appended, tail:"; tail -3 /data/repo/agent-logs/PROGRESS.md
appended, tail:
  ProxyPass collapsing peers to 127.0.0.1; recommend **A**, or A+B; no config
  change made until the call. (b) **SOA MNAME placeholder** — provider panel only;
  visible each run as the `SOA:gladex.de` warning above.
> build · mimo-v2.6-flash-free
$ cd /data/repo && git add -A && git commit -m "feat: smtp-relay-probe asserts the 220 greeting (exit 4) — a one-endpoint :25-greeting regression could no longer pass green; 38-assertion hermetic suite with 2 mutations; root-caused the head -c N false alarm (banner is 48B, full_read blocks), live relay probe verified exit 0, snakeoil→LE re-verified on 993/465/587, 0.4.26" && git push origin main 2>&1 | tail -3; echo "---"; git log --oneline -2; git status --short | head -3
[main 0320d28] feat: smtp-relay-probe asserts the 220 greeting (exit 4) — a one-endpoint :25-greeting regression could no longer pass green; 38-assertion hermetic suite with 2 mutations; root-caused the head -c N false alarm (banner is 48B, full_read blocks), live relay probe verified exit 0, snakeoil→LE re-verified on 993/465/587, 0.4.26
 5 files changed, 447 insertions(+), 18 deletions(-)
 create mode 100755 tests/test_smtp_relay_probe.sh
To git://git.gladex.de/gladex.git
   24fff4b..0320d28  main -> main
---
0320d28 feat: smtp-relay-probe asserts the 220 greeting (exit 4) — a one-endpoint :25-greeting regression could no longer pass green; 38-assertion hermetic suite with 2 mutations; root-caused the head -c N false alarm (banner is 48B, full_read blocks), live relay probe verified exit 0, snakeoil→LE re-verified on 993/465/587, 0.4.26
24fff4b feat: source-sync-check — read-only repo<->env drift guard; first run caught 16 drifted workflow files (dev+prod served pre-0.4.11 broken templates, 8/8 INVALID) — deployed committed fix, 26-assertion hermetic suite + mutation, 0.4.25
> build · mimo-v2.6-flash-free
Run complete. Summary:

## STEP 0 — Investor (done first)
- **0 unread** `investor_to_agent` rows in **both** dev and prod message DBs; **all INBOX.md entries already HANDLED**; `mailboxes/*` empty — no Dispatcher assignments; identity Maildirs 0 new. **Nothing owed → no reply rows needed** (noted in PROGRESS.md).

## One small verifiable step — `smtp-relay-probe` now *asserts* the greeting (CHANGELOG 0.4.26)
While verifying the operator's standing item *"live relay probe still blocked by your :25-greeting bug — verify after that fix"*, I found a real guard gap: the probe only *reported* the banner — a greeting regression on **one of four** endpoints would still have exited 0. Now:
- Banner phase isolated: timeout/EOF/non-220 → `banner_ok=false` + `greeting_error`, dialogue skipped, **not** folded into `unreachable`; new **exit 4**, precedence `1 > 4 > 3 > 0`, documented in `--help` + REGISTRY.
- **`tests/test_smtp_relay_probe.sh` — 38 assertions**, hermetic (stub SMTP servers on ephemeral ports; live stack untouched), incl. the regression guard *1 silent endpoint among healthy → 4 (previously silently 0)* and **2 mutations** that must run and wrongly exit 0.
- **Two self-infixed bugs caught before commit**: non-executable mutants faked a pass (rc=126); `EP=$(start_stub)` ran in a subshell so cleanup killed nothing — 8 orphan stubs hung captured runs (proven fixed: **3/3 captured runs green, 0 orphans**).

## Verification sweep — all operator "Next" items now closed with live proof
- **snakeoil → LE**: already done — re-verified live :993/:465/:587 present `CN=gladex.de` (LE, exp 2026-12-22), 0 snakeoil hits in postfix+dovecot.
- **:25-greeting bug genuinely FIXED** — my first re-probe's "zero bytes + `commands=0/0`" was a **false alarm root-caused**: banner is exactly 48 bytes, and `head -c 100` blocks in coreutils `full_read()` waiting for bytes the server correctly never sends. Proof: `head -c 48` → instant rc 0 vs `head -c 60` → rc 124; **40/40 python probes ≤15ms**, nc, bash `read -t`, strace all show the `220`.
- **Live relay probe: exit 0** — :587/:465 deny `554`, public path `77.90.15.49:25` denies `454`, queue empty; all 4 endpoints `banner_ok=true`.
- **MX** green on both resolvers, serial sane.

## Health
**16/16 suites, 0 failures** · system-status **ok, 27 checks** · healthz dev+prod **200** · TLS **89d** · BUDGET **5.00 / 1.50 / 3.50**, model spend **0.00** (`*-free` only) · committed `0320d28` + pushed to `git://git.gladex.de/gladex.git`.

**Still open (investor)**: (a) A/B/C public-gating decision (msg #57, unopened); (b) SOA MNAME placeholder (provider panel).
exit=0

Select another run log from the list above. Only files matching run-YYYYMMDD-HHMMSS-N.log are readable.

[Fri Sep 25 01:07:30 2026] 127.0.0.1:42756 Accepted
[Fri Sep 25 01:07:30 2026] 127.0.0.1:42756 Closing
[Fri Sep 25 01:07:30 2026] 127.0.0.1:42762 Accepted
[Fri Sep 25 01:07:30 2026] 127.0.0.1:42762 Closing
[Fri Sep 25 01:07:30 2026] 127.0.0.1:42770 Accepted
[Fri Sep 25 01:07:30 2026] 127.0.0.1:42770 Closing
[Fri Sep 25 01:07:30 2026] 127.0.0.1:42780 Accepted
[Fri Sep 25 01:07:30 2026] 127.0.0.1:42780 Closing
[Fri Sep 25 01:09:05 2026] 127.0.0.1:43758 Accepted
[Fri Sep 25 01:09:05 2026] 127.0.0.1:43758 Closing
[Fri Sep 25 01:29:04 2026] 127.0.0.1:52792 Accepted
[Fri Sep 25 01:29:04 2026] 127.0.0.1:52792 Closing
[Fri Sep 25 01:29:05 2026] 127.0.0.1:33658 Accepted
[Fri Sep 25 01:29:05 2026] 127.0.0.1:33658 Closing
[Fri Sep 25 01:29:05 2026] 127.0.0.1:33674 Accepted
[Fri Sep 25 01:29:05 2026] 127.0.0.1:33674 Closing
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49482 Accepted
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49482 Closing
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49488 Accepted
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49488 Closing
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49494 Accepted
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49494 Closing
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49504 Accepted
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49504 Closing
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49508 Accepted
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49508 Closing
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49522 Accepted
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49522 Closing
[Fri Sep 25 01:34:56 2026] 127.0.0.1:49536 Accepted
[Fri Sep 25 01:34:57 2026] 127.0.0.1:49536 Closing
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60400 Accepted
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60400 Closing
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60408 Accepted
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60408 Closing
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60422 Accepted
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60422 Closing
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60428 Accepted
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60428 Closing
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60442 Accepted
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60442 Closing
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60446 Accepted
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60446 Closing
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60462 Accepted
[Fri Sep 25 01:37:40 2026] 127.0.0.1:60462 Closing
[Fri Sep 25 01:42:50 2026] 127.0.0.1:37798 Accepted
[Fri Sep 25 01:42:50 2026] 127.0.0.1:37798 Closing
[Fri Sep 25 01:42:50 2026] 127.0.0.1:37802 Accepted
[Fri Sep 25 01:42:50 2026] 127.0.0.1:37802 Closing
[Fri Sep 25 01:42:51 2026] 127.0.0.1:37814 Accepted
[Fri Sep 25 01:42:51 2026] 127.0.0.1:37814 Closing
[Fri Sep 25 01:42:51 2026] 127.0.0.1:37818 Accepted
[Fri Sep 25 01:42:51 2026] 127.0.0.1:37818 Closing
[Fri Sep 25 01:42:54 2026] 127.0.0.1:37824 Accepted
[Fri Sep 25 01:42:54 2026] 127.0.0.1:37824 Closing
[Fri Sep 25 01:42:55 2026] 127.0.0.1:44098 Accepted
[Fri Sep 25 01:42:55 2026] 127.0.0.1:44098 Closing
[Fri Sep 25 01:42:55 2026] 127.0.0.1:44106 Accepted
[Fri Sep 25 01:42:55 2026] 127.0.0.1:44106 Closing
[Fri Sep 25 01:43:00 2026] 127.0.0.1:44120 Accepted