Verify CoinPay runtime capability before vault alerts

README.md

@@ -500,3 +500,18 @@ threatcrush-nightly --to a@b.com,c@d    someone else (default anthony@ + preshy@
 
 Runs from cron at 06:00 UTC. A failure is mailed as "ThreatCrush nightly FAILED"
 and raised with `alert --key threatcrush-nightly`.
+
+### CoinPay signing monitor
+
+`coinpay-vault-watch` runs on dev2 every 15 minutes. `coinpay-vault-watch --check`
+reports current runtime capability and counts incompatible signing records without
+sending notifications, unlocking keys, decrypting keys, or forwarding payments.
+It requires Docker access and the deployed app's existing Supabase configuration.
+
+Legacy deployments with operator-sealed signing records produce a deployment/format
+mismatch alert, with no instruction to visit a nonexistent unlock page. Legacy
+deployments without such records have no vault-unlock requirement. Vault-capable
+deployments use explicit status logs; missing status is unknown, never proof of a lock.
+Signing-format counts are compatibility checks, not evidence of current on-chain balances.
+One critical smshub alert per six-hour incident cycle replaces duplicate text/call
+requests; smshub selects its supported delivery channel. Existing unrelated alerts remain.

bin/coinpay-vault-watch

@@ -0,0 +1,87 @@
+#!/usr/bin/env bash
+# coinpay-vault-watch --check: read-only capability and signing-format report.
+# Remind Anthony before CoinPay's operator vault locks, and tell him at once when it is locked
+# (only applicable when the deployed runtime supports the vault).
+# Capability and pending signing formats come from a read-only runtime probe.
+# For vault-capable runtimes, status comes from the CURRENT container log:
+#   "[operator-keys] unlocked until <ISO>"  /  "operator keys locked".
+# Cron: */15 * * * *  coinpay-vault-watch      (stamps in ~/.local/state/coinpay-vault-watch)
+set -uo pipefail
+export PATH="$HOME/.local/bin:$HOME/.local/share/mise/shims:/usr/local/bin:/usr/bin:/bin"
+C=coinpayportalcom-app-1; S="$HOME/.local/state/coinpay-vault-watch"; mkdir -p "$S"
+URL="https://coinpayportal.com/admin/operator-vault"
+TO=$(grep -h '^ALERT_EMAIL=' "$HOME/.config/alert/env" 2>/dev/null | cut -d= -f2- | tr -d '"')
+log(){ printf '%s %s\n' "$(date -u +%FT%TZ)" "$*" >> "$S/watch.log"; }
+notify(){ # <subject> <body> — one critical alert; smshub chooses the delivery channel.
+  if [ -n "$TO" ]; then
+    env -u RESEND_API_KEY mail send --to "$TO" --subject "$1" --via resend --body "$2" >/dev/null 2>&1 || log "email failed: $1"
+  fi
+  alert --severity critical --key coinpay-vault-call --every 6h "$1" >/dev/null 2>&1 || log "critical alert failed"
+  log "notified: $1"
+}
+notify_issue(){
+  local now last
+  now=$(date -u +%s); last=$(cat "$S/locked" 2>/dev/null || echo 0)
+  if [ $((now - last)) -ge 21600 ]; then
+    echo "$now" > "$S/locked"
+    notify "$1" "$2"
+  fi
+}
+# Ask the CURRENT runtime and database, rather than interpreting missing logs as a lock.
+tool_root=$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")/.." && pwd)
+probe_file="${COINPAY_WATCH_PROBE:-$tool_root/lib/coinpay-vault-watch/probe.mjs}"
+probe=$(docker exec -i -w /app "$C" node --input-type=module < "$probe_file" 2>/dev/null) || probe='{"mode":"probe-error"}'
+if [ "${1:-}" = --check ]; then printf '%s\n' "$probe"; exit 0; fi
+mode=$(printf '%s' "$probe" | python3 -c 'import json,sys;print(json.load(sys.stdin)["mode"])' 2>/dev/null || echo probe-error)
+sealed=$(printf '%s' "$probe" | python3 -c 'import json,sys;print(json.load(sys.stdin).get("sealedSigningRows",0))' 2>/dev/null || echo 0)
+case "$mode" in
+  legacy)
+    if [ "$sealed" -gt 0 ]; then
+      notify_issue "CoinPay signing mismatch: deployed app cannot read $sealed vault-sealed payment keys" "The deployed CoinPay version has no operator-vault workflow, but $sealed payment signing records require that format. Restore a compatible signing version or complete a reviewed key migration. The old /admin/operator-vault link is unavailable on this deployment. No keys or payments were changed by this monitor."
+    else
+      log "legacy signing: no vault-sealed payment keys requiring forwarding; vault unlock alert not applicable"
+    fi
+    exit 0 ;;
+  operator-vault) ;;
+  *)
+    notify_issue "CoinPay signing readiness could not be verified" "The current runtime capability probe returned $mode. This does not establish that the vault is locked or that payments have failed. Check the deployment and monitor."
+    exit 1 ;;
+esac
+line=$(docker logs "$C" 2>&1 | grep -E '\[operator-keys\] unlocked until|operator keys locked' | tail -1)
+now=$(date -u +%s)
+if [[ "$line" =~ unlocked\ until\ ([0-9TZ:.-]+) ]] && until=$(date -u -d "${BASH_REMATCH[1]}" +%s 2>/dev/null) && [ "$until" -gt "$now" ]; then
+  rm -f "$S/locked"
+  left=$(( (until - now) / 3600 )); when=$(date -u -d "@$until" '+%a %b %-d %H:%M UTC')
+  for h in 48 24 6 1; do
+    stamp="$S/remind-$until-$h"
+    if [ $((until - now)) -le $((h * 3600)) ] && [ ! -e "$stamp" ]; then
+      touch "$stamp"
+      notify "CoinPay vault locks in ${left}h (${when})" "CoinPay's operator vault locks at ${when}. When it locks, customer payments stop being forwarded to merchants.
+
+Unlock it before then: ${URL}  (click the eye, type your password, Unlock, 7 days).
+
+-- riotcoder (coinpay-vault-watch on dev2)"
+      break
+    fi
+  done
+  exit 0
+fi
+# A container under 10 minutes old is still mid-deploy: deploy-app.sh hands the old
+# process's unlock to it once healthy (operator-vault-handoff.sh). Judge it next tick.
+started=$(date -u -d "$(docker inspect -f '{{.State.StartedAt}}' "$C" 2>/dev/null)" +%s 2>/dev/null || echo 0)
+[ $((now - started)) -lt 600 ] && exit 0
+if [[ ! "$line" =~ operator\ keys\ locked ]] && [[ ! "$line" =~ unlocked\ until ]]; then
+  notify_issue "CoinPay vault state is unknown; signing readiness needs verification" "This deployed version supports the operator vault, but its log has no explicit locked or unlocked status. No claim that payments are blocked can be made from missing logs. Check ${URL}."
+  exit 0
+fi
+# Locked (expired, redeployed or restarted). Repeat every 6h until unlocked.
+last=$(cat "$S/locked" 2>/dev/null || echo 0)
+if [ $((now - last)) -ge 21600 ]; then
+  echo "$now" > "$S/locked"
+  notify "CoinPay vault is LOCKED: payments are not forwarding" "CoinPay's operator vault is locked (it expired, or CoinPay was redeployed/restarted). Customer payments are NOT being forwarded to merchants until you unlock it.
+
+Unlock now: ${URL}  (click the eye, type your password, Unlock, 7 days).
+
+This repeats every 6 hours until it is unlocked.
+-- riotcoder (coinpay-vault-watch on dev2)"
+fi

lib/coinpay-vault-watch/probe.mjs

@@ -0,0 +1,24 @@
+// Read-only: reports capability and format counts, never opens or returns keys.
+import { existsSync, readFileSync } from 'node:fs';
+import { createClient } from '@supabase/supabase-js';
+const paths = JSON.parse(readFileSync('/app/.next/server/app-paths-manifest.json', 'utf8'));
+const operatorModule = existsSync('/app/src/lib/wallets/operator-keys.ts');
+const operatorRoutes = Object.keys(paths).some((p) => p.includes('operator-vault'));
+const db = createClient(process.env.NEXT_PUBLIC_SUPABASE_URL, process.env.SUPABASE_SERVICE_ROLE_KEY, { global: { fetch: (url, init) => fetch(url, { ...init, signal: AbortSignal.timeout(15000) }) } });
+const payments = [];
+for (let from = 0; ; from += 500) {
+  const { data, error } = await db.from('payments').select('id,status')
+    .in('status', ['confirmed', 'forwarding', 'forwarding_failed'])
+    .is('forwarded_at', null).is('forward_tx_hash', null)
+    .order('id').range(from, from + 499);
+  if (error) throw new Error('Could not read signing readiness');
+  payments.push(...data);
+  if (data.length < 500) break;
+}
+let sealed = 0;
+for (let i = 0; i < payments.length; i += 100) {
+  const { data, error } = await db.from('payment_addresses').select('encrypted_private_key').in('payment_id', payments.slice(i, i + 100).map((p) => p.id));
+  if (error) throw new Error('Could not inspect signing formats');
+  sealed += data.filter((p) => p.encrypted_private_key?.startsWith('cpv1.')).length;
+}
+console.log(JSON.stringify({ mode: operatorModule && operatorRoutes ? 'operator-vault' : !operatorModule && !operatorRoutes ? 'legacy' : 'inconsistent', sealedSigningRows: sealed, paymentsChecked: payments.length }));

test/coinpay-vault-watch.py

@@ -0,0 +1,31 @@
+import os,subprocess,tempfile,time,json
+from pathlib import Path
+root=Path(__file__).resolve().parent.parent/'bin'
+cases=[
+ ('legacy-clear',{'mode':'legacy','sealedSigningRows':0},'',0,None),
+ ('legacy-sealed',{'mode':'legacy','sealedSigningRows':5},'',1,'signing mismatch'),
+ ('vault-unknown',{'mode':'operator-vault','sealedSigningRows':5},'',1,'state is unknown'),
+ ('vault-locked',{'mode':'operator-vault','sealedSigningRows':5},'operator keys locked',1,'vault is LOCKED'),
+ ('probe-failed',{'mode':'probe-error'},'',1,'could not be verified'),
+]
+for name,probe,logs,want,phrase in cases:
+ with tempfile.TemporaryDirectory() as temp:
+  t=Path(temp);b=t/'bin';b.mkdir();state=t/'state';sent=t/'sent';fixture=t/'probe';fixture.write_text(json.dumps(probe));probe_input=t/'input.mjs';probe_input.write_text('')
+  mocks={
+   'docker':'#!/bin/sh\ncase "$1" in exec) cat "$WATCH_TEST_JSON";; logs) printf "%s\\n" "$WATCH_TEST_LOGS";; inspect) echo 2026-10-01T00:00:00Z;; esac\n',
+   'alert':'#!/bin/sh\nprintf "%s\\n" "$*" >> "$WATCH_TEST_SENT"\n',
+   'mail':'#!/bin/sh\nexit 0\n',
+  }
+  for command,body in mocks.items():p=b/command;p.write_text(body);p.chmod(0o755)
+  s=(root/'coinpay-vault-watch').read_text();s='\n'.join(l for l in s.splitlines() if not l.startswith('export PATH='));s=s.replace('S="$HOME/.local/state/coinpay-vault-watch"',f'S="{state}"')
+  script=t/'watch';script.write_text(s)
+  env={**os.environ,'PATH':str(b)+':'+os.environ['PATH'],'WATCH_TEST_JSON':str(fixture),'WATCH_TEST_LOGS':logs,'WATCH_TEST_SENT':str(sent),'COINPAY_WATCH_PROBE':str(probe_input)}
+  subprocess.run(['bash',str(script)],env=env,check=name!='probe-failed',capture_output=True)
+  lines=sent.read_text().splitlines() if sent.exists() else []
+  assert len(lines)==want,(name,lines)
+  if phrase:assert phrase in lines[0],(name,lines)
+  if name=='legacy-sealed':assert 'Unlock the CoinPay operator vault' not in lines[0]
+  if want:
+   subprocess.run(['bash',str(script)],env=env,capture_output=True)
+   assert len(sent.read_text().splitlines())==1,'repeat was not throttled'
+  print('PASS',name)

test/coinpay-vault-watch.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+set -euo pipefail
+python3 "$(dirname "$0")/coinpay-vault-watch.py"