diff --git a/.env.example b/.env.example index 3d67fc2..e03f717 100644 --- a/.env.example +++ b/.env.example @@ -46,6 +46,8 @@ STREAMR_CRONSCHEDULE="0 * * * *" # Every hour # CENTRAL_DECISION_TIMEOUT_MS=8000 # CENTRAL_RECOVERY_HEALTHY_MS=15000 # AGENT_QUEUE_LIMIT=10000 +# Gateway billing pruning (UTC): confirmed readings are removed only after central accepts them. +# GATEWAY_PRUNE_CRONSCHEDULE="*/5 * * * *" # Never enable this outside isolated development fixtures. # CENTRAL_ALLOW_INSECURE_DEVELOPMENT=false # Only explicit reverse-proxy peer IPs; forwarded headers are otherwise ignored. diff --git a/README.md b/README.md index c9518ce..b87503e 100644 --- a/README.md +++ b/README.md @@ -682,6 +682,34 @@ integration tests are complete. ## Central Console and gateway agent (opt-in test branch) +Managed gateways also transfer stored billing history to central, including +readings collected before enrollment. Every five seconds, while the central +connection is healthy, the agent can transfer one batch of up to 100 readings +from one meter, oldest nonces first. Central verifies signatures and nonce +continuity and merges missing rows without overwriting conflicting transactions. +Accepted batches are acknowledged durably on the gateway. Lost responses are +retried idempotently; historical transfers do not send device replies or create +new agent operation records. A rejected batch stays on the gateway for correction +and retry. Central's existing chain pruning and publishing policies still apply +to imported history; this is not a permanent archive of confirmed readings. + +The gateway Console runs a pruning cron job every five minutes (UTC). Set +`GATEWAY_PRUNE_CRONSCHEDULE="0 * * * *"` for hourly pruning, for example. For each +meter, it reads the confirmed nonce from the chain and removes transaction rows +with `nonce <= confirmedNonce` only when the exact record was accepted by the +configured central/enrollment. Unconfirmed readings and readings awaiting central +acceptance remain. Chain failures or failed transfers preserve data for a later +run. A separate checkpoint retains at most one last confirmed reading per meter +so a device retry can still be acknowledged after its transaction row is pruned. +The checkpoint is not part of the pending `transactions` table. + +Update central and both gateway services (`console` and `agent`) to enable this +policy. The agent negotiates history support with central; an older central keeps +the existing live billing connection and no new history acknowledgments are +created. Existing databases gain the receipt/checkpoint tables automatically. +This policy does not prune the agent's duplicate-detection operation journal or +change `AGENT_QUEUE_LIMIT`. + The default `standalone` role preserves the local collector and LAN desktop. `central` adds authenticated multi-gateway access; `gateway` keeps a local fallback Console while an outbound agent owns MQTT. The same default multi-architecture diff --git a/docs/M3tering-Console-Manual.pdf b/docs/M3tering-Console-Manual.pdf index d17fa29..b7fa600 100644 Binary files a/docs/M3tering-Console-Manual.pdf and b/docs/M3tering-Console-Manual.pdf differ diff --git a/docs/deployment-guide.md b/docs/deployment-guide.md index 6a1fb20..a07ffce 100644 --- a/docs/deployment-guide.md +++ b/docs/deployment-guide.md @@ -178,6 +178,18 @@ While central is unreachable, a managed gateway sends the readings it decides it its own `.env` sets `PREFERRED_PROVER_NODE`. Central's scheduled runs resend them later anyway. A gateway never publishes to Streamr or Arweave; central does that. +To send a batch now instead of waiting for the schedule, run this in the Console folder on central or +a standalone Console: + +```sh +./prover-batch.sh # each meter's lowest 100 unconfirmed nonces +./prover-batch.sh --limit 250 # each meter's lowest 250 +./prover-batch.sh --all # every unconfirmed reading, like the scheduled job +``` + +It prints, per meter, which nonces it sends, then the prover's answer. The lowest nonces go first because +they are the ones the prover needs next. + Logs: `Sending N transactions to prover at …`, then `Prover response (text): …`. ## The Streamr app diff --git a/prover-batch.sh b/prover-batch.sh new file mode 100755 index 0000000..deac5ac --- /dev/null +++ b/prover-batch.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Send readings not yet confirmed on-chain to the prover now. For each meter: its lowest 100 nonces. +# Run in the Console folder on central or a standalone Console. +# ./prover-batch.sh lowest 100 unconfirmed nonces per meter +# ./prover-batch.sh --limit 250 lowest 250 per meter +# ./prover-batch.sh --all every unconfirmed reading (what the scheduled job sends) +set -euo pipefail +limit=100 +while [[ $# -gt 0 ]]; do + case "$1" in + --limit) [[ $# -ge 2 && "$2" =~ ^[1-9][0-9]*$ ]] || { echo 'Use --limit followed by a positive whole number.' >&2; exit 2; }; limit=$2; shift 2 ;; + --all) limit=0; shift ;; + -h|--help) sed -n '2,6p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + *) echo "Unknown option: $1 (use --limit N or --all)" >&2; exit 2 ;; + esac +done +cd -- "$(dirname -- "${BASH_SOURCE[0]}")" +[[ -f .env ]] || { echo 'No .env here: run this in the Console folder of a deployed Console.' >&2; exit 2; } +env_value() { + sed -n "s/^[[:space:]]*$1[[:space:]]*=[[:space:]]*//p" .env | tail -1 \ + | sed -e 's/[[:space:]]*$//' -e 's/^"\(.*\)"$/\1/' -e "s/^'\(.*\)'\$/\1/" +} +# Use the role's compose file: the one ./auto-compose.sh recorded, or the one CONSOLE_ROLE implies. +compose_file=$(env_value COMPOSE_FILE) +if [[ -z "$compose_file" ]]; then + case "$(env_value CONSOLE_ROLE)" in + central) compose_file=compose.central.yaml ;; + gateway) compose_file=compose.gateway.yaml ;; + *) compose_file=compose.yaml ;; + esac +fi +project=$(env_value COMPOSE_PROJECT_NAME) +args=(docker compose -f "$compose_file") +[[ -z "$project" ]] || args+=(--project-name "$project") +[[ -n "$("${args[@]}" ps -q console 2>/dev/null)" ]] || { echo "The Console isn't running here ($compose_file). Start it with ./auto-compose.sh." >&2; exit 1; } +exec "${args[@]}" exec -T -e PROVER_BATCH_LIMIT="$limit" console node - < scripts/prover-batch.cjs diff --git a/scripts/prover-batch.cjs b/scripts/prover-batch.cjs new file mode 100644 index 0000000..a26bcfa --- /dev/null +++ b/scripts/prover-batch.cjs @@ -0,0 +1,29 @@ +// Sends readings not yet confirmed on-chain to the prover, now. For each meter it sends the lowest +// PROVER_BATCH_LIMIT nonces (default 100; 0 = all), the ones the prover needs next. +// Run through ./prover-batch.sh, inside the Console container. +const role = process.env.CONSOLE_ROLE || 'standalone'; +const url = process.env.PREFERRED_PROVER_NODE; +if (role === 'gateway') { console.log('This is a managed gateway: central sends batches to the prover. Run this on central.'); process.exit(2); } +if (!url) { console.log('PREFERRED_PROVER_NODE is not set in .env, so there is no prover to send to.'); process.exit(2); } +require('/opt/app/dist/store/sqlite').default(); +if (role === 'central') require('/opt/app/dist/central/distribution').configurePublications('.data/central.db'); +const Prover = require('/opt/app/dist/lib/core/prover').default; +const { getSyncedPendingTransactions } = require('/opt/app/dist/lib/sync'); +(async () => { + const pending = await getSyncedPendingTransactions(); // checks each meter on-chain and prunes confirmed readings first + console.log(pending.length + ' reading(s) not yet confirmed on-chain.'); + if (!pending.length) return; + const limit = Number(process.env.PROVER_BATCH_LIMIT ?? 100); + if (!Number.isInteger(limit) || limit < 0) { console.log('PROVER_BATCH_LIMIT must be a whole number (0 = all).'); process.exit(2); } + const byMeter = new Map(); + for (const record of pending) byMeter.set(record.identifier, [...(byMeter.get(record.identifier) || []), record]); + const batch = []; + for (const [meter, records] of [...byMeter].sort((a, b) => a[0] - b[0])) { + const lowest = records.sort((a, b) => a.nonce - b.nonce).slice(0, limit || records.length); + console.log('meter ' + meter + ': sending nonces ' + lowest[0].nonce + '-' + lowest[lowest.length - 1].nonce + + ' (' + lowest.length + ' of ' + records.length + ' unconfirmed)'); + batch.push(...lowest); + } + const response = await new Prover().sendPendingTransactionsToProver(url, batch); + console.log(response ? 'Prover answered: ' + await response.text() : 'Nothing was sent.'); +})().then(() => process.exit(0), e => { console.error('Batch failed: ' + e.message); process.exit(1); }); diff --git a/src/agent/index.ts b/src/agent/index.ts index 039baeb..90f668d 100644 --- a/src/agent/index.ts +++ b/src/agent/index.ts @@ -11,6 +11,7 @@ import { LocalDecider } from '../central/local-channel'; import { SocketDecider, parseOperation } from '../central/rpc'; import { parseMessage, MAX_MESSAGE_BYTES, Health, UplinkEvent } from '../central/protocol'; import { Replication } from '../central/replication'; +import { syncGatewayHistory } from '../central/history'; import { mqttConnection } from '../lib/mqtt-config'; import { applicationId } from '../lib/chirpstack-config'; import { managedQueue } from '../services/grpc'; @@ -55,6 +56,7 @@ export async function runAgent():Promise<()=>Promise> { const local=new LocalDecider(gatewayFallbackPort(),readOwnerSecret(config.fallbackSecretPath),config.decisionTimeoutMs); let socket:WebSocket|undefined,stopped=false,reconnect:ReturnType|undefined; let connectedAt=0,attempt=0,mqttState:Health['mqtt']='connecting',lastError:Health['lastError']='none'; + let historyAvailable=false; const central=new SocketDecider(()=>socket,config.decisionTimeoutMs); const replication=new Replication(journal,central,local); const healthy=()=>socket?.readyState===WebSocket.OPEN&&Date.now()-connectedAt>=config.recoveryHealthyMs; @@ -78,9 +80,11 @@ export async function runAgent():Promise<()=>Promise> { let failing:string|null=null; const connectCentral=()=>{ if(stopped)return; + historyAvailable=false; const ws=socket=new WebSocket(new URL('/agent',config.centralUrl),{headers:{Authorization:`Bearer ${currentCredential}`},maxPayload:MAX_MESSAGE_BYTES, perMessageDeflate:false,handshakeTimeout:8000,followRedirects:false}); const inputLimit=new RateLimit(512,1000,1); + ws.on('upgrade',response=>{historyAvailable=response.headers['x-console-history']==='1';}); const tunnels=new TunnelMux(ws,request=>gatewayTarget(request)); ws.on('open',()=>{connectedAt=Date.now();attempt=0;lastError='none'; logger.info(failing?`Connected to central again (${config.centralUrl}).`:`Connected to central (${config.centralUrl}).`);failing=null; @@ -158,8 +162,22 @@ export async function runAgent():Promise<()=>Promise> { } }; let draining:Promise|undefined; + let historySync:Promise|undefined; + let historyFailed=false; const timer=setInterval(()=>{ - sendHealth();if(draining)return; + sendHealth(); + // A bounded history batch runs independently of live decisions/replay. Slow + // chain validation on central must not hold up the gateway's fallback loop. + if(healthy()&&historyAvailable&&!historySync){ + historySync=syncGatewayHistory(gateway,local,central).then(count=>{ + if(count)logger.info(`Central accepted ${count} historical readings.`); + historyFailed=false; + }).catch(()=>{ + if(!historyFailed)logger.warn('Historical synchronization deferred; readings stay on this gateway until central accepts them.'); + historyFailed=true; + }).finally(()=>{historySync=undefined;}); + } + if(draining)return; draining=(async()=>{ try{const state=await updater.status();updatesEnabled=state.enabled; if(socket?.readyState===WebSocket.OPEN)for(const job of state.jobs){if(updateStatuses.get(job.id)!==job.status){socket.send(JSON.stringify({v:1,type:'update-progress',id:job.id,status:job.status}));updateStatuses.set(job.id,job.status);}} @@ -167,7 +185,7 @@ export async function runAgent():Promise<()=>Promise> { try{await replication.flush('local');}catch{}try{await replication.flush('central');}catch{}await coordinator.drain();})() .catch(()=>{lastError='storage';}).finally(()=>{draining=undefined;}); },5000);timer.unref();connectCentral(); - const stop=async()=>{stopped=true;stopResolverWatch();clearTimeout(reconnect);clearInterval(timer);central.disconnect();socket?.terminate();await mqtt.endAsync(false);await coordinator.stop();await draining;journal.close();}; + const stop=async()=>{stopped=true;stopResolverWatch();clearTimeout(reconnect);clearInterval(timer);central.disconnect();socket?.terminate();await mqtt.endAsync(false);await coordinator.stop();await draining;await historySync;journal.close();}; const stopResolverWatch=watchResolver(()=>{ logger.error('This container has no DNS server (Docker copied the host\'s DNS settings before its network was up). Restarting so Docker provides fresh ones.'); void stop().finally(()=>process.exit(1)); diff --git a/src/central/billing.ts b/src/central/billing.ts index 6a6508d..4cb4b77 100644 --- a/src/central/billing.ts +++ b/src/central/billing.ts @@ -21,7 +21,7 @@ export function createBillingProcessor(filename:string):DurableProcessor { const logger=createLogger('[central]'); /** Accept another decider's billing state (a gateway's fallback decisions) after * checking it against the meter's signatures, its token and the chain. */ -export async function importBillingSnapshot(snapshot:BillingSnapshot):Promise { +export async function importBillingSnapshot(snapshot:BillingSnapshot, distribute=true):Promise { const meter=snapshot.meter;if(!meter)return; const known=getMeterByPublicKey(meter.publicKey); const token=known?.tokenId ?? await meterIdFromPublicKeyContractCall(meter.publicKey); @@ -48,6 +48,6 @@ export async function importBillingSnapshot(snapshot:BillingSnapshot):Promise{}); } diff --git a/src/central/gateway-pruning.ts b/src/central/gateway-pruning.ts new file mode 100644 index 0000000..1c980fe --- /dev/null +++ b/src/central/gateway-pruning.ts @@ -0,0 +1,30 @@ +import cron from 'node-cron'; +import { getAllMeterRecords, pruneGatewayHistory } from '../store/sqlite'; +import { meterLatestNonceContractCall } from '../utils/contracts'; +import { createLogger } from '../utils/logger'; + +const logger = createLogger('[gateway-pruning]'); + +export async function pruneConfirmedGatewayHistory(target: string): Promise { + let removed = 0; + for (const meter of getAllMeterRecords()) { + try { + const confirmed = await meterLatestNonceContractCall(meter.tokenId); + removed += pruneGatewayHistory(target, meter.tokenId, confirmed); + } catch { + // A failed chain read is never evidence that a local record may be deleted. + logger.warn(`Kept readings for meter ${meter.tokenId}: confirmation or pruning failed.`); + } + } + if (removed) logger.info(`Removed ${removed} confirmed readings already accepted by central.`); + return removed; +} + +export function startGatewayPruning(target: () => string) { + const schedule = process.env.GATEWAY_PRUNE_CRONSCHEDULE || '*/5 * * * *'; + if (!cron.validate(schedule)) throw new Error('GATEWAY_PRUNE_CRONSCHEDULE must be a valid cron expression.'); + return cron.schedule(schedule, async () => { + try { await pruneConfirmedGatewayHistory(target()); } + catch { logger.warn('Pruning deferred until gateway enrollment is ready.'); } + }, { name: 'gateway-prune-confirmed-history', noOverlap: true, timezone: 'UTC' }); +} diff --git a/src/central/history.ts b/src/central/history.ts new file mode 100644 index 0000000..578c40a --- /dev/null +++ b/src/central/history.ts @@ -0,0 +1,13 @@ +import type { Decider } from './processor'; + +/** Separate from live operation replay: historical rows must never enqueue a + * device reply or consume an agent operation/idempotency slot. A lost response + * leaves the batch unacknowledged locally, so the same bytes are retried. */ +export async function syncGatewayHistory(gateway: string, local: Decider, central: Decider): Promise { + if (!local.history || !central.history) throw new Error('History synchronization is unavailable.'); + const batch = await local.history(gateway, 'history-read'); + if (!batch) return 0; + await central.history(gateway, 'history-import', batch); + await local.history(gateway, 'history-ack', batch); + return batch.transactions.length; +} diff --git a/src/central/hub.ts b/src/central/hub.ts index 30b16ee..baa3e71 100644 --- a/src/central/hub.ts +++ b/src/central/hub.ts @@ -32,6 +32,9 @@ export class GatewayHub { }; mount(app:Express,server:http.Server):void { const wss=new WebSocket.Server({noServer:true,maxPayload:MAX_MESSAGE_BYTES,perMessageDeflate:false}); + // Capability negotiation keeps upgraded gateways from sending new RPCs to + // older central servers, which would close the live billing connection. + wss.on('headers',headers=>headers.push('X-Console-History: 1')); registerSocket(server,'/agent',wss,req=>{ if((!this.development&&!this.auth.secure(req))||!this.upgradeLimit.allow(req.socket.remoteAddress||'unknown'))return false; const token=req.headers.authorization?.match(/^Bearer ([A-Za-z0-9_-]{43})$/)?.[1]; @@ -55,7 +58,9 @@ export class GatewayHub { let message;try{message=parseOperation(data.toString());if(message.type!=='operation')throw new Error();}catch{socket.close(1008,'Invalid operation');return;} inFlight++; void serveOperation(this.processor(),gateway,message).then(reply=>{// Learn which gateway serves a meter, also from reports it decided while central was away. - if(!reply.error&&(message.action==='offer'||message.action==='reconcile')&&message.event)this.identities.observeDevice(message.event.deviceInfo.devEui,gateway);if(socket.readyState===WebSocket.OPEN)socket.send(JSON.stringify(reply));}).catch(()=>socket.close(1011,'Operation storage unavailable')).finally(()=>inFlight--); + if(!reply.error&&(message.action==='offer'||message.action==='reconcile')&&message.event)this.identities.observeDevice(message.event.deviceInfo.devEui,gateway); + if(!reply.error&&message.action==='history-import'&&message.history?.meter?.devEui)this.identities.observeDevice(message.history.meter.devEui,gateway); + if(socket.readyState===WebSocket.OPEN)socket.send(JSON.stringify(reply));}).catch(()=>socket.close(1011,'Operation storage unavailable')).finally(()=>inFlight--); return; } try { diff --git a/src/central/local-channel.ts b/src/central/local-channel.ts index c0b4175..e9f9908 100644 --- a/src/central/local-channel.ts +++ b/src/central/local-channel.ts @@ -1,9 +1,9 @@ import http from 'http'; import express from 'express'; -import type { Decider, Outcome } from './processor'; +import type { Decider, Outcome, BillingSnapshot, HistoryAction } from './processor'; import type { UplinkEvent } from './protocol'; import { MAX_MESSAGE_BYTES } from './protocol'; -import { parseOperation, serveOperation, OperationRequest } from './rpc'; +import { parseOperation, serveOperation, OperationRequest, OperationResponse } from './rpc'; import { hashSecret, equalSecretHash } from './secrets'; import { eventIdentity } from './operations'; import { randomUUID } from 'crypto'; @@ -30,11 +30,17 @@ export class LocalDecider implements Decider { if(!Number.isInteger(port)||port<1024||port>65535)throw new Error('Invalid fallback port.'); } offer(_gateway:string,event:UplinkEvent):Promise{return this.call('offer',eventIdentity(event),event).then(()=>{});} - commit(_gateway:string,id:string):Promise{return this.call('commit',id).then(outcome=>{if(!outcome)throw new Error('No local outcome.');return outcome;});} - outcome(_gateway:string,id:string):Promise{return this.call('outcome',id);} + commit(_gateway:string,id:string):Promise{return this.call('commit',id).then(result=>{if(!result.outcome)throw new Error('No local outcome.');return result.outcome;});} + outcome(_gateway:string,id:string):Promise{return this.call('outcome',id).then(result=>result.outcome);} reconcile(_gateway:string,event:UplinkEvent,outcome:Outcome):Promise{return this.call('reconcile',outcome.id,event,outcome).then(()=>{});} - private async call(action:OperationRequest['action'],id:string,event?:UplinkEvent,outcome?:Outcome):Promise { - const body:OperationRequest={v:1,type:'operation',requestId:randomUUID(),action,id,...(event?{event}:{}),...(outcome?{outcome}:{})}; + history(_gateway:string,action:HistoryAction,history?:BillingSnapshot):Promise{ + return this.call(action,'history',undefined,undefined,history).then(result=>{ + if(result.history===undefined)throw new Error('Local history synchronization is unavailable.');return result.history; + }); + } + private async call(action:OperationRequest['action'],id:string,event?:UplinkEvent,outcome?:Outcome,history?:BillingSnapshot):Promise { + const body:OperationRequest={v:1,type:'operation',requestId:randomUUID(),action,id,...(event?{event}:{}),...(outcome?{outcome}:{}),...(history?{history}:{})}; + parseOperation(JSON.stringify(body)); const response=await fetch(`http://127.0.0.1:${this.port}/operation`,{method:'POST',redirect:'error',signal:AbortSignal.timeout(this.timeoutMs), headers:{Authorization:`Bearer ${this.secret}`,'Content-Type':'application/json'},body:JSON.stringify(body)}); if(!response.ok)throw new Error('Local fallback is unavailable.'); @@ -45,6 +51,6 @@ export class LocalDecider implements Decider { }}finally{reader.releaseLock();} const result=parseOperation(Buffer.concat(chunks)); if(result.type!=='operation-result'||result.id!==id||result.requestId!==body.requestId||result.error)throw new Error('Local operation is unavailable.'); - return result.outcome; + return result; } } diff --git a/src/central/processor.ts b/src/central/processor.ts index 7cc11bf..d170ff8 100644 --- a/src/central/processor.ts +++ b/src/central/processor.ts @@ -12,17 +12,20 @@ export interface Outcome { active: boolean; decision: StateDownlink | null; snapshot: BillingSnapshot | null; } export interface Decider { + history?(gateway:string, action:HistoryAction, snapshot?:BillingSnapshot):Promise; offer(gateway: string, event: UplinkEvent): Promise; commit(gateway: string, id: string): Promise; outcome(gateway: string, id: string): Promise; reconcile?(gateway:string,event:UplinkEvent,outcome:Outcome):Promise; } +export type HistoryAction = 'history-read' | 'history-import' | 'history-ack'; type ProcessorRow = { id:string; gateway:string; fingerprint:string; event:string; state:Outcome['state']; decision:string|null; snapshot:string|null }; /** Both central and fallback use this wrapper around the SAME billing handler. * Offer does no billing. A durable processing claim precedes the handler. Once * claimed, a missing result is never permission to recompute or change deciders. */ export class DurableProcessor implements Decider { + history?: Decider['history']; private db: Database.Database; private active = new Map>(); private devices = new Map>(); diff --git a/src/central/rpc.ts b/src/central/rpc.ts index e2c1585..53af2e7 100644 --- a/src/central/rpc.ts +++ b/src/central/rpc.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'crypto'; import WebSocket from 'ws'; -import type { Decider, Outcome, BillingSnapshot } from './processor'; +import type { Decider, Outcome, BillingSnapshot, HistoryAction } from './processor'; import { parseMessage, UplinkEvent, MAX_MESSAGE_BYTES } from './protocol'; import { eventIdentity } from './operations'; import { createLogger } from '../utils/logger'; @@ -8,8 +8,8 @@ import { errorText } from '../utils/report-log'; const logger=createLogger('[central]'); -export type OperationRequest = {v:1;type:'operation';requestId:string;action:'offer'|'commit'|'outcome'|'reconcile';id:string;event?:UplinkEvent;outcome?:Outcome}; -export type OperationResponse = {v:1;type:'operation-result';requestId:string;id:string;outcome:Outcome|null;error:boolean}; +export type OperationRequest = {v:1;type:'operation';requestId:string;action:'offer'|'commit'|'outcome'|'reconcile'|HistoryAction;id:string;event?:UplinkEvent;outcome?:Outcome;history?:BillingSnapshot}; +export type OperationResponse = {v:1;type:'operation-result';requestId:string;id:string;outcome:Outcome|null;error:boolean;history?:BillingSnapshot|null}; const record=(value:unknown):value is Record=>typeof value==='object'&&value!==null&&!Array.isArray(value); const identifier=(value:unknown):value is string=>typeof value==='string'&&/^[\w-]{1,128}$/.test(value)&&!/[\s]/.test(value); const int=(v:unknown,min=0):v is number=>typeof v==='number'&&Number.isSafeInteger(v)&&v>=min; @@ -37,9 +37,16 @@ export function parseOperation(input:Buffer|string):OperationRequest|OperationRe const value:unknown=JSON.parse(input.toString()); if(!record(value)||value.v!==1||!identifier(value.requestId)||!identifier(value.id))throw new Error('Invalid operation envelope.'); if(value.type==='operation-result'){ - if(Object.keys(value).length!==6||typeof value.error!=='boolean'||(value.outcome!==null&&!validOutcome(value.outcome,value.id)))throw new Error('Invalid operation result.'); + if(Object.keys(value).length!==(value.history===undefined?6:7)||typeof value.error!=='boolean'||(value.outcome!==null&&!validOutcome(value.outcome,value.id))|| + (value.history!==undefined && (value.id!=='history'||value.outcome!==null||(value.history!==null&&!snapshot(value.history)))))throw new Error('Invalid operation result.'); return value as OperationResponse; } + if(value.type==='operation'&&['history-read','history-import','history-ack'].includes(String(value.action))){ + const hasBatch=value.action!=='history-read'; + if(value.id!=='history'||Object.keys(value).length!==(hasBatch?6:5)|| + (hasBatch&&(!snapshot(value.history)||!value.history.meter||value.history.transactions.length<1||value.history.transactions.length>100)))throw new Error('Invalid history batch.'); + return value as OperationRequest; + } if(value.type!=='operation'||!['offer','commit','outcome','reconcile'].includes(String(value.action)))throw new Error('Unknown operation.'); const wantsEvent=value.action==='offer'||value.action==='reconcile'; const wantsOutcome=value.action==='reconcile'; @@ -53,8 +60,14 @@ export function parseOperation(input:Buffer|string):OperationRequest|OperationRe } export async function serveOperation(processor:Decider,gateway:string,request:OperationRequest):Promise { let outcome:Outcome|null=null,error=false; + const isHistory=request.action.startsWith('history-'); + let history:BillingSnapshot|null=null; try { - if(request.action==='offer')await processor.offer(gateway,request.event!); + if(isHistory){ + if(!processor.history)throw new Error('History synchronization is unavailable.'); + history=await processor.history(gateway,request.action as HistoryAction,request.history); + } + else if(request.action==='offer')await processor.offer(gateway,request.event!); else if(request.action==='commit')outcome=await processor.commit(gateway,request.id); else if(request.action==='outcome')outcome=await processor.outcome(gateway,request.id); else if(processor.reconcile)await processor.reconcile(gateway,request.event!,request.outcome!); @@ -63,30 +76,35 @@ export async function serveOperation(processor:Decider,gateway:string,request:Op error=true; logger.warn(`Refused ${request.action} of report ${request.id} from gateway ${gateway}: ${errorText(failure)}`); } - return {v:1,type:'operation-result',requestId:request.requestId,id:request.id,outcome,error}; + return {v:1,type:'operation-result',requestId:request.requestId,id:request.id,outcome,error,...(isHistory?{history}:{})}; } /** One bounded RPC channel over the agent's persistent outbound socket. The * gateway identity is assigned by authentication, never by a frame's payload. */ export class SocketDecider implements Decider { - private pending=new Mapvoid;reject:(error:Error)=>void;timer:ReturnType}>(); + private pending=new Mapvoid;reject:(error:Error)=>void;timer:ReturnType}>(); constructor(private socket:()=>WebSocket|undefined,private timeoutMs=8000) {} offer(_gateway:string,event:UplinkEvent):Promise{return this.call('offer',eventIdentity(event),event).then(()=>{});} - commit(_gateway:string,id:string):Promise{return this.call('commit',id).then(value=>{if(!value)throw new Error('No stored outcome.');return value;});} - outcome(_gateway:string,id:string):Promise{return this.call('outcome',id);} + commit(_gateway:string,id:string):Promise{return this.call('commit',id).then(value=>{if(!value.outcome)throw new Error('No stored outcome.');return value.outcome;});} + outcome(_gateway:string,id:string):Promise{return this.call('outcome',id).then(value=>value.outcome);} reconcile(_gateway:string,event:UplinkEvent,outcome:Outcome):Promise{return this.call('reconcile',outcome.id,event,outcome).then(()=>{});} + history(_gateway:string,action:HistoryAction,history?:BillingSnapshot):Promise{ + return this.call(action,'history',undefined,undefined,history).then(value=>{ + if(value.history===undefined)throw new Error('Central does not support history synchronization.');return value.history; + }); + } receive(message:OperationResponse):void { const pending=this.pending.get(message.requestId);if(!pending)return; if(message.id!==pending.id){this.disconnect();return;} clearTimeout(pending.timer);this.pending.delete(message.requestId); - if(message.error)pending.reject(new Error('Operation refused or unavailable.'));else pending.resolve(message.outcome); + if(message.error)pending.reject(new Error('Operation refused or unavailable.'));else pending.resolve(message); } disconnect():void {for(const p of this.pending.values()){clearTimeout(p.timer);p.reject(new Error('Agent link disconnected.'));}this.pending.clear();} - private call(action:OperationRequest['action'],id:string,event?:UplinkEvent,outcome?:Outcome):Promise { + private call(action:OperationRequest['action'],id:string,event?:UplinkEvent,outcome?:Outcome,history?:BillingSnapshot):Promise { const socket=this.socket(); if(!socket||socket.readyState!==WebSocket.OPEN||this.pending.size>=64||socket.bufferedAmount>MAX_MESSAGE_BYTES*8)return Promise.reject(new Error('Agent link unavailable or busy.')); const requestId=randomUUID(); - const message:OperationRequest={v:1,type:'operation',requestId,action,id,...(event?{event}:{}),...(outcome?{outcome}:{})}; + const message:OperationRequest={v:1,type:'operation',requestId,action,id,...(event?{event}:{}),...(outcome?{outcome}:{}),...(history?{history}:{})}; parseOperation(JSON.stringify(message)); return new Promise((resolve,reject)=>{ const timer=setTimeout(()=>{this.pending.delete(requestId);reject(new Error('Operation timed out.'));},this.timeoutMs); diff --git a/src/central/runtime.ts b/src/central/runtime.ts index 2b0cb0a..bde0d3d 100644 --- a/src/central/runtime.ts +++ b/src/central/runtime.ts @@ -36,6 +36,13 @@ export async function initializeManagedRole():Promise { if(!config.development&&!config.trustedProxies.length)throw new Error('Configure an explicit TLS reverse proxy peer in CENTRAL_TRUSTED_PROXIES.'); configurePublications('.data/central.db'); processor=createBillingProcessor('.data/central.db'); + processor.history=async(_gateway,action,snapshot)=>{ + if(action!=='history-import'||!snapshot?.meter)throw new Error('Central only accepts history imports.'); + // Reuse the signature, identity, nonce-continuity and atomic merge checks. + // Central's recovery loop publishes only unconfirmed rows after chain sync. + await (await import('./billing')).importBillingSnapshot(snapshot,false); + return null; + }; let publishing=false; const recover=()=>{if(publishing)return;publishing=true; void import('../lib/sync').then(module=>module.getSyncedPendingTransactions()).then(distributeManaged).catch(()=>{}).finally(()=>{publishing=false;}); @@ -52,6 +59,13 @@ export async function initializeManagedRole():Promise { const identity=readOwnerSecret(config.credentialPath!).split(':')[0]; if(!/^[\w-]{36}$/.test(identity))throw new Error('Agent enrollment is not ready.');return identity; }; + const historyTarget=()=>`${config.centralUrl}|${getGateway()}`; + processor.history=async(_gateway,action,snapshot)=>{ + const store=await import('../store/sqlite'); + if(action==='history-read')return store.readGatewayHistory(historyTarget()); + if(action==='history-ack'&&snapshot){store.acknowledgeGatewayHistory(historyTarget(),snapshot);return null;} + throw new Error('Gateway history only supports export and acknowledgment.'); + }; const server=fallbackServer(processor,secret,getGateway); const port=gatewayFallbackPort(); if(!Number.isInteger(port)||port<1024||port>65535)throw new Error('GATEWAY_FALLBACK_PORT must be 1024–65535.'); @@ -64,6 +78,8 @@ export async function initializeManagedRole():Promise { }); server.listen(port,'127.0.0.1',()=>{server.removeAllListeners('error');resolve();}); }); + const pruning=(await import('./gateway-pruning')).startGatewayPruning(historyTarget); + server.once('close',()=>{void pruning.destroy();}); return server; } } diff --git a/src/services/mqtt.ts b/src/services/mqtt.ts index c685ad8..c822bcd 100644 --- a/src/services/mqtt.ts +++ b/src/services/mqtt.ts @@ -7,7 +7,7 @@ import { managedBilling } from '../central/billing-context'; import { encode } from '../lib/encode'; import { getAllTransactionRecords, getMeterByDevEui, getMeterByPublicKey, - getTransactionByNonce, recordMeterTransaction, saveMeter, updateMeterDevEui, updateMeterNonce, + getTransactionByNonce, getReplyTransaction, recordMeterTransaction, saveMeter, updateMeterDevEui, updateMeterNonce, } from '../store/sqlite'; import type { State, TransactionRecord } from '../types'; import { decodePayload } from '../lib/decode'; @@ -131,7 +131,7 @@ export async function handleMessage(blob: Buffer, downlink: DownlinkSink = chirp recordMeterTransaction(key, record); newTransaction = true; } else if (decoded.nonce !== 0) { - const previous = getTransactionByNonce(decoded.nonce, meter.tokenId); + const previous = getReplyTransaction(decoded.nonce, meter.tokenId); if (decoded.nonce !== meter.latestNonce || !previous || decoded.energy < decodePayload(Buffer.from(previous.raw, 'hex')).energy) { throw new Error(`Invalid nonce. Expected ${expectedNonce}, got ${decoded.nonce}`); } diff --git a/src/store/sqlite.ts b/src/store/sqlite.ts index 92f919a..35dacb6 100644 --- a/src/store/sqlite.ts +++ b/src/store/sqlite.ts @@ -36,6 +36,11 @@ export default function setupDatabase(databaseName = "m3tering.db") { initializeTransactionsTable(); initializeMetersTable(); + db.exec(`CREATE TABLE IF NOT EXISTS gateway_history_receipts ( + target TEXT NOT NULL, identifier INTEGER NOT NULL, nonce INTEGER NOT NULL, raw TEXT NOT NULL, + PRIMARY KEY(target, identifier, nonce)); + CREATE TABLE IF NOT EXISTS gateway_reply_checkpoints ( + identifier INTEGER PRIMARY KEY, nonce INTEGER NOT NULL, receivedAt INTEGER NOT NULL, raw TEXT NOT NULL);`); prepareQueries(); } @@ -285,7 +290,7 @@ export function getAllTransactionRecords(): TransactionRecord[] { export function readBillingSnapshot(devEui:string):BillingSnapshot { const meter=getMeterByDevEuiQuery.get(devEui) as MeterRecord|undefined; if(!meter)return {meter:null,transactions:[]}; - const transaction=getTransactionByNonceQuery.get(meter.latestNonce,meter.tokenId) as TransactionRecord|undefined; + const transaction=getReplyTransaction(meter.latestNonce,meter.tokenId); return {meter,transactions:transaction?[transaction]:[]}; } @@ -298,7 +303,7 @@ export function mergeBillingSnapshot(snapshot: BillingSnapshot): void { const existing = getMeterByPublicKeyQuery.get(meter.publicKey) as MeterRecord|undefined; if (existing && existing.tokenId !== meter.tokenId) throw new Error('Conflicting meter registration.'); for (const record of snapshot.transactions) { - const previous = getTransactionByNonceQuery.get(record.nonce, record.identifier) as TransactionRecord|undefined; + const previous = getReplyTransaction(record.nonce, record.identifier); if (previous && previous.raw !== record.raw) throw new Error('Conflicting transaction requires reconciliation; data was not overwritten.'); if (!previous) createTransactionQuery.run(record); } @@ -306,6 +311,63 @@ export function mergeBillingSnapshot(snapshot: BillingSnapshot): void { })(); } +/** Keep the last acknowledged reading outside the pending transaction table so + * a confirmed, pruned report can still be acknowledged if the device retries it. */ +export function getReplyTransaction(nonce: number, identifier: number): TransactionRecord | null { + return (getTransactionByNonceQuery.get(nonce, identifier) as TransactionRecord | undefined) + ?? (db.prepare('SELECT identifier,nonce,receivedAt,raw FROM gateway_reply_checkpoints WHERE identifier=? AND nonce=?') + .get(identifier, nonce) as TransactionRecord | undefined) ?? null; +} + +/** One meter per bounded batch; ascending nonces let central fill old gaps first. */ +export function readGatewayHistory(target: string): BillingSnapshot | null { + const first = db.prepare(`SELECT t.identifier FROM transactions t + JOIN meters m ON m.tokenId=t.identifier + WHERE NOT EXISTS (SELECT 1 FROM gateway_history_receipts r + WHERE r.target=? AND r.identifier=t.identifier AND r.nonce=t.nonce AND r.raw=t.raw) + ORDER BY t.identifier,t.nonce LIMIT 1`).get(target) as {identifier:number} | undefined; + if (!first) return null; + const meter = getMeterByTokenIdQuery.get(first.identifier) as MeterRecord; + const transactions = db.prepare(`SELECT t.* FROM transactions t WHERE t.identifier=? AND NOT EXISTS + (SELECT 1 FROM gateway_history_receipts r WHERE r.target=? AND r.identifier=t.identifier AND r.nonce=t.nonce AND r.raw=t.raw) + ORDER BY t.nonce LIMIT 100`).all(first.identifier, target) as TransactionRecord[]; + return { meter: {...meter, latestNonce: transactions[transactions.length - 1].nonce}, transactions }; +} + +/** An acknowledgment applies to the exact bytes central accepted, not a watermark. */ +export function acknowledgeGatewayHistory(target: string, snapshot: BillingSnapshot): void { + db.transaction(() => { + for (const record of snapshot.transactions) { + const local = getTransactionByNonceQuery.get(record.nonce, record.identifier) as TransactionRecord | undefined; + if (!local || local.raw !== record.raw) continue; + db.prepare(`INSERT INTO gateway_history_receipts VALUES (?,?,?,?) + ON CONFLICT(target,identifier,nonce) DO UPDATE SET raw=excluded.raw`) + .run(target, record.identifier, record.nonce, record.raw); + } + })(); +} + +/** Delete only chain-confirmed records whose exact contents reached this central. */ +export function pruneGatewayHistory(target: string, tokenId: number, confirmedNonce: number): number { + if (!Number.isSafeInteger(confirmedNonce) || confirmedNonce < 0) throw new Error('Invalid confirmed nonce.'); + return db.transaction(() => { + const meter = getMeterByTokenIdQuery.get(tokenId) as MeterRecord | undefined; + if (!meter) return 0; + const latest = getTransactionByNonceQuery.get(meter.latestNonce, tokenId) as TransactionRecord | undefined; + if (latest && latest.nonce <= confirmedNonce) { + db.prepare(`INSERT INTO gateway_reply_checkpoints(identifier,nonce,receivedAt,raw) VALUES (@identifier,@nonce,@receivedAt,@raw) + ON CONFLICT(identifier) DO UPDATE SET nonce=excluded.nonce,receivedAt=excluded.receivedAt,raw=excluded.raw`).run(latest); + } + const count = db.prepare(`DELETE FROM transactions WHERE identifier=? AND nonce<=? AND EXISTS + (SELECT 1 FROM gateway_history_receipts r WHERE r.target=? AND r.identifier=transactions.identifier + AND r.nonce=transactions.nonce AND r.raw=transactions.raw)`).run(tokenId, confirmedNonce, target).changes; + db.prepare(`DELETE FROM gateway_history_receipts WHERE identifier=? AND NOT EXISTS + (SELECT 1 FROM transactions t WHERE t.identifier=gateway_history_receipts.identifier AND t.nonce=gateway_history_receipts.nonce)`) + .run(tokenId); + return count; + })(); +} + export function pruneTransactionsBefore(nonce: number, meterNumber: number) { try { const result = db diff --git a/tests/services/central-link.test.ts b/tests/services/central-link.test.ts index 0c78221..8661622 100644 --- a/tests/services/central-link.test.ts +++ b/tests/services/central-link.test.ts @@ -11,6 +11,7 @@ import { LocalDecider, fallbackServer } from '../../src/central/local-channel'; import { protectSessionSockets, registerSocket } from '../../src/services/socket-router'; import { hashPassword } from '../../src/central/secrets'; import type { UplinkEvent } from '../../src/central/protocol'; +import { syncGatewayHistory } from '../../src/central/history'; const report:UplinkEvent={deduplicationId:'wire-report',fPort:2,deviceInfo:{applicationId:'11111111-1111-4111-8111-111111111111',devEui:'0102030405060708'},data:Buffer.alloc(74).toString('base64')}; const start=async(server:http.Server)=>{await new Promise(resolve=>server.listen(0,'127.0.0.1',resolve));return `http://127.0.0.1:${(server.address() as import('net').AddressInfo).port}`;}; @@ -30,7 +31,9 @@ async function fixture(){ const response=await fetch(base+'/api/agent/enroll',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({token})}); const enrolled=await response.json() as {gateway:string;credential:string}; const sockets:WebSocket[]=[]; - const connect=async()=>{const ws=new WebSocket(base.replace('http:','ws:')+'/agent',{headers:{authorization:`Bearer ${enrolled.credential}`}});sockets.push(ws);await once(ws,'open');return ws;}; + const connect=async()=>{const ws=new WebSocket(base.replace('http:','ws:')+'/agent',{headers:{authorization:`Bearer ${enrolled.credential}`}}); + ws.on('upgrade',response=>expect(response.headers['x-console-history']).toBe('1')); + sockets.push(ws);await once(ws,'open');return ws;}; return {store,cookie,processor,hub,compute,base,enrolled,token,connect,close:async()=>{ for(const ws of sockets)ws.terminate();for(const ws of hub.sockets.values())ws.terminate(); terminal.close();await close(server);processor.close();store.close(); @@ -58,6 +61,35 @@ test('authenticated outbound link returns the same stored decision by ChirpStack }finally{client?.disconnect();await f.close();} }); +test('historical batches traverse authenticated local HTTP and the central socket without billing a new report',async()=>{ + const f=await fixture(); + const local=new DurableProcessor(':memory:',jest.fn()); + const batch={meter:{publicKey:`0x${'ab'.repeat(32)}`,tokenId:7,devEui:'0102030405060708',latestNonce:2}, + transactions:[{identifier:7,nonce:2,receivedAt:1,raw:'00'.repeat(74)}]}; + let acknowledged=false; + local.history=jest.fn(async(_gateway,action)=>{ + if(action==='history-read')return acknowledged?null:batch; + acknowledged=true;return null; + }); + f.processor.history=jest.fn(async()=>null); + const server=fallbackServer(local,'history-test-secret',f.enrolled.gateway); + const base=await start(server); + let client:SocketDecider|undefined; + try{ + const ws=await f.connect();client=new SocketDecider(()=>ws,1000); + ws.on('message',data=>{const response=parseOperation(data.toString());if(response.type==='operation-result')client!.receive(response);}); + const fallback=new LocalDecider(Number(new URL(base).port),'history-test-secret',1000); + expect(await syncGatewayHistory(f.enrolled.gateway,fallback,client)).toBe(1); + expect(f.processor.history).toHaveBeenCalledWith(f.enrolled.gateway,'history-import',batch); + expect(f.store.devices()).toEqual(expect.arrayContaining([expect.objectContaining({devEui:batch.meter.devEui,gateway:f.enrolled.gateway})])); + expect(await syncGatewayHistory(f.enrolled.gateway,fallback,client)).toBe(0); + expect(f.compute).not.toHaveBeenCalled(); + const unauthenticated=await fetch(base+'/operation',{method:'POST',headers:{'Content-Type':'application/json'}, + body:JSON.stringify({v:1,type:'operation',requestId:'test',id:'history',action:'history-read'})}); + expect(unauthenticated.status).toBe(401); + }finally{client?.disconnect();await close(server);local.close();await f.close();} +}); + test('rename updates the list; delete disconnects the agent and removes the gateway',async()=>{ const f=await fixture(); try{ diff --git a/tests/services/deployment.test.ts b/tests/services/deployment.test.ts index b3cdfdf..de1c447 100644 --- a/tests/services/deployment.test.ts +++ b/tests/services/deployment.test.ts @@ -297,3 +297,27 @@ test('the role lines in .env are updated in place, keep the file owner-only, and expect(fs.statSync(path.join(dir,'.env')).mode&0o777).toBe(0o600); } finally {fs.rmSync(dir,{recursive:true,force:true});} }); + +test('prover-batch.sh runs the batch script in the running Console, using the role\'s compose file', () => { + const dir=fs.mkdtempSync(path.join(os.tmpdir(),'console-prover-')); + try { + fs.copyFileSync('prover-batch.sh',path.join(dir,'prover-batch.sh'));fs.chmodSync(path.join(dir,'prover-batch.sh'),0o755); + fs.mkdirSync(path.join(dir,'scripts'));fs.writeFileSync(path.join(dir,'scripts','prover-batch.cjs'),'// batch script\n'); + const log=path.join(dir,'calls'); + fs.writeFileSync(path.join(dir,'docker'),`#!/bin/bash +printf '%s\\n' "$*" >> "$DEPLOY_LOG" +case "$*" in *' ps -q console') [[ "$RUNNING" == true ]] && echo container ;; *' console node -') cat >> "$DEPLOY_LOG" ;; esac +`,{mode:0o755}); + const run=(env:string,running=true,args:string[]=[])=>{fs.writeFileSync(path.join(dir,'.env'),env);fs.writeFileSync(log,''); + const r=spawnSync('bash',['prover-batch.sh',...args],{cwd:dir,encoding:'utf8',env:{...process.env,PATH:dir+':'+process.env.PATH,DEPLOY_LOG:log,RUNNING:String(running)}}); + return {r,calls:fs.readFileSync(log,'utf8')};}; + let {r,calls}=run('CONSOLE_ROLE=central\nCOMPOSE_FILE=compose.central.yaml\nCOMPOSE_PROJECT_NAME=console\n'); + expect(r.status).toBe(0); + expect(calls).toContain('compose -f compose.central.yaml --project-name console exec -T -e PROVER_BATCH_LIMIT=100 console node -'); + expect(calls).toContain('// batch script'); + ({r,calls}=run('CONSOLE_ROLE=central\n',true,['--limit','250']));expect(calls).toContain('compose -f compose.central.yaml exec -T -e PROVER_BATCH_LIMIT=250 console node -'); + ({r,calls}=run('API_TOKEN=x\n',true,['--all']));expect(calls).toContain('compose -f compose.yaml exec -T -e PROVER_BATCH_LIMIT=0 console node -'); + for(const bad of [['--limit','0'],['--limit','x'],['--nope']]){({r,calls}=run('CONSOLE_ROLE=central\n',true,bad));expect(r.status).toBe(2);expect(calls).toBe('');} + ({r,calls}=run('CONSOLE_ROLE=central\n',false));expect(r.status).toBe(1);expect(r.stderr).toContain("isn't running");expect(calls).not.toContain('exec -T'); + } finally {fs.rmSync(dir,{recursive:true,force:true});} +}); diff --git a/tests/services/gateway-pruning.test.ts b/tests/services/gateway-pruning.test.ts new file mode 100644 index 0000000..1c813b4 --- /dev/null +++ b/tests/services/gateway-pruning.test.ts @@ -0,0 +1,31 @@ +jest.mock('../../src/utils/contracts',()=>({meterLatestNonceContractCall:jest.fn()})); +jest.mock('../../src/store/sqlite',()=>({getAllMeterRecords:jest.fn(),pruneGatewayHistory:jest.fn()})); +jest.mock('node-cron',()=>({validate:jest.fn(()=>true),schedule:jest.fn(()=>({destroy:jest.fn()}))})); +import cron from 'node-cron'; +import { getAllMeterRecords, pruneGatewayHistory } from '../../src/store/sqlite'; +import { meterLatestNonceContractCall } from '../../src/utils/contracts'; +import { pruneConfirmedGatewayHistory, startGatewayPruning } from '../../src/central/gateway-pruning'; + +beforeEach(()=>{jest.clearAllMocks();}); + +test('a failed confirmation read keeps that meter while other meters can prune',async()=>{ + jest.mocked(getAllMeterRecords).mockReturnValue([{tokenId:7},{tokenId:8}] as any); + jest.mocked(meterLatestNonceContractCall).mockRejectedValueOnce(new Error('offline')).mockResolvedValueOnce(10); + jest.mocked(pruneGatewayHistory).mockReturnValue(3); + expect(await pruneConfirmedGatewayHistory('central|gateway')).toBe(3); + expect(pruneGatewayHistory).toHaveBeenCalledTimes(1); + expect(pruneGatewayHistory).toHaveBeenCalledWith('central|gateway',8,10); +}); + +test('the default and configurable cron run in UTC without overlapping',()=>{ + const previous=process.env.GATEWAY_PRUNE_CRONSCHEDULE; + try{ + delete process.env.GATEWAY_PRUNE_CRONSCHEDULE; + startGatewayPruning(()=> 'target'); + expect(cron.schedule).toHaveBeenLastCalledWith('*/5 * * * *',expect.any(Function),expect.objectContaining({noOverlap:true,timezone:'UTC'})); + process.env.GATEWAY_PRUNE_CRONSCHEDULE='0 * * * *';startGatewayPruning(()=> 'target'); + expect(cron.schedule).toHaveBeenLastCalledWith('0 * * * *',expect.any(Function),expect.any(Object)); + jest.mocked(cron.validate).mockReturnValueOnce(false); + expect(()=>startGatewayPruning(()=> 'target')).toThrow('GATEWAY_PRUNE_CRONSCHEDULE'); + }finally{if(previous===undefined)delete process.env.GATEWAY_PRUNE_CRONSCHEDULE;else process.env.GATEWAY_PRUNE_CRONSCHEDULE=previous;} +}); diff --git a/tests/services/history-import.test.ts b/tests/services/history-import.test.ts new file mode 100644 index 0000000..8bca0cf --- /dev/null +++ b/tests/services/history-import.test.ts @@ -0,0 +1,34 @@ +jest.mock('../../src/services/mqtt',()=>({handleMessage:jest.fn()})); +jest.mock('../../src/utils/contracts',()=>({meterIdFromPublicKeyContractCall:jest.fn(async()=>7),meterLatestNonceContractCall:jest.fn(async()=>0)})); +jest.mock('../../src/lib/utils',()=>({...jest.requireActual('../../src/lib/utils'),distributeManaged:jest.fn(async()=>{})})); +import { generateKeyPairSync, sign } from 'crypto'; +import setupDatabase, { saveMeter, insertTransaction, getAllTransactionRecords, getMeterByTokenId, deleteDatabase } from '../../src/store/sqlite'; +import { importBillingSnapshot } from '../../src/central/billing'; +import { distributeManaged } from '../../src/lib/utils'; +import type { BillingSnapshot } from '../../src/central/processor'; + +const keys=generateKeyPairSync('ed25519'); +const key='0x'+keys.publicKey.export({format:'der',type:'spki'}).subarray(-32).toString('hex'); +const meter={publicKey:key,devEui:'0102030405060708',tokenId:7,latestNonce:3}; +const record=(nonce:number,energy=1000000)=>{ + const bytes=Buffer.alloc(74);bytes.writeUInt32BE(nonce);bytes.writeUInt32BE(energy,4); + sign(null,bytes.subarray(0,8),keys.privateKey).copy(bytes,8); + return {nonce,identifier:7,receivedAt:1,raw:bytes.toString('hex')}; +}; +const batch:BillingSnapshot={meter:{...meter,latestNonce:2},transactions:[record(1),record(2)]}; +beforeEach(()=>{jest.clearAllMocks();setupDatabase('history-import-test.db');saveMeter(meter);insertTransaction(record(3));}); +afterEach(()=>deleteDatabase('history-import-test.db')); + +test('central fills missing older history idempotently without rolling back its latest nonce or publishing confirmed history immediately',async()=>{ + await importBillingSnapshot(batch,false);await importBillingSnapshot(batch,false); + expect(getAllTransactionRecords().map(r=>r.nonce).sort()).toEqual([1,2,3]); + expect(getMeterByTokenId(7)!.latestNonce).toBe(3); + expect(distributeManaged).not.toHaveBeenCalled(); +}); + +test('a bad signature or conflicting signed record fails the batch without partially inserting history',async()=>{ + await expect(importBillingSnapshot({...batch,transactions:[record(1),{...record(2),raw:'00'.repeat(74)}]},false)).rejects.toThrow('Invalid signed'); + expect(getAllTransactionRecords().map(r=>r.nonce)).toEqual([3]); + await expect(importBillingSnapshot({...batch,transactions:[record(1),record(3,2000000)]},false)).rejects.toThrow('Conflicting transaction'); + expect(getAllTransactionRecords().map(r=>r.nonce)).toEqual([3]); +}); diff --git a/tests/services/history.test.ts b/tests/services/history.test.ts new file mode 100644 index 0000000..cc5b0ba --- /dev/null +++ b/tests/services/history.test.ts @@ -0,0 +1,58 @@ +import { syncGatewayHistory } from '../../src/central/history'; +import { parseOperation, serveOperation } from '../../src/central/rpc'; +import type { BillingSnapshot, Decider } from '../../src/central/processor'; + +const batch:BillingSnapshot={meter:{publicKey:`0x${'ab'.repeat(32)}`,devEui:'0102030405060708',tokenId:7,latestNonce:2}, + transactions:[1,2].map(nonce=>({nonce,identifier:7,receivedAt:1,raw:'00'.repeat(74)}))}; +const decider=():Decider=>({offer:jest.fn(),commit:jest.fn(),outcome:jest.fn(),history:jest.fn()}); + +test('a lost central response keeps the batch for idempotent retry; no device operation is created',async()=>{ + const local=decider(),central=decider(); + const imported=new Map(); + let acknowledged=false,lost=true; + local.history=jest.fn(async(_gateway,action)=>{ + if(action==='history-read')return acknowledged?null:batch; + acknowledged=true;return null; + }); + central.history=jest.fn(async(_gateway,_action,snapshot)=>{ + for(const record of snapshot!.transactions)imported.set(record.nonce,record.raw); + if(lost){lost=false;throw new Error('Response lost after commit');}return null; + }); + await expect(syncGatewayHistory('gateway',local,central)).rejects.toThrow('Response lost'); + expect(acknowledged).toBe(false); + expect(await syncGatewayHistory('gateway',local,central)).toBe(2); + expect(imported.size).toBe(2); + expect(await syncGatewayHistory('gateway',local,central)).toBe(0); + expect(central.offer).not.toHaveBeenCalled();expect(central.commit).not.toHaveBeenCalled(); +}); + +test('a lost local acknowledgment response retries the same accepted batch safely',async()=>{ + const local=decider(),central=decider();let lost=true; + local.history=jest.fn(async(_gateway,action)=>{ + if(action==='history-read')return batch; + if(lost){lost=false;throw new Error('Local response lost');}return null; + }); + central.history=jest.fn(async()=>null); + await expect(syncGatewayHistory('gateway',local,central)).rejects.toThrow(); + expect(await syncGatewayHistory('gateway',local,central)).toBe(2); + expect(central.history).toHaveBeenCalledTimes(2); +}); + +test('history protocol bounds batches, rejects malformed data and retains authenticated gateway identity',async()=>{ + const request={v:1,type:'operation',requestId:'request',id:'history',action:'history-import',history:batch}; + const parsed=parseOperation(JSON.stringify(request)); + if(parsed.type!=='operation')throw new Error(); + const processor=decider();processor.history=jest.fn(async()=>null); + const response=await serveOperation(processor,'authenticated-gateway',parsed); + expect(processor.history).toHaveBeenCalledWith('authenticated-gateway','history-import',batch); + expect(parseOperation(JSON.stringify(response))).toMatchObject({error:false,history:null}); + for(const invalid of [ + {...request,gateway:'spoofed'}, + {...request,history:{...batch,transactions:Array(101).fill(batch.transactions[0])}}, + {...request,history:{...batch,transactions:[{...batch.transactions[0],raw:'invalid'}]}}, + {...request,history:{meter:null,transactions:[]}}, + {...request,action:'history-read'}, + ])expect(()=>parseOperation(JSON.stringify(invalid))).toThrow(); + delete processor.history; + expect(await serveOperation(processor,'gateway',parsed)).toMatchObject({error:true,history:null}); +}); diff --git a/tests/services/mqtt.test.ts b/tests/services/mqtt.test.ts index 5fec5f7..65a26fe 100644 --- a/tests/services/mqtt.test.ts +++ b/tests/services/mqtt.test.ts @@ -6,7 +6,7 @@ import { enqueue } from '../../src/services/grpc'; jest.mock('mqtt', () => ({ connect: jest.fn() })); jest.mock('../../src/services/grpc', () => ({ enqueue: jest.fn() })); -jest.mock('../../src/store/sqlite', () => ({ getAllTransactionRecords: jest.fn(), getMeterByDevEui: jest.fn(), getMeterByPublicKey: jest.fn(), getTransactionByNonce: jest.fn(), recordMeterTransaction: jest.fn(), saveMeter: jest.fn(), updateMeterDevEui: jest.fn(), updateMeterNonce: jest.fn() })); +jest.mock('../../src/store/sqlite', () => ({ getAllTransactionRecords: jest.fn(), getMeterByDevEui: jest.fn(), getMeterByPublicKey: jest.fn(), getTransactionByNonce: jest.fn(), getReplyTransaction: jest.fn(), recordMeterTransaction: jest.fn(), saveMeter: jest.fn(), updateMeterDevEui: jest.fn(), updateMeterNonce: jest.fn() })); jest.mock('../../src/lib/sync', () => ({ getLatestTransactionNonce: jest.fn() })); jest.mock('../../src/utils/contracts', () => ({ meterIdFromPublicKeyContractCall: jest.fn() })); jest.mock('../../src/lib/utils', () => ({ runHook: jest.fn(), notifyHook: jest.fn(), verifyPayloadSignature: jest.fn() })); @@ -34,6 +34,7 @@ function resetFixture() { (store.updateMeterDevEui as jest.Mock).mockReturnValue(true); (store.getAllTransactionRecords as jest.Mock).mockImplementation(() => records); (store.getTransactionByNonce as jest.Mock).mockImplementation(nonce => records.find(record => record.nonce === nonce)); + (store.getReplyTransaction as jest.Mock).mockImplementation(nonce => records.find(record => record.nonce === nonce)); (store.recordMeterTransaction as jest.Mock).mockImplementation((_, record) => { records.push(record); meter.latestNonce = record.nonce; }); } beforeEach(resetFixture); @@ -90,6 +91,16 @@ test('bootstrap messages use the balance hook and computation failures return of expect((enqueue as jest.Mock).mock.calls[0][1][0]).toBe(0); expect(store.recordMeterTransaction).not.toHaveBeenCalled(); }); + +test('a confirmed report pruned from transactions is acknowledged using its saved checkpoint',async()=>{ + await handleMessage(uplink()); + const checkpoint=records[0];records=[]; + (store.getReplyTransaction as jest.Mock).mockImplementation(nonce=>nonce===1?checkpoint:null); + await handleMessage(uplink(1,'device-a',2000000)); + expect(enqueue).toHaveBeenCalledTimes(2); + expect(store.recordMeterTransaction).toHaveBeenCalledTimes(1); + expect(records).toEqual([]); +}); test('persistence failure never acknowledges uncommitted data and releases the lock', async () => { (store.recordMeterTransaction as jest.Mock).mockImplementationOnce(() => { throw new Error('disk full'); }); await handleMessage(uplink()); expect(enqueue).not.toHaveBeenCalled(); diff --git a/tests/store/gateway-history.test.ts b/tests/store/gateway-history.test.ts new file mode 100644 index 0000000..110c74d --- /dev/null +++ b/tests/store/gateway-history.test.ts @@ -0,0 +1,69 @@ +import setupDatabase, { + saveMeter, insertTransaction, getAllTransactionRecords, getMeterByTokenId, deleteDatabase, + readGatewayHistory, acknowledgeGatewayHistory, pruneGatewayHistory, getReplyTransaction, + readBillingSnapshot, mergeBillingSnapshot, +} from '../../src/store/sqlite'; + +const filename = 'gateway-history-test.db'; +const target = 'wss://central.example|gateway-a'; +const meter = { publicKey: `0x${'ab'.repeat(32)}`, tokenId: 7, devEui: '0102030405060708', latestNonce: 205 }; +const record = (nonce:number, identifier=7) => ({nonce, identifier, receivedAt:1, raw:nonce.toString(16).padStart(148,'0')}); + +beforeEach(() => { setupDatabase(filename); saveMeter(meter); }); +afterEach(() => deleteDatabase(filename)); + +test('exports all historical rows in bounded ordered batches and persists exact acknowledgments across restart', () => { + for(let nonce=205;nonce>=1;nonce--)insertTransaction(record(nonce)); + const first=readGatewayHistory(target)!; + expect(first.transactions).toHaveLength(100); + expect(first.transactions.map(r=>r.nonce)).toEqual(Array.from({length:100},(_,i)=>i+1)); + expect(first.meter!.latestNonce).toBe(100); + acknowledgeGatewayHistory(target,first); + setupDatabase(filename); + const second=readGatewayHistory(target)!; + expect(second.transactions[0].nonce).toBe(101); + acknowledgeGatewayHistory(target,second); + const third=readGatewayHistory(target)!; + expect(third.transactions.map(r=>r.nonce)).toEqual([201,202,203,204,205]); + acknowledgeGatewayHistory(target,third); + expect(readGatewayHistory(target)).toBeNull(); + // A different central/enrollment needs its own transfer, not old receipts. + expect(readGatewayHistory('wss://other.example|gateway-b')!.transactions[0].nonce).toBe(1); +}); + +test('pruning includes the confirmed boundary, keeps unconfirmed and unacknowledged rows, and is scoped to the meter', () => { + [1,2,3,4].forEach(n=>insertTransaction(record(n))); + insertTransaction(record(1,8)); + const batch=readGatewayHistory(target)!; + acknowledgeGatewayHistory(target,{...batch,transactions:batch.transactions.filter(r=>r.nonce!==2)}); + expect(pruneGatewayHistory('wrong-central',7,3)).toBe(0); + expect(pruneGatewayHistory(target,7,3)).toBe(2); + expect(getAllTransactionRecords().map(r=>[r.identifier,r.nonce])).toEqual([[7,2],[7,4],[8,1]]); + expect(getMeterByTokenId(7)!.latestNonce).toBe(205); + expect(()=>pruneGatewayHistory(target,7,NaN)).toThrow(); +}); + +test('an acknowledgment for different bytes never permits pruning', () => { + insertTransaction(record(1)); + const batch=readGatewayHistory(target)!; + acknowledgeGatewayHistory(target,{...batch,transactions:[{...record(1),raw:'ff'.repeat(74)}]}); + expect(pruneGatewayHistory(target,7,100)).toBe(0); + expect(readGatewayHistory(target)!.transactions).toEqual([record(1)]); +}); + +test('the last reply survives inclusive pruning and restart without remaining in the transaction table', () => { + saveMeter({...meter,latestNonce:2}); + insertTransaction(record(1));insertTransaction(record(2)); + const batch=readGatewayHistory(target)!; + acknowledgeGatewayHistory(target,batch); + expect(pruneGatewayHistory(target,7,2)).toBe(2); + setupDatabase(filename); + expect(getAllTransactionRecords()).toEqual([]); + expect(getReplyTransaction(2,7)).toEqual(record(2)); + expect(readBillingSnapshot(meter.devEui).transactions).toEqual([record(2)]); + expect(getReplyTransaction(1,7)).toBeNull(); + // Replaying the same outcome must not put the boundary back into pending data. + mergeBillingSnapshot({...batch,transactions:[record(2)]}); + expect(getAllTransactionRecords()).toEqual([]); + expect(()=>mergeBillingSnapshot({...batch,transactions:[{...record(2),raw:'ff'}]})).toThrow('Conflicting transaction'); +});