From 7af44ae7df9fef7ed9a934111710f636c9702fde Mon Sep 17 00:00:00 2001 From: Anthony Ettinger Date: Mon, 19 Oct 2026 13:16:55 +0000 Subject: [PATCH] @profullstack/deploy 0.1.0: deploy in batches, 3 at a time deploy-gate (bash + flock): every deploy on a box takes one of N slots (default 3) shared by root and every user, waits for free memory, runs with the lock closed so daemons never hold it; nested gates pass through; exit 75 on timeout. pfdeploy batch / batch(): roll one change out to many repos N at a time, stop at the first failure. Zero dependencies. Born of the dev2 OOM of 2026-10-19. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci.yml | 16 ++++ .gitignore | 2 + LICENSE | 21 ++++++ README.md | 69 +++++++++++++++++ bin/deploy-gate | 156 +++++++++++++++++++++++++++++++++++++++ bin/pfdeploy.js | 102 +++++++++++++++++++++++++ index.d.ts | 33 +++++++++ package.json | 32 ++++++++ src/index.js | 109 +++++++++++++++++++++++++++ test/deploy.test.js | 150 +++++++++++++++++++++++++++++++++++++ 10 files changed, 690 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 README.md create mode 100755 bin/deploy-gate create mode 100755 bin/pfdeploy.js create mode 100644 index.d.ts create mode 100644 package.json create mode 100644 src/index.js create mode 100644 test/deploy.test.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..b498c9d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,16 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npm test diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fee63cc --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.claude/ +node_modules/ diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..84ffdee --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Profullstack, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..8fb1661 --- /dev/null +++ b/README.md @@ -0,0 +1,69 @@ +# @profullstack/deploy + +Deploy in batches, **3 at a time**, so one box never builds everything at once. + +On 2026-10-19 a change rolled out to 38 repos at once on our production box. Every repo installed, tested, type-checked and built together, then every merge's deploy built again; the box ran out of memory and the kernel killed the shared production Postgres. This package makes that impossible from both ends: + +- **`deploy-gate`** (bash + `flock`, on the box): every deploy takes one of N slots before it builds. Merges can land all at once; builds queue and run three at a time. +- **`pfdeploy batch`** (and `batch()`, where the rollout is driven): run one command across many repos, three at a time, stopping at the first failure. + +Zero dependencies. Linux (`flock`, `/proc/meminfo`); the batch runner works anywhere Node 18+ does. + +## On the box: the gate + +```sh +npx @profullstack/deploy install /usr/local/bin # as root: one copy every user's deploys share +deploy-gate --name crawlproof -- ./deploy-app.sh "$SHA" +deploy-gate status +``` + +``` +slot 0 busy crawlproof pid 41233 since 2026-10-19T13:20:02Z +slot 1 busy smshub pid 41290 since 2026-10-19T13:20:09Z +slot 2 free +2 of 3 busy, 31 GB available +``` + +How it decides: + +1. Take one of `DEPLOY_SLOTS` (3) lock files in `/run/lock/profullstack-deploy` (shared by root and every user). All taken: wait, saying who holds them every 30 s. +2. Wait up to `DEPLOY_MEM_WAIT` (600 s) for `DEPLOY_MIN_FREE_GB` (8) of available memory, then go on. +3. Run the deploy with the lock descriptor closed, so a server it starts never keeps the slot. Signals are forwarded; the exit code is the deploy's. +4. Still no slot after `DEPLOY_WAIT` (3600 s): exit 75 (try again), or run anyway with `DEPLOY_GATE_ON_TIMEOUT=run`. + +A deploy already inside the gate (`DEPLOY_GATED=1`) passes straight through, so a gated script that calls another gated script never waits on itself. Without `flock` the gate runs the deploy ungated rather than blocking it. + +### Wiring a deploy script + +Put this right after `set -euo pipefail`; if the gate is not installed the script runs exactly as before: + +```sh +# Three deploys at a time on this box (@profullstack/deploy). +if [ -z "${DEPLOY_GATED:-}" ]; then + for gate in /usr/local/bin/deploy-gate "$HOME/.local/bin/deploy-gate"; do + [ -x "$gate" ] && exec "$gate" --name "" -- "$0" "$@" + done +fi +``` + +## Where the rollout runs: batches + +```sh +# one change, many repos, three at a time; {} is the item, shell-quoted +pfdeploy batch -c 3 -- 'cd ~/src/profullstack/{} && bun test && gh pr merge --squash --auto' crawlproof smshub ugig.net +ls ~/www | pfdeploy batch -- '~/www/{}/deploy-app.sh origin/main' +pfdeploy batch -c 3 --keep-going --min-free-gb 12 -- 'make -C {} build' < repos.txt +``` + +Output lines are prefixed with the item. The first failure stops new starts (running ones finish); the summary names what failed and what never started. `--keep-going` runs everything. + +```js +import { batch } from '@profullstack/deploy'; + +const results = await batch(repos, (repo) => rollOut(repo), { concurrency: 3, minFreeGb: 8 }); +// [{ item, ok, value | error, skipped }] +``` + +## License + +MIT diff --git a/bin/deploy-gate b/bin/deploy-gate new file mode 100755 index 0000000..434bfbd --- /dev/null +++ b/bin/deploy-gate @@ -0,0 +1,156 @@ +#!/usr/bin/env bash +# deploy-gate: at most N deploys build at once on this box, whoever starts them. +# +# deploy-gate [--slots 3] [--name site] [--wait 3600] [--min-free-gb 8] -- command [args...] +# deploy-gate status +# +# Every deploy path on the box (cli-tools deploy-app.sh, OpenInstall bin/install.sh, +# a hand-run build) wraps itself in this. A deploy takes one of N slots, a file +# lock under DEPLOY_LOCK_DIR shared by every user, waits while all are taken, +# then waits a little for free memory, then runs. Thirty-eight merges at once +# become three builds at a time and a queue, instead of an out-of-memory kill of +# the production database (dev2, 2026-10-19). +# +# The command runs with the lock's descriptor closed, so a daemon it starts never +# holds the slot. A deploy already inside the gate (DEPLOY_GATED=1) passes +# straight through, so nested scripts never wait on themselves. +# +# Environment (flags win): DEPLOY_SLOTS=3, DEPLOY_WAIT=3600 (seconds for a slot; +# then exit 75, or run anyway with DEPLOY_GATE_ON_TIMEOUT=run), DEPLOY_MIN_FREE_GB=8, +# DEPLOY_MEM_WAIT=600 (seconds to wait for memory, then go on), DEPLOY_POLL=5, DEPLOY_LOCK_DIR. +set -uo pipefail + +SLOTS=${DEPLOY_SLOTS:-3} +WAIT=${DEPLOY_WAIT:-3600} +MIN_FREE_GB=${DEPLOY_MIN_FREE_GB:-8} +MEM_WAIT=${DEPLOY_MEM_WAIT:-600} +POLL=${DEPLOY_POLL:-5} +LOCK_DIR=${DEPLOY_LOCK_DIR:-/run/lock/profullstack-deploy} +NAME="" + +say() { printf '[deploy-gate] %s\n' "$*" >&2; } +usage() { sed -n '4,6p' "$0" | sed 's/^# \{0,1\}//' >&2; exit 64; } + +lock_dir() { + if [ ! -d "$LOCK_DIR" ]; then + (umask 000; mkdir -p "$LOCK_DIR") 2>/dev/null || { say "cannot make $LOCK_DIR"; return 1; } + chmod 1777 "$LOCK_DIR" 2>/dev/null || true + fi +} + +slot_file() { printf '%s/slot-%s' "$LOCK_DIR" "$1"; } + +# The holder line written into a slot: who, since when. +holder() { head -c 200 "$(slot_file "$1")" 2>/dev/null | head -1; } + +status() { + lock_dir || exit 1 + local i f busy=0 + for ((i = 0; i < SLOTS; i++)); do + f=$(slot_file "$i") + [ -e "$f" ] || (umask 000; : > "$f") + if flock -n "$f" true 2>/dev/null; then + printf 'slot %d free\n' "$i" + else + busy=$((busy + 1)) + printf 'slot %d busy %s\n' "$i" "$(holder "$i")" + fi + done + printf '%d of %d busy, %s GB available\n' "$busy" "$SLOTS" "$(free_gb)" +} + +free_gb() { awk '/^MemAvailable:/ { printf "%d", $2 / 1048576 }' /proc/meminfo 2>/dev/null || echo 0; } + +[ "${1:-}" = status ] && { shift; [ "${1:-}" = --slots ] && SLOTS=$2; status; exit 0; } + +while [ $# -gt 0 ]; do + case "$1" in + --slots) SLOTS=$2; shift 2 ;; + --name) NAME=$2; shift 2 ;; + --wait) WAIT=$2; shift 2 ;; + --min-free-gb) MIN_FREE_GB=$2; shift 2 ;; + --) shift; break ;; + -h|--help) usage ;; + *) break ;; + esac +done +[ $# -gt 0 ] || usage +case "$SLOTS" in ''|*[!0-9]*|0) say "--slots must be a positive number"; exit 64 ;; esac + +# Already inside the gate: never take a second slot. +if [ "${DEPLOY_GATED:-}" = 1 ]; then exec "$@"; fi +NAME=${NAME:-$(basename "$PWD")} +export DEPLOY_GATED=1 + +if ! command -v flock >/dev/null || ! lock_dir; then + say "no flock or lock dir: running ungated" + exec "$@" +fi + +# Take a slot. Start at a random slot so waiters spread out; poll every DEPLOY_POLL (5) s. +fd="" +start=$(date +%s) +last_note=0 +while :; do + first=$((RANDOM % SLOTS)) + for ((n = 0; n < SLOTS; n++)); do + i=$(((first + n) % SLOTS)) + f=$(slot_file "$i") + [ -e "$f" ] || (umask 000; : > "$f") 2>/dev/null + { exec {fd}<>"$f"; } 2>/dev/null || continue + if flock -n "$fd"; then + slot=$i + break 2 + fi + exec {fd}>&- + fd="" + done + now=$(date +%s) + if [ $((now - start)) -ge "$WAIT" ]; then + if [ "${DEPLOY_GATE_ON_TIMEOUT:-fail}" = run ]; then + say "waited ${WAIT}s for a slot; running anyway (DEPLOY_GATE_ON_TIMEOUT=run)" + exec "$@" + fi + say "waited ${WAIT}s for one of $SLOTS deploy slots; giving up (exit 75). Busy: $(for ((i = 0; i < SLOTS; i++)); do printf '%s; ' "$(holder "$i")"; done)" + exit 75 + fi + if [ $((now - last_note)) -ge 30 ]; then + say "$NAME waiting for a deploy slot ($SLOTS busy): $(for ((i = 0; i < SLOTS; i++)); do printf '%s; ' "$(holder "$i")"; done)" + last_note=$now + fi + sleep "$POLL" +done + +# Say who holds it (truncate, then write: the lock is on the descriptor, not the content). +printf '%s pid %s since %s\n' "$NAME" "$$" "$(date -u +%FT%TZ)" > "$(slot_file "$slot")" 2>/dev/null || true +waited=$(($(date +%s) - start)) +say "$NAME took deploy slot $slot of $SLOTS$([ "$waited" -gt 0 ] && printf ' after %ss' "$waited")" + +# Then a little room: never start a build into a box that is already out of memory. +mem_start=$(date +%s) +while [ "$(free_gb)" -lt "$MIN_FREE_GB" ]; do + if [ $(($(date +%s) - mem_start)) -ge "$MEM_WAIT" ]; then + say "only $(free_gb) GB available after ${MEM_WAIT}s; going on" + break + fi + say "$(free_gb) GB available, want $MIN_FREE_GB; waiting" + sleep 15 +done + +# Run with the lock descriptor closed, forwarding signals, and release when it ends. +child="" +forward() { [ -n "$child" ] && kill "-$1" "$child" 2>/dev/null; } +trap 'forward TERM' TERM +trap 'forward INT' INT +trap 'forward HUP' HUP +"$@" {fd}>&- & +child=$! +while :; do + wait "$child" + rc=$? + # wait returns early (>128) when a trapped signal arrives; keep waiting for the child. + kill -0 "$child" 2>/dev/null || break +done +: > "$(slot_file "$slot")" 2>/dev/null || true +exec {fd}>&- +exit "$rc" diff --git a/bin/pfdeploy.js b/bin/pfdeploy.js new file mode 100755 index 0000000..5877c4f --- /dev/null +++ b/bin/pfdeploy.js @@ -0,0 +1,102 @@ +#!/usr/bin/env node +import { spawnSync } from 'node:child_process'; +import { copyFileSync, chmodSync, readFileSync } from 'node:fs'; +import { join } from 'node:path'; +import { batch, DEFAULT_CONCURRENCY, gatePath, runCommand } from '../src/index.js'; + +const HELP = `pfdeploy: deploy in batches, ${DEFAULT_CONCURRENCY} at a time. + + pfdeploy batch [-c 3] [--keep-going] [--min-free-gb 8] -- '' item1 item2 ... + run the command once per item, at most -c at once ({} is the quoted item; + without {} the item is appended). Items may also come on stdin, one per line. + Stops starting new items after the first failure unless --keep-going. + pfdeploy gate [--slots 3] [--name site] -- + take one of the box's deploy slots, then run the command (bin/deploy-gate). + pfdeploy status [--slots 3] which deploy slots are busy, and by whom + pfdeploy install [dir] copy deploy-gate to dir (default /usr/local/bin) + +Examples: + pfdeploy batch -c 3 -- 'cd ~/src/profullstack/{} && bun test && gh pr merge --squash' crawlproof smshub ugig + ls ~/www | pfdeploy batch -c 3 -- '~/www/{}/deploy-app.sh origin/main' +`; + +const args = process.argv.slice(2); +const command = args.shift(); + +const gate = (rest) => { + const r = spawnSync('bash', [gatePath, ...rest], { stdio: 'inherit' }); + process.exit(r.status ?? 1); +}; + +if (!command || command === '-h' || command === '--help' || command === 'help') { + process.stdout.write(HELP); + process.exit(command ? 0 : 64); +} +if (command === 'version' || command === '--version') { + process.stdout.write(`${JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')).version}\n`); + process.exit(0); +} +if (command === 'gate') gate(args); +if (command === 'status') gate(['status', ...args]); +if (command === 'install') { + const dir = args[0] ?? '/usr/local/bin'; + const to = join(dir, 'deploy-gate'); + try { + copyFileSync(gatePath, to); + chmodSync(to, 0o755); + process.stdout.write(`installed ${to}\n`); + process.exit(0); + } catch (e) { + process.stderr.write(`pfdeploy install: ${e.message}${e.code === 'EACCES' ? ' (run with sudo)' : ''}\n`); + process.exit(1); + } +} +if (command !== 'batch') { + process.stderr.write(`pfdeploy: unknown command "${command}"\n\n${HELP}`); + process.exit(64); +} + +let concurrency = DEFAULT_CONCURRENCY; +let keepGoing = false; +let minFreeGb = 0; +while (args.length && args[0] !== '--') { + const a = args.shift(); + if (a === '-c' || a === '--concurrency') concurrency = Number(args.shift()); + else if (a === '--keep-going' || a === '-k') keepGoing = true; + else if (a === '--min-free-gb') minFreeGb = Number(args.shift()); + else { + process.stderr.write(`pfdeploy batch: unknown option ${a}\n`); + process.exit(64); + } +} +if (args[0] === '--') args.shift(); +const template = args.shift(); +if (!template || !(concurrency >= 1)) { + process.stderr.write(HELP); + process.exit(64); +} +let items = args; +if (!items.length && !process.stdin.isTTY) items = readFileSync(0, 'utf8').split('\n').map((s) => s.trim()).filter(Boolean); +if (!items.length) { + process.stderr.write('pfdeploy batch: no items\n'); + process.exit(64); +} + +const log = (s) => process.stderr.write(`[pfdeploy] ${s}\n`); +log(`${items.length} item${items.length === 1 ? '' : 's'}, ${concurrency} at a time`); +const results = await batch(items, (item) => runCommand(template, item), { + concurrency, + keepGoing, + minFreeGb, + onEvent: (e) => { + if (e.type === 'start') log(`start ${e.item}`); + if (e.type === 'done') log(`${e.ok ? 'ok ' : 'FAIL '} ${e.item}${e.ok ? '' : `: ${e.error.message}`}`); + if (e.type === 'memory') log(`${e.freeGb.toFixed(1)} GB free, want ${e.wantGb}: holding ${e.item}`); + }, +}); +const failed = results.filter((r) => !r.ok && !r.skipped); +const skipped = results.filter((r) => r.skipped); +log(`done: ${results.length - failed.length - skipped.length} ok, ${failed.length} failed${skipped.length ? `, ${skipped.length} not started (after a failure; --keep-going runs them)` : ''}`); +for (const r of failed) log(`failed: ${r.item}`); +for (const r of skipped) log(`not started: ${r.item}`); +process.exit(failed.length || skipped.length ? 1 : 0); diff --git a/index.d.ts b/index.d.ts new file mode 100644 index 0000000..1976cd1 --- /dev/null +++ b/index.d.ts @@ -0,0 +1,33 @@ +export const DEFAULT_CONCURRENCY: number; +/** Path of the bundled bash deploy gate. */ +export const gatePath: string; +/** GB of memory available now (Linux MemAvailable), Infinity where unknown. */ +export function freeGb(read?: () => string): number; + +export type BatchEvent = + | { type: 'start'; item: T; index: number } + | { type: 'done'; item: T; index: number; ok: boolean; error?: unknown } + | { type: 'memory'; item: T; freeGb: number; wantGb: number }; + +export type BatchResult = { item: T; ok: boolean; value?: R; error?: unknown; skipped: boolean }; + +export interface BatchOptions { + /** At most this many at once. Default 3. */ + concurrency?: number; + /** Keep starting items after a failure. Default false. */ + keepGoing?: boolean; + /** Before each start, wait while less than this many GB are available. Default 0 (off). */ + minFreeGb?: number; + /** Longest wait for memory before starting anyway. Default 600000. */ + memWaitMs?: number; + pollMs?: number; + freeGb?: () => number; + onEvent?: (event: BatchEvent) => void; +} + +/** Run task for every item, at most `concurrency` at once, in list order. */ +export function batch(items: Iterable, task: (item: T, index: number) => Promise | R, options?: BatchOptions): Promise[]>; + +export function shellQuote(s: string): string; +/** sh -c the template with {} replaced by the quoted item; rejects on a non-zero exit. */ +export function runCommand(template: string, item: string, options?: { prefix?: boolean; stdout?: NodeJS.WritableStream; stderr?: NodeJS.WritableStream; env?: NodeJS.ProcessEnv }): Promise<0>; diff --git a/package.json b/package.json new file mode 100644 index 0000000..8d7b593 --- /dev/null +++ b/package.json @@ -0,0 +1,32 @@ +{ + "name": "@profullstack/deploy", + "version": "0.1.0", + "description": "Deploy in batches, 3 at a time: a box-wide deploy gate (bash + flock) every deploy takes a slot from, and a batch runner for rolling one change out to many repos. Zero dependencies.", + "type": "module", + "license": "MIT", + "author": "Profullstack, Inc.", + "repository": { + "type": "git", + "url": "git+https://github.com/profullstack/deploy.git" + }, + "homepage": "https://github.com/profullstack/deploy#readme", + "keywords": ["deploy", "concurrency", "batch", "flock", "semaphore", "oom", "fleet", "rollout"], + "exports": { + ".": { + "types": "./index.d.ts", + "import": "./src/index.js" + } + }, + "types": "./index.d.ts", + "bin": { + "pfdeploy": "bin/pfdeploy.js", + "deploy-gate": "bin/deploy-gate" + }, + "files": ["bin", "src", "index.d.ts", "README.md", "LICENSE"], + "engines": { + "node": ">=18" + }, + "scripts": { + "test": "node --test test/*.test.js" + } +} diff --git a/src/index.js b/src/index.js new file mode 100644 index 0000000..34e24b6 --- /dev/null +++ b/src/index.js @@ -0,0 +1,109 @@ +/** + * @profullstack/deploy: deploy in batches, a few at a time, so a fleet-wide + * change never builds everything at once on one box. + * + * Two halves: + * - deploy-gate (bin/deploy-gate, bash + flock): every deploy on a box takes + * one of N slots before it builds. Use it on the box. + * - batch() / `pfdeploy batch`: an agent or a person rolling one change out to + * many repos runs them N at a time. Use it where the rollout is driven. + * + * Zero dependencies. + */ +import { spawn } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +export const DEFAULT_CONCURRENCY = 3; + +/** Path of the bundled bash gate. */ +export const gatePath = fileURLToPath(new URL('../bin/deploy-gate', import.meta.url)); + +/** GB of memory available right now (Linux), or Infinity where /proc/meminfo is not. */ +export function freeGb(read = () => readFileSync('/proc/meminfo', 'utf8')) { + try { + const kb = Number(/^MemAvailable:\s+(\d+)/m.exec(read())?.[1]); + return Number.isFinite(kb) ? kb / 1048576 : Infinity; + } catch { + return Infinity; + } +} + +const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + +/** + * Run `task(item, index)` for every item, at most `concurrency` at once, in order + * of the list. Before each start, waits (up to memWaitMs) while less than + * minFreeGb is available. On the first failure no new item starts unless + * keepGoing; items already running finish. Resolves to one result per item: + * { item, ok, value | error, skipped }. + */ +export async function batch(items, task, options = {}) { + const concurrency = Math.max(1, Math.floor(options.concurrency ?? DEFAULT_CONCURRENCY)); + const minFreeGb = options.minFreeGb ?? 0; + const memWaitMs = options.memWaitMs ?? 600_000; + const pollMs = options.pollMs ?? 15_000; + const free = options.freeGb ?? freeGb; + const onEvent = options.onEvent ?? (() => {}); + const list = [...items]; + const results = list.map((item) => ({ item, ok: false, skipped: true })); + let next = 0; + let failed = false; + + const roomFor = async (item) => { + if (!(minFreeGb > 0)) return; + const until = Date.now() + memWaitMs; + for (let gb = free(); gb < minFreeGb && Date.now() < until; gb = free()) { + onEvent({ type: 'memory', item, freeGb: gb, wantGb: minFreeGb }); + await sleep(pollMs); + } + }; + + const worker = async () => { + while (next < list.length && !(failed && !options.keepGoing)) { + const index = next++; + const item = list[index]; + await roomFor(item); + if (failed && !options.keepGoing) return; + onEvent({ type: 'start', item, index }); + try { + const value = await task(item, index); + results[index] = { item, ok: true, value, skipped: false }; + onEvent({ type: 'done', item, index, ok: true }); + } catch (error) { + failed = true; + results[index] = { item, ok: false, error, skipped: false }; + onEvent({ type: 'done', item, index, ok: false, error }); + } + } + }; + await Promise.all(Array.from({ length: Math.min(concurrency, list.length) }, worker)); + return results; +} + +/** POSIX single-quote a string for sh -c. */ +export const shellQuote = (s) => `'${String(s).replace(/'/g, `'\\''`)}'`; + +/** + * Run a shell command, with every `{}` replaced by the quoted item, and resolve + * to its exit code. Output lines are prefixed with the item. + */ +export function runCommand(template, item, { prefix = true, stdout = process.stdout, stderr = process.stderr, env = process.env } = {}) { + const command = template.includes('{}') ? template.replaceAll('{}', shellQuote(item)) : `${template} ${shellQuote(item)}`; + return new Promise((resolve, reject) => { + const child = spawn('sh', ['-c', command], { stdio: ['ignore', 'pipe', 'pipe'], env }); + const pipe = (from, to) => { + let carry = ''; + from.on('data', (chunk) => { + const lines = (carry + chunk).split('\n'); + carry = lines.pop(); + for (const line of lines) to.write(prefix ? `[${item}] ${line}\n` : `${line}\n`); + }); + from.on('end', () => carry && to.write(prefix ? `[${item}] ${carry}\n` : `${carry}\n`)); + }; + pipe(child.stdout, stdout); + pipe(child.stderr, stderr); + child.on('error', reject); + child.on('close', (code, signal) => (code === 0 ? resolve(0) : reject(Object.assign(new Error(`${item}: exit ${code ?? signal}`), { code: code ?? 1 })))); + }); +} diff --git a/test/deploy.test.js b/test/deploy.test.js new file mode 100644 index 0000000..876a5cd --- /dev/null +++ b/test/deploy.test.js @@ -0,0 +1,150 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawn, spawnSync } from 'node:child_process'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { batch, freeGb, gatePath, runCommand, shellQuote } from '../src/index.js'; + +const scratch = () => mkdtempSync(join(tmpdir(), 'pfdeploy-')); +const run = (args, env = {}) => + new Promise((resolve) => { + const child = spawn('bash', [gatePath, ...args], { env: { ...process.env, DEPLOY_MIN_FREE_GB: '0', DEPLOY_POLL: '0.2', ...env }, stdio: ['ignore', 'pipe', 'pipe'] }); + let out = ''; + let err = ''; + child.stdout.on('data', (d) => (out += d)); + child.stderr.on('data', (d) => (err += d)); + child.on('close', (code) => resolve({ code, out, err })); + }); + +test('the gate never lets more than --slots deploys run at once', async () => { + const dir = scratch(); + const log = join(dir, 'log'); + writeFileSync(log, ''); + // Each job appends +1 when it starts and -1 when it ends; we replay the log for the peak. + const job = `echo +1 >> ${log}; sleep 0.6; echo -1 >> ${log}`; + const started = Date.now(); + const results = await Promise.all(Array.from({ length: 6 }, (_, i) => run(['--slots', '2', '--name', `site${i}`, '--', 'bash', '-c', job], { DEPLOY_LOCK_DIR: join(dir, 'locks') }))); + assert.ok(results.every((r) => r.code === 0), results.map((r) => r.err).join('\n')); + let now = 0; + let peak = 0; + for (const line of readFileSync(log, 'utf8').trim().split('\n')) { + now += Number(line); + peak = Math.max(peak, now); + } + assert.equal(peak, 2); + // Six jobs, two at a time, 0.6 s each: at least three rounds. + assert.ok(Date.now() - started >= 1700, `took ${Date.now() - started} ms`); + assert.ok(results.every((r) => /took deploy slot [01] of 2/.test(r.err)), results.map((r) => r.err).join('|')); +}) + +test('exit codes pass through; a nested gate never waits on itself; status names the holder', async () => { + const dir = scratch(); + const env = { DEPLOY_LOCK_DIR: join(dir, 'locks') }; + assert.equal((await run(['--slots', '1', '--', 'bash', '-c', 'exit 7'], env)).code, 7); + // With one slot, a gated command that runs the gate again would deadlock without DEPLOY_GATED. + const nested = await run(['--slots', '1', '--', 'bash', gatePath, '--slots', '1', '--', 'echo', 'inner'], env); + assert.equal(nested.code, 0); + assert.equal(nested.out.trim(), 'inner'); + const holding = run(['--slots', '1', '--name', 'crawlproof', '--', 'sleep', '1'], env); + await new Promise((r) => setTimeout(r, 300)); + const status = spawnSync('bash', [gatePath, 'status', '--slots', '1'], { env: { ...process.env, ...env }, encoding: 'utf8' }); + assert.match(status.stdout, /slot 0 {2}busy {2}crawlproof pid \d+/); + await holding; + assert.match(spawnSync('bash', [gatePath, 'status', '--slots', '1'], { env: { ...process.env, ...env }, encoding: 'utf8' }).stdout, /slot 0 {2}free/); +}) + +test('a daemon the deploy starts does not keep the slot', async () => { + const dir = scratch(); + const env = { DEPLOY_LOCK_DIR: join(dir, 'locks') }; + // The deploy leaves a background process running (like `nohup server &`), then exits. + const first = await run(['--slots', '1', '--', 'bash', '-c', 'sleep 3 & disown; echo started'], env); + assert.equal(first.code, 0); + const t = Date.now(); + const second = await run(['--slots', '1', '--wait', '2', '--', 'true'], env); + assert.equal(second.code, 0, second.err); + assert.ok(Date.now() - t < 1500, 'the second deploy got the slot at once'); +}) + +test('a slot that never frees times out with 75, or runs anyway when asked', async () => { + const dir = scratch(); + const env = { DEPLOY_LOCK_DIR: join(dir, 'locks') }; + const holder = run(['--slots', '1', '--', 'sleep', '8'], env); + await new Promise((r) => setTimeout(r, 300)); + const late = await run(['--slots', '1', '--wait', '1', '--', 'true'], env); + assert.equal(late.code, 75); + assert.match(late.err, /giving up/); + const forced = await run(['--slots', '1', '--wait', '1', '--', 'echo', 'ran'], { ...env, DEPLOY_GATE_ON_TIMEOUT: 'run' }); + assert.equal(forced.out.trim(), 'ran'); + spawnSync('pkill', ['-f', 'sleep 8']); + await holder; +}) + +test('batch: at most `concurrency` at once, in order, and stops starting after a failure', async () => { + let running = 0; + let peak = 0; + const order = []; + const results = await batch([1, 2, 3, 4, 5, 6, 7], async (n) => { + order.push(n); + running++; + peak = Math.max(peak, running); + await new Promise((r) => setTimeout(r, 30)); + running--; + return n * 2; + }, { concurrency: 3 }); + assert.equal(peak, 3); + assert.deepEqual(order, [1, 2, 3, 4, 5, 6, 7]); + assert.deepEqual(results.map((r) => r.value), [2, 4, 6, 8, 10, 12, 14]); + + const stopped = await batch(['a', 'b', 'c', 'd', 'e'], async (x) => { + if (x === 'b') throw new Error('build failed'); + await new Promise((r) => setTimeout(r, 20)); + }, { concurrency: 2 }); + assert.equal(stopped.find((r) => r.item === 'b').ok, false); + assert.ok(stopped.filter((r) => r.skipped).length >= 2); + const kept = await batch(['a', 'b', 'c'], async (x) => { + if (x === 'a') throw new Error('x'); + }, { concurrency: 1, keepGoing: true }); + assert.deepEqual(kept.map((r) => r.ok), [false, true, true]); +}) + +test('batch waits for memory before each start, then gives up waiting', async () => { + const seen = []; + let calls = 0; + await batch(['x'], async () => {}, { minFreeGb: 8, pollMs: 5, memWaitMs: 1000, freeGb: () => (++calls < 3 ? 2 : 20), onEvent: (e) => seen.push(e.type) }); + assert.deepEqual(seen, ['memory', 'memory', 'start', 'done']); + const t = Date.now(); + await batch(['y'], async () => {}, { minFreeGb: 8, pollMs: 5, memWaitMs: 50, freeGb: () => 1 }); + assert.ok(Date.now() - t < 500); +}) + +test('helpers: free memory from /proc/meminfo, shell quoting, commands per item', async () => { + assert.equal(freeGb(() => 'MemTotal: 1 kB\nMemAvailable: 25165824 kB\n'), 24); + assert.equal(freeGb(() => { throw new Error('no proc'); }), Infinity); + assert.equal(shellQuote("it's"), `'it'\\''s'`); + const out = []; + const sink = { write: (s) => out.push(s) }; + await runCommand('echo deploying {}', "o'neil.com", { stdout: sink, stderr: sink }); + assert.deepEqual(out, ["[o'neil.com] deploying o'neil.com\n"]); + await assert.rejects(runCommand('exit 3', 'x', { stdout: sink, stderr: sink }), /x: exit 3/); +}) + +test('the CLI runs a batch three at a time by default', () => { + const dir = scratch(); + const log = join(dir, 'log'); + writeFileSync(log, ''); + const r = spawnSync('node', [new URL('../bin/pfdeploy.js', import.meta.url).pathname, 'batch', '--', `echo +1 >> ${log}; sleep 0.3; echo -1 >> ${log}; true`, 'a', 'b', 'c', 'd', 'e', 'f'], { encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + assert.match(r.stderr, /6 items, 3 at a time/); + let now = 0; + let peak = 0; + for (const line of readFileSync(log, 'utf8').trim().split('\n')) { + now += Number(line); + peak = Math.max(peak, now); + } + assert.equal(peak, 3); + const bad = spawnSync('node', [new URL('../bin/pfdeploy.js', import.meta.url).pathname, 'batch', '-c', '1', '--', 'test {} != b', 'a', 'b', 'c'], { encoding: 'utf8' }); + assert.equal(bad.status, 1); + assert.match(bad.stderr, /failed: b/); + assert.match(bad.stderr, /not started: c/); +})