Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
341 changes: 254 additions & 87 deletions Cargo.lock

Large diffs are not rendered by default.

8 changes: 6 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[package]
name = "request-logging-proxy"
version = "0.3.0"
version = "0.4.0"
edition = "2024"
authors = ["Christian Beilschmidt <christian.beilschmidt@geoengine.de>"]
description = "A simple HTTP proxy that logs requests and responses"
Expand Down Expand Up @@ -48,8 +48,10 @@ unwrap_used = "warn"
[dependencies]
anyhow = "1.0"
async-trait = "0.1"
brotli = "9.1.1"
bytes = "1.12"
clap = { version = "4.6", features = ["env", "derive"] }
encoding_rs = "0.8.44"
flate2 = "1.1"
http = "1.5"
http-body-util = "0.1"
Expand All @@ -59,8 +61,10 @@ hyper-util = { version = "0.1", features = [
"server-auto",
"server-graceful",
] }
ruzstd = "0.9.1"
serde_json = "1.0"
tokio = { version = "1.53", features = ["macros", "rt-multi-thread", "signal"] }
sha2 = "0.11.1"
tokio = { version = "1.53", features = ["fs", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] }
url = { version = "2.5", features = [] }

[dev-dependencies]
Expand Down
32 changes: 32 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,18 @@ cargo run -- \
--logger vscode
```

## Example

```bash
examples/demo.sh
```

This starts a demo upstream server ([examples/demo_upstream.rs](examples/demo_upstream.rs)) with the proxy in front of it and sends requests for compressed JSON (`gzip`, `br`, `zstd`), image tiles, a GeoTIFF up- and download, ISO-8859-1 text, an unsupported content coding, a `HEAD` request and a `304` response.
The log is written to `target/demo/requests.http`, binary bodies to `target/demo/bodies/`.
Pass a directory as the first argument to write somewhere else; `UPSTREAM_PORT` and `PROXY_PORT` change the ports (defaults `18484` and `18123`).

The demo upstream can also be run on its own: `cargo run --example demo_upstream -- 8484`.

## Testing

```bash
Expand All @@ -48,6 +60,26 @@ request-logging-proxy --target-url http://localhost:8484

- `vscode`: Logs requests and responses in a format compatible with VSCode's REST (.http) extension.

Every request/response pair is logged, regardless of its body.
Compressed bodies (`gzip`, `deflate`, `br`, `zstd`) are decompressed for the log only; the client receives the original bytes.
Textual bodies (`text/*`, JSON, XML, `+json`/`+xml` types, …) are decoded by their charset and logged, JSON pretty-printed.
Binary bodies (e.g. `image/png`) are logged as a placeholder comment:

```http
# Body: image/png, 131072 bytes, sha256 3f1c…
```

Use `--max-logged-body-bytes <N>` (or `MAX_LOGGED_BODY_BYTES`) to truncate long textual bodies in the log.

Use `--binary-body-dir <DIR>` (or `BINARY_BODY_DIR`) to store binary bodies as files named `<sha256>.<ext>`.
Identical bodies are stored once.
Request bodies are referenced as `< ./DIR/<file>`, so the logged request can be sent again; response bodies as a `# File:` comment.
Relative paths are written as given, so write the log into the directory you start the proxy from:

```bash
request-logging-proxy --target-url http://localhost:8484 --binary-body-dir bodies > requests.http
```

If you want other formats, feel free to open an issue or a pull request.

## License
Expand Down
78 changes: 78 additions & 0 deletions examples/demo.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
#!/usr/bin/env bash
#
# Run the proxy in front of the demo upstream, send a few requests and write the log.
#
# examples/demo.sh [OUTPUT_DIR]
#
# OUTPUT_DIR defaults to target/demo and contains `requests.http` and `bodies/` afterwards.
# Requires curl.

set -euo pipefail

ROOT="$(cd "$(dirname "$0")/.." && pwd)"
OUT="$(mkdir -p "${1:-$ROOT/target/demo}" && cd "${1:-$ROOT/target/demo}" && pwd)"
UPSTREAM_PORT="${UPSTREAM_PORT:-18484}"
PROXY_PORT="${PROXY_PORT:-18123}"
PROXY="http://localhost:$PROXY_PORT"

cargo build --quiet --manifest-path "$ROOT/Cargo.toml" --bin request-logging-proxy --example demo_upstream

rm -rf "$OUT/requests.http" "$OUT/bodies"

pids=()
cleanup() { kill "${pids[@]}" 2>/dev/null || true; }
trap cleanup EXIT

"$ROOT/target/debug/examples/demo_upstream" "$UPSTREAM_PORT" &
pids+=($!)

# Run from the output directory, so that the relative body paths in the log resolve.
(
cd "$OUT"
exec "$ROOT/target/debug/request-logging-proxy" \
--target-url "http://127.0.0.1:$UPSTREAM_PORT" \
--port "$PROXY_PORT" \
--binary-body-dir bodies \
--max-logged-body-bytes 4096 \
>requests.http
) &
pids+=($!)

# Wait until both servers accept connections, without sending a request that would be logged.
port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; }
for _ in $(seq 100); do
if port_open "$UPSTREAM_PORT" && port_open "$PROXY_PORT"; then break; fi
sleep 0.1
done

send() {
echo " $*" >&2
curl -s -o /dev/null "$@"
}

echo "Sending requests through $PROXY:" >&2
send "$PROXY/collections" # gzip-compressed JSON
send "$PROXY/collections/ndvi" # brotli-compressed JSON
send "$PROXY/conformance" # zstd-compressed problem+json
send "$PROXY/tiles/0/0/0.png" # binary, stored in bodies/
send "$PROXY/tiles/0/0/0.png" # same tile, stored only once
echo " $PROXY/coverage.tif" >&2 # GeoTIFF with media type parameters
curl -s -o "$OUT/upload.tif" "$PROXY/coverage.tif"
send -X PUT -H 'Content-Type: image/tiff; application=geotiff' \
--data-binary "@$OUT/upload.tif" "$PROXY/coverage.tif" # binary request body
send "$PROXY/legacy.txt" # ISO-8859-1 text
send "$PROXY/old.json" # unsupported content coding
send -I "$PROXY/collections" # HEAD: no body
send "$PROXY/cached" # 304: no body
rm -f "$OUT/upload.tif"

# The proxy writes all pending log entries before it exits.
proxy_pid="${pids[1]}"
kill -TERM "$proxy_pid"
wait "$proxy_pid" || true
cleanup
wait 2>/dev/null || true

echo >&2
echo "Log: $OUT/requests.http" >&2
echo "Bodies: $OUT/bodies/" >&2
143 changes: 143 additions & 0 deletions examples/demo_upstream.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
//! A small upstream server for trying out the proxy.
//!
//! It serves the kinds of responses an OGC API server returns:
//! compressed JSON, image tiles, GeoTIFF coverages, legacy charsets and bodiless responses.
//!
//! ```bash
//! cargo run --example demo_upstream -- 8484
//! ```
//!
//! See `examples/demo.sh` for a complete run with the proxy in front of it.

use bytes::Bytes;
use http::{Method, Request, Response, StatusCode, header};
use http_body_util::{BodyExt, Full};
use hyper::body::Incoming;
use hyper::service::service_fn;
use hyper_util::rt::{TokioExecutor, TokioIo};
use std::convert::Infallible;
use std::io::Write;
use std::net::SocketAddr;
use tokio::net::TcpListener;

const DEFAULT_PORT: u16 = 8484;

#[tokio::main]
async fn main() -> std::io::Result<()> {
let port = std::env::args()
.nth(1)
.and_then(|port| port.parse().ok())
.unwrap_or(DEFAULT_PORT);
let addr = SocketAddr::from(([127, 0, 0, 1], port));
let listener = TcpListener::bind(addr).await?;
eprintln!("Demo upstream listening on http://{addr}");

loop {
let (stream, _) = listener.accept().await?;
tokio::spawn(async move {
let server = hyper_util::server::conn::auto::Builder::new(TokioExecutor::new());
if let Err(e) = server
.serve_connection(TokioIo::new(stream), service_fn(handle))
.await
{
eprintln!("Error serving connection: {e}");
}
});
}
}

async fn handle(request: Request<Incoming>) -> Result<Response<Full<Bytes>>, Infallible> {
let (parts, body) = request.into_parts();
// consume uploads, the content does not matter
let _ = body.collect().await;

let response = match (&parts.method, parts.uri.path()) {
(&Method::PUT, _) => empty(StatusCode::NO_CONTENT),
(_, "/cached") => empty(StatusCode::NOT_MODIFIED),
(_, "/collections") => ok(
"application/json",
Some("gzip"),
gzip(br#"{"collections":[{"id":"ndvi","title":"NDVI"}]}"#),
),
(_, "/collections/ndvi") => ok(
"application/json",
Some("br"),
brotli(br#"{"id":"ndvi","extent":{"spatial":{"bbox":[[-180,-90,180,90]]}}}"#),
),
(_, "/conformance") => ok(
"application/problem+json",
Some("zstd"),
ruzstd::encoding::compress_to_vec(
&br#"{"type":"about:blank","status":404}"#[..],
ruzstd::encoding::CompressionLevel::Fastest,
),
),
(_, "/tiles/0/0/0.png") => ok("image/png", None, fake_png()),
(_, "/coverage.tif") => ok("image/tiff; application=geotiff", None, fake_tiff()),
// "Grüße aus München" in ISO-8859-1
(_, "/legacy.txt") => ok(
"text/plain; charset=ISO-8859-1",
None,
b"Gr\xfc\xdfe aus M\xfcnchen".to_vec(),
),
// `compress` is a registered content coding the proxy cannot decode
(_, "/old.json") => ok(
"application/json",
Some("compress"),
b"\x1f\x9d\x90{".to_vec(),
),
_ => Response::builder()
.status(StatusCode::NOT_FOUND)
.header(header::CONTENT_TYPE, "application/problem+json")
.body(Full::from(r#"{"type":"about:blank","status":404}"#))
.unwrap_or_default(),
};

Ok(response)
}

fn ok(content_type: &str, content_encoding: Option<&str>, body: Vec<u8>) -> Response<Full<Bytes>> {
let mut builder = Response::builder().header(header::CONTENT_TYPE, content_type);
if let Some(content_encoding) = content_encoding {
builder = builder.header(header::CONTENT_ENCODING, content_encoding);
}
builder.body(Full::from(body)).unwrap_or_default()
}

fn empty(status: StatusCode) -> Response<Full<Bytes>> {
Response::builder()
.status(status)
.body(Full::default())
.unwrap_or_default()
}

fn gzip(data: &[u8]) -> Vec<u8> {
let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
encoder
.write_all(data)
.and_then(|()| encoder.finish())
.unwrap_or_default()
}

fn brotli(data: &[u8]) -> Vec<u8> {
let mut output = Vec::new();
{
let mut writer = brotli::CompressorWriter::new(&mut output, 4096, 5, 22);
let _ = writer.write_all(data);
}
output
}

/// PNG signature followed by filler bytes; enough for the proxy to see binary data.
fn fake_png() -> Vec<u8> {
let mut data = b"\x89PNG\r\n\x1a\n".to_vec();
data.extend(0..=255u8);
data
}

/// Little-endian TIFF header followed by zeros.
fn fake_tiff() -> Vec<u8> {
let mut data = b"II*\0".to_vec();
data.resize(1024, 0);
data
}
Loading
Loading