From 32f023f8c5e25826a2cdaf6100c0f478aebbec2d Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Wed, 11 Nov 2026 15:11:56 +0100 Subject: [PATCH 1/3] feat: log binary and compressed bodies instead of dropping the whole entry --- Cargo.lock | 339 ++++++++++++++++------ Cargo.toml | 6 +- README.md | 32 +++ examples/demo.sh | 76 +++++ examples/demo_upstream.rs | 143 ++++++++++ src/body.rs | 573 ++++++++++++++++++++++++++++++++++++++ src/config.rs | 10 + src/logger.rs | 358 +++++++++++++++++------- src/main.rs | 16 +- 9 files changed, 1368 insertions(+), 185 deletions(-) create mode 100755 examples/demo.sh create mode 100644 examples/demo_upstream.rs create mode 100644 src/body.rs diff --git a/Cargo.lock b/Cargo.lock index a55dc08..fd012b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,21 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "alloc-no-stdlib" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2fb6cfd47bf496ff64095c20eaba0c201404ee38714d4142fcfa1dc334fcc7a" + +[[package]] +name = "alloc-stdlib" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5c1865780388bfa186411ab5f247819487fc4864c6e9c3106611fa347586e1" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "anstream" version = "1.0.0" @@ -72,7 +87,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -81,6 +96,36 @@ version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "brotli" +version = "9.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "627b7cfb0578f0eafce087a90fcd40add1231900b69e4d4de2d456260a4f5d81" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "6.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75c74c9c51b9140663e196b5fb88bc94c334d7a497c8311cb759d8557874747f" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bytes" version = "1.12.1" @@ -89,9 +134,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "clap" @@ -124,7 +169,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] @@ -139,21 +184,62 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570" +[[package]] +name = "const-oid" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be95d4a884358a21a199689323c29e9256b4a1b315e22f183d155422041d3a9a" + +[[package]] +name = "core_detect" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f8f80099a98041a3d1622845c271458a2d73e688351bf3cb999266764b81d48" + +[[package]] +name = "cpufeatures" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566" +dependencies = [ + "libc", +] + [[package]] name = "crc32fast" -version = "1.5.2" +version = "1.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +checksum = "2b929dd2a135771f40afabd1073f85eed59c800b7c61901e5c254270497dad64" dependencies = [ "cfg-if", ] +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "hybrid-array", +] + [[package]] name = "diff" version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer", + "const-oid", + "crypto-common", +] + [[package]] name = "displaydoc" version = "0.2.7" @@ -162,14 +248,28 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", +] + +[[package]] +name = "encoding_rs" +version = "0.8.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5" +dependencies = [ + "cfg-if", + "core_detect", + "multiversion_no_op", + "rustversion", + "scopeguard", + "simdutf8", ] [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" @@ -183,9 +283,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.10" +version = "1.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +checksum = "626d4928326b646e70f65ed36fc7cf9f206bdb68a0019ee7c6b4709da4e084c4" dependencies = [ "crc32fast", "miniz_oxide", @@ -230,9 +330,9 @@ checksum = "1944426bf7d03f1d14f708785e4b33efd750b36d48a157b836b3efc15ede8e1d" [[package]] name = "h2" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16" +checksum = "7d29020232d6aa3fb1daca64c1127cf662cf97f254ae16c18c05b8ab635fc118" dependencies = [ "atomic-waker", "bytes", @@ -304,11 +404,20 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "hybrid-array" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27f864f10dfb56725ce5ce5472bc52252c8f93a4ab86327122cebf62c5f59a17" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43" +checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c" dependencies = [ "atomic-waker", "bytes", @@ -328,9 +437,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.20" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff" dependencies = [ "bytes", "futures-channel", @@ -345,9 +454,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513" +checksum = "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd" dependencies = [ "displaydoc", "potential_utf", @@ -359,9 +468,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb" +checksum = "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0" dependencies = [ "displaydoc", "litemap", @@ -372,9 +481,9 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f" +checksum = "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599" dependencies = [ "icu_collections", "icu_normalizer_data", @@ -386,15 +495,15 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0" +checksum = "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc" [[package]] name = "icu_properties" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148" +checksum = "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f" dependencies = [ "displaydoc", "icu_collections", @@ -407,15 +516,15 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa" +checksum = "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf" [[package]] name = "icu_provider" -version = "2.3.1" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73" +checksum = "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514" dependencies = [ "displaydoc", "icu_locale_core", @@ -465,15 +574,15 @@ checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itoa" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "litemap" @@ -499,15 +608,21 @@ dependencies = [ [[package]] name = "mio" -version = "1.2.3" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b18443e9c262bfe8fa82f51666e2642c53393f7e5c27b3e1aeab922cff5b9d8" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "wasi", "windows-sys", ] +[[package]] +name = "multiversion_no_op" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "743fb55ba31b18fb1ecef6bdc9aa2743314978ac084044301a7eee33fb99a20d" + [[package]] name = "once_cell" version = "1.21.4" @@ -575,19 +690,44 @@ version = "0.3.0" dependencies = [ "anyhow", "async-trait", + "brotli", "bytes", "clap", + "encoding_rs", "flate2", "http", "http-body-util", "hyper", "hyper-util", "pretty_assertions", + "ruzstd", "serde_json", + "sha2", "tokio", "url", ] +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "ruzstd" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69e4ce4a039f42bf25920b95667e4d789e98749ab11bbb8d2c2936f788421bb5" +dependencies = [ + "twox-hash", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + [[package]] name = "serde" version = "1.0.229" @@ -614,14 +754,14 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ "itoa", "memchr", @@ -630,6 +770,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d7069beb7d6ac7b9acd1039986e73443f24234f41074da099d6f994ac9ad19" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "signal-hook-registry" version = "1.4.8" @@ -646,6 +797,12 @@ version = "0.3.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + [[package]] name = "slab" version = "0.4.12" @@ -654,9 +811,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.16.1" +version = "1.16.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" +checksum = "149309498a5f7334b12ddae0cad8aba81ce785fe91755068867e017cf98fee46" [[package]] name = "socket2" @@ -682,20 +839,9 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" [[package]] name = "syn" -version = "2.0.119" +version = "3.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" +checksum = "4ae9ccbea3a7b0b09443036490c8f34a8b1b760fd345e5c0dae0aac718faea79" dependencies = [ "proc-macro2", "quote", @@ -704,20 +850,20 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", ] [[package]] name = "tinystr" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643" +checksum = "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a" dependencies = [ "displaydoc", "zerovec", @@ -725,9 +871,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.53.1" +version = "1.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3" dependencies = [ "bytes", "libc", @@ -747,14 +893,14 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "tokio-util" -version = "0.7.19" +version = "0.7.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +checksum = "e464cf451ba96ebfc6f9b6542f17ee8b8956e33f1e40d9690624e59d7a7f8a4b" dependencies = [ "bytes", "futures-core", @@ -795,11 +941,23 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "url" @@ -815,9 +973,12 @@ dependencies = [ [[package]] name = "utf8_iter" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0" +dependencies = [ + "utf_types", +] [[package]] name = "utf8parse" @@ -825,11 +986,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "utf_types" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6" + [[package]] name = "want" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +checksum = "ec4cdd0dd910afe868b7ef477227d8d538b46b3075031afee8a9f2acb0a2ed0b" dependencies = [ "try-lock", ] @@ -869,9 +1036,9 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "yoke" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +checksum = "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3" dependencies = [ "stable_deref_trait", "yoke-derive", @@ -880,13 +1047,13 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.2" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +checksum = "ec8ebde2db3681e8c9980cc27822030e68752690ddfa9473e739aeb4dbde6d71" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] @@ -901,13 +1068,13 @@ dependencies = [ [[package]] name = "zerofrom-derive" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" dependencies = [ "proc-macro2", "quote", - "syn 2.0.119", + "syn", "synstructure", ] @@ -924,9 +1091,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8" +checksum = "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40" dependencies = [ "yoke", "zerofrom", @@ -935,20 +1102,20 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.6" +version = "0.11.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da" +checksum = "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.5", + "syn", ] [[package]] name = "zlib-rs" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" [[package]] name = "zmij" diff --git a/Cargo.toml b/Cargo.toml index 4597ae9..51ccd04 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -48,8 +48,10 @@ unwrap_used = "warn" [dependencies] anyhow = "1.0" async-trait = "0.1" +brotli = "9.1.1" bytes = "1.12" clap = { version = "4.6", features = ["env", "derive"] } +encoding_rs = "0.8.44" flate2 = "1.1" http = "1.5" http-body-util = "0.1" @@ -59,8 +61,10 @@ hyper-util = { version = "0.1", features = [ "server-auto", "server-graceful", ] } +ruzstd = "0.9.1" serde_json = "1.0" -tokio = { version = "1.53", features = ["macros", "rt-multi-thread", "signal"] } +sha2 = "0.11.1" +tokio = { version = "1.53", features = ["fs", "macros", "net", "rt-multi-thread", "signal"] } url = { version = "2.5", features = [] } [dev-dependencies] diff --git a/README.md b/README.md index 935a66f..1addd3d 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,18 @@ cargo run -- \ --logger vscode ``` +## Example + +```bash +examples/demo.sh +``` + +This starts a demo upstream server ([examples/demo_upstream.rs](examples/demo_upstream.rs)) with the proxy in front of it and sends requests for compressed JSON (`gzip`, `br`, `zstd`), image tiles, a GeoTIFF up- and download, ISO-8859-1 text, an unsupported content coding, a `HEAD` request and a `304` response. +The log is written to `target/demo/requests.http`, binary bodies to `target/demo/bodies/`. +Pass a directory as the first argument to write somewhere else; `UPSTREAM_PORT` and `PROXY_PORT` change the ports (defaults `18484` and `18123`). + +The demo upstream can also be run on its own: `cargo run --example demo_upstream -- 8484`. + ## Testing ```bash @@ -48,6 +60,26 @@ request-logging-proxy --target-url http://localhost:8484 - `vscode`: Logs requests and responses in a format compatible with VSCode's REST (.http) extension. +Every request/response pair is logged, regardless of its body. +Compressed bodies (`gzip`, `deflate`, `br`, `zstd`) are decompressed for the log only; the client receives the original bytes. +Textual bodies (`text/*`, JSON, XML, `+json`/`+xml` types, …) are decoded by their charset and logged, JSON pretty-printed. +Binary bodies (e.g. `image/png`) are logged as a placeholder comment: + +```http +# Body: image/png, 131072 bytes, sha256 3f1c… +``` + +Use `--max-logged-body-bytes ` (or `MAX_LOGGED_BODY_BYTES`) to truncate long textual bodies in the log. + +Use `--binary-body-dir ` (or `BINARY_BODY_DIR`) to store binary bodies as files named `.`. +Identical bodies are stored once. +Request bodies are referenced as `< ./DIR/`, so the logged request can be sent again; response bodies as a `# File:` comment. +Relative paths are written as given, so write the log into the directory you start the proxy from: + +```bash +request-logging-proxy --target-url http://localhost:8484 --binary-body-dir bodies > requests.http +``` + If you want other formats, feel free to open an issue or a pull request. ## License diff --git a/examples/demo.sh b/examples/demo.sh new file mode 100755 index 0000000..54ad370 --- /dev/null +++ b/examples/demo.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# +# Run the proxy in front of the demo upstream, send a few requests and write the log. +# +# examples/demo.sh [OUTPUT_DIR] +# +# OUTPUT_DIR defaults to target/demo and contains `requests.http` and `bodies/` afterwards. +# Requires curl. + +set -euo pipefail + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +OUT="$(mkdir -p "${1:-$ROOT/target/demo}" && cd "${1:-$ROOT/target/demo}" && pwd)" +UPSTREAM_PORT="${UPSTREAM_PORT:-18484}" +PROXY_PORT="${PROXY_PORT:-18123}" +PROXY="http://localhost:$PROXY_PORT" + +cargo build --quiet --manifest-path "$ROOT/Cargo.toml" --bin request-logging-proxy --example demo_upstream + +rm -rf "$OUT/requests.http" "$OUT/bodies" + +pids=() +cleanup() { kill "${pids[@]}" 2>/dev/null || true; } +trap cleanup EXIT + +"$ROOT/target/debug/examples/demo_upstream" "$UPSTREAM_PORT" & +pids+=($!) + +# Run from the output directory, so that the relative body paths in the log resolve. +( + cd "$OUT" + exec "$ROOT/target/debug/request-logging-proxy" \ + --target-url "http://127.0.0.1:$UPSTREAM_PORT" \ + --port "$PROXY_PORT" \ + --binary-body-dir bodies \ + --max-logged-body-bytes 4096 \ + >requests.http +) & +pids+=($!) + +# Wait until both servers accept connections, without sending a request that would be logged. +port_open() { (exec 3<>"/dev/tcp/127.0.0.1/$1") 2>/dev/null; } +for _ in $(seq 100); do + if port_open "$UPSTREAM_PORT" && port_open "$PROXY_PORT"; then break; fi + sleep 0.1 +done + +send() { + echo " $*" >&2 + curl -s -o /dev/null "$@" +} + +echo "Sending requests through $PROXY:" >&2 +send "$PROXY/collections" # gzip-compressed JSON +send "$PROXY/collections/ndvi" # brotli-compressed JSON +send "$PROXY/conformance" # zstd-compressed problem+json +send "$PROXY/tiles/0/0/0.png" # binary, stored in bodies/ +send "$PROXY/tiles/0/0/0.png" # same tile, stored only once +echo " $PROXY/coverage.tif" >&2 # GeoTIFF with media type parameters +curl -s -o "$OUT/upload.tif" "$PROXY/coverage.tif" +send -X PUT -H 'Content-Type: image/tiff; application=geotiff' \ + --data-binary "@$OUT/upload.tif" "$PROXY/coverage.tif" # binary request body +send "$PROXY/legacy.txt" # ISO-8859-1 text +send "$PROXY/old.json" # unsupported content coding +send -I "$PROXY/collections" # HEAD: no body +send "$PROXY/cached" # 304: no body +rm -f "$OUT/upload.tif" + +# Give the logger time to write the last entries before shutting down. +sleep 0.5 +cleanup +wait 2>/dev/null || true + +echo >&2 +echo "Log: $OUT/requests.http" >&2 +echo "Bodies: $OUT/bodies/" >&2 diff --git a/examples/demo_upstream.rs b/examples/demo_upstream.rs new file mode 100644 index 0000000..ff30a42 --- /dev/null +++ b/examples/demo_upstream.rs @@ -0,0 +1,143 @@ +//! A small upstream server for trying out the proxy. +//! +//! It serves the kinds of responses an OGC API server returns: +//! compressed JSON, image tiles, GeoTIFF coverages, legacy charsets and bodiless responses. +//! +//! ```bash +//! cargo run --example demo_upstream -- 8484 +//! ``` +//! +//! See `examples/demo.sh` for a complete run with the proxy in front of it. + +use bytes::Bytes; +use http::{Method, Request, Response, StatusCode, header}; +use http_body_util::{BodyExt, Full}; +use hyper::body::Incoming; +use hyper::service::service_fn; +use hyper_util::rt::{TokioExecutor, TokioIo}; +use std::convert::Infallible; +use std::io::Write; +use std::net::SocketAddr; +use tokio::net::TcpListener; + +const DEFAULT_PORT: u16 = 8484; + +#[tokio::main] +async fn main() -> std::io::Result<()> { + let port = std::env::args() + .nth(1) + .and_then(|port| port.parse().ok()) + .unwrap_or(DEFAULT_PORT); + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + let listener = TcpListener::bind(addr).await?; + eprintln!("Demo upstream listening on http://{addr}"); + + loop { + let (stream, _) = listener.accept().await?; + tokio::spawn(async move { + let server = hyper_util::server::conn::auto::Builder::new(TokioExecutor::new()); + if let Err(e) = server + .serve_connection(TokioIo::new(stream), service_fn(handle)) + .await + { + eprintln!("Error serving connection: {e}"); + } + }); + } +} + +async fn handle(request: Request) -> Result>, Infallible> { + let (parts, body) = request.into_parts(); + // consume uploads, the content does not matter + let _ = body.collect().await; + + let response = match (&parts.method, parts.uri.path()) { + (&Method::PUT, _) => empty(StatusCode::NO_CONTENT), + (_, "/cached") => empty(StatusCode::NOT_MODIFIED), + (_, "/collections") => ok( + "application/json", + Some("gzip"), + gzip(br#"{"collections":[{"id":"ndvi","title":"NDVI"}]}"#), + ), + (_, "/collections/ndvi") => ok( + "application/json", + Some("br"), + brotli(br#"{"id":"ndvi","extent":{"spatial":{"bbox":[[-180,-90,180,90]]}}}"#), + ), + (_, "/conformance") => ok( + "application/problem+json", + Some("zstd"), + ruzstd::encoding::compress_to_vec( + &br#"{"type":"about:blank","status":404}"#[..], + ruzstd::encoding::CompressionLevel::Fastest, + ), + ), + (_, "/tiles/0/0/0.png") => ok("image/png", None, fake_png()), + (_, "/coverage.tif") => ok("image/tiff; application=geotiff", None, fake_tiff()), + // "Grüße aus München" in ISO-8859-1 + (_, "/legacy.txt") => ok( + "text/plain; charset=ISO-8859-1", + None, + b"Gr\xfc\xdfe aus M\xfcnchen".to_vec(), + ), + // `compress` is a registered content coding the proxy cannot decode + (_, "/old.json") => ok( + "application/json", + Some("compress"), + b"\x1f\x9d\x90{".to_vec(), + ), + _ => Response::builder() + .status(StatusCode::NOT_FOUND) + .header(header::CONTENT_TYPE, "application/problem+json") + .body(Full::from(r#"{"type":"about:blank","status":404}"#)) + .unwrap_or_default(), + }; + + Ok(response) +} + +fn ok(content_type: &str, content_encoding: Option<&str>, body: Vec) -> Response> { + let mut builder = Response::builder().header(header::CONTENT_TYPE, content_type); + if let Some(content_encoding) = content_encoding { + builder = builder.header(header::CONTENT_ENCODING, content_encoding); + } + builder.body(Full::from(body)).unwrap_or_default() +} + +fn empty(status: StatusCode) -> Response> { + Response::builder() + .status(status) + .body(Full::default()) + .unwrap_or_default() +} + +fn gzip(data: &[u8]) -> Vec { + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder + .write_all(data) + .and_then(|()| encoder.finish()) + .unwrap_or_default() +} + +fn brotli(data: &[u8]) -> Vec { + let mut output = Vec::new(); + { + let mut writer = brotli::CompressorWriter::new(&mut output, 4096, 5, 22); + let _ = writer.write_all(data); + } + output +} + +/// PNG signature followed by filler bytes; enough for the proxy to see binary data. +fn fake_png() -> Vec { + let mut data = b"\x89PNG\r\n\x1a\n".to_vec(); + data.extend(0..=255u8); + data +} + +/// Little-endian TIFF header followed by zeros. +fn fake_tiff() -> Vec { + let mut data = b"II*\0".to_vec(); + data.resize(1024, 0); + data +} diff --git a/src/body.rs b/src/body.rs new file mode 100644 index 0000000..908f8cb --- /dev/null +++ b/src/body.rs @@ -0,0 +1,573 @@ +//! Turns raw HTTP message bodies into something that can be written to a log. +//! +//! The functions here never fail: anything that cannot be shown as text +//! (unknown content coding, binary media type, undecodable charset, …) +//! becomes a [`LoggedBody::Binary`] placeholder. + +use http::header::{CONTENT_ENCODING, CONTENT_TYPE}; +use http::{HeaderMap, Method, StatusCode}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use std::borrow::Cow; +use std::fmt::Write as _; +use std::io::Read; +use std::path::{Path, PathBuf}; + +/// Upper bound for decompressed bodies, to guard against decompression bombs. +const MAX_DECODED_BYTES: u64 = 64 * 1024 * 1024; + +/// A body prepared for logging. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum LoggedBody { + /// The body is empty or the message cannot have a body. + None, + /// A textual body, decoded to UTF-8 and possibly truncated. + Text { + text: String, + truncated_bytes: usize, + }, + /// A body that is not logged verbatim. + Binary { + content_type: Option, + len: usize, + sha256: String, + }, +} + +impl LoggedBody { + /// The one-line description of a binary body, e.g. + /// `image/png, 1234 bytes, sha256 3f1c…`. + pub fn binary_summary(content_type: Option<&str>, len: usize, sha256: &str) -> String { + match content_type { + Some(content_type) => format!("{content_type}, {len} bytes, sha256 {sha256}"), + None => format!("{len} bytes, sha256 {sha256}"), + } + } +} + +/// Whether a response to `method` with `status` can carry a body (RFC 9110 §6.4.1). +pub fn response_expects_body(method: &Method, status: StatusCode) -> bool { + !(method == Method::HEAD + || status.is_informational() + || status == StatusCode::NO_CONTENT + || status == StatusCode::NOT_MODIFIED) +} + +/// Prepare a body for logging. +/// +/// `bytes` are the bytes as sent on the wire. They are only decoded for the log; +/// the caller keeps forwarding the original bytes. +pub fn render_body( + headers: &HeaderMap, + bytes: &[u8], + expect_body: bool, + max_text_bytes: Option, +) -> LoggedBody { + if !expect_body || bytes.is_empty() { + return LoggedBody::None; + } + + let content_type = headers.get(CONTENT_TYPE).map(|value| match value.to_str() { + Ok(s) => Cow::Borrowed(s), + Err(_) => String::from_utf8_lossy(value.as_bytes()), + }); + + let binary = || LoggedBody::Binary { + content_type: content_type.as_deref().map(str::to_owned), + len: bytes.len(), + sha256: sha256_hex(bytes), + }; + + let Some(decoded) = remove_content_encoding(headers, bytes) else { + return binary(); + }; + + let text = match content_type.as_deref() { + Some(content_type) => { + let media_type = MediaType::parse(content_type); + if !media_type.is_text() { + return binary(); + } + match decode_charset(&decoded, media_type.charset.as_deref()) { + Some(text) => text, + None => return binary(), + } + } + None => match String::from_utf8(decoded.into_owned()) { + Ok(text) => text, + Err(_) => return binary(), + }, + }; + + let text = pretty_print_json(text); + let (text, truncated_bytes) = truncate(text, max_text_bytes); + + LoggedBody::Text { + text, + truncated_bytes, + } +} + +/// Store a binary body in `dir` as `.` and return the path of the file. +/// +/// Files are content-addressed, so a body that was already stored is not written again. +/// The bytes are stored as received, i.e., with any content coding still applied. +pub async fn store_binary( + dir: &Path, + bytes: &[u8], + content_type: Option<&str>, + sha256: &str, +) -> std::io::Result { + let extension = content_type.map_or("bin", |content_type| { + let media_type = MediaType::parse(content_type); + file_extension(&media_type.type_, &media_type.subtype) + }); + let path = dir.join(format!("{sha256}.{extension}")); + if !tokio::fs::try_exists(&path).await? { + tokio::fs::write(&path, bytes).await?; + } + Ok(path) +} + +/// A file extension for a media type, so that stored bodies open in the right application. +fn file_extension(type_: &str, subtype: &str) -> &'static str { + match (type_, subtype) { + ("image", "png") => "png", + ("image", "jpeg") => "jpg", + ("image", "gif") => "gif", + ("image", "webp") => "webp", + ("image", "tiff") => "tif", + ("image", "avif") => "avif", + ("application", "pdf") => "pdf", + ("application", "zip") => "zip", + ("application", "gzip") => "gz", + ("application", "vnd.mapbox-vector-tile") => "mvt", + ("application", "x-protobuf" | "vnd.google.protobuf") => "pbf", + ("application", "vnd.apache.parquet") => "parquet", + ("application", "vnd.apache.arrow.file") => "arrow", + ("application", "geopackage+sqlite3") => "gpkg", + ("application", "netcdf" | "x-netcdf") => "nc", + _ => "bin", + } +} + +/// Undo all content codings (RFC 9110 §8.4). +/// Returns `None` if a coding is unknown or decoding fails. +fn remove_content_encoding<'b>(headers: &HeaderMap, bytes: &'b [u8]) -> Option> { + let mut codings = Vec::new(); + for value in headers.get_all(CONTENT_ENCODING) { + let value = value.to_str().ok()?; + codings.extend( + value + .split(',') + .map(|coding| coding.trim().to_ascii_lowercase()) + .filter(|coding| !coding.is_empty()), + ); + } + + let mut data = Cow::Borrowed(bytes); + // codings are listed in the order they were applied + for coding in codings.iter().rev() { + data = match coding.as_str() { + "identity" => data, + "gzip" | "x-gzip" => Cow::Owned(read_limited(flate2::read::GzDecoder::new(&*data))?), + "deflate" => Cow::Owned( + // `deflate` is the zlib format, but some servers send raw deflate data + read_limited(flate2::read::ZlibDecoder::new(&*data)) + .or_else(|| read_limited(flate2::read::DeflateDecoder::new(&*data)))?, + ), + "br" => Cow::Owned(read_limited(brotli::Decompressor::new(&*data, 4096))?), + "zstd" => Cow::Owned(read_limited( + ruzstd::decoding::StreamingDecoder::new(&*data).ok()?, + )?), + _ => return None, + }; + } + + Some(data) +} + +/// Read everything from `reader`, failing if it exceeds [`MAX_DECODED_BYTES`]. +fn read_limited(reader: R) -> Option> { + let mut output = Vec::new(); + reader + .take(MAX_DECODED_BYTES + 1) + .read_to_end(&mut output) + .ok()?; + if output.len() as u64 > MAX_DECODED_BYTES { + return None; + } + Some(output) +} + +/// The parts of a `Content-Type` value (RFC 9110 §8.3) that are relevant for logging. +#[derive(Debug, PartialEq, Eq)] +struct MediaType { + /// lower-case `type` + type_: String, + /// lower-case `subtype` + subtype: String, + charset: Option, +} + +impl MediaType { + fn parse(value: &str) -> Self { + let mut parts = value.split(';'); + let essence = parts.next().unwrap_or_default().trim().to_ascii_lowercase(); + let (type_, subtype) = essence + .split_once('/') + .map(|(t, s)| (t.trim().to_owned(), s.trim().to_owned())) + .unwrap_or((essence.clone(), String::new())); + + let charset = parts.find_map(|param| { + let (name, value) = param.split_once('=')?; + if !name.trim().eq_ignore_ascii_case("charset") { + return None; + } + let value = value.trim(); + let value = value + .strip_prefix('"') + .and_then(|v| v.strip_suffix('"')) + .unwrap_or(value); + Some(value.to_owned()) + }); + + Self { + type_, + subtype, + charset, + } + } + + fn is_text(&self) -> bool { + if self.type_ == "text" { + return true; + } + if self.type_ != "application" { + return false; + } + matches!( + self.subtype.as_str(), + "json" | "xml" | "x-www-form-urlencoded" | "javascript" + ) || self.subtype.ends_with("+json") + || self.subtype.ends_with("+xml") + } +} + +/// Decode `bytes` with the given charset, defaulting to UTF-8 (RFC 8259 §8.1, RFC 6657). +fn decode_charset(bytes: &[u8], charset: Option<&str>) -> Option { + let encoding = match charset { + Some(label) => encoding_rs::Encoding::for_label(label.as_bytes())?, + None => encoding_rs::UTF_8, + }; + encoding + .decode_without_bom_handling_and_without_replacement(bytes) + .map(Cow::into_owned) +} + +fn pretty_print_json(text: String) -> String { + serde_json::from_str::(&text) + .and_then(|json| serde_json::to_string_pretty(&json)) + .unwrap_or(text) +} + +/// Cut `text` to at most `max_bytes` on a char boundary. +/// Returns the text and the number of bytes that were cut off. +fn truncate(mut text: String, max_bytes: Option) -> (String, usize) { + let Some(max_bytes) = max_bytes else { + return (text, 0); + }; + if text.len() <= max_bytes { + return (text, 0); + } + let mut cut = max_bytes; + while !text.is_char_boundary(cut) { + cut -= 1; + } + let truncated_bytes = text.len() - cut; + text.truncate(cut); + (text, truncated_bytes) +} + +fn sha256_hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .fold(String::with_capacity(64), |mut hex, byte| { + let _ = write!(hex, "{byte:02x}"); + hex + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use http::HeaderValue; + use pretty_assertions::assert_eq; + use std::io::Write; + + const PNG_BYTES: &[u8] = b"\x89PNG\r\n\x1a\n\0\0\0\rIHDR"; + const JSON: &[u8] = br#"{"a":1}"#; + const PRETTY_JSON: &str = "{\n \"a\": 1\n}"; + + fn headers(pairs: &[(&'static str, &'static str)]) -> HeaderMap { + let mut map = HeaderMap::new(); + for (name, value) in pairs { + map.append(*name, HeaderValue::from_static(value)); + } + map + } + + fn text(text: &str) -> LoggedBody { + LoggedBody::Text { + text: text.to_owned(), + truncated_bytes: 0, + } + } + + fn render(pairs: &[(&'static str, &'static str)], bytes: &[u8]) -> LoggedBody { + render_body(&headers(pairs), bytes, true, None) + } + + #[test] + fn png_is_binary() { + assert_eq!( + render(&[("content-type", "image/png")], PNG_BYTES), + LoggedBody::Binary { + content_type: Some("image/png".to_owned()), + len: PNG_BYTES.len(), + sha256: sha256_hex(PNG_BYTES), + } + ); + } + + #[test] + fn sha256_is_lower_hex() { + assert_eq!( + sha256_hex(b"abc"), + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" + ); + } + + #[test] + fn binary_keeps_content_type_parameters() { + let LoggedBody::Binary { content_type, .. } = render( + &[("content-type", "image/tiff; application=geotiff")], + b"II*\0", + ) else { + panic!("expected binary"); + }; + assert_eq!( + content_type.as_deref(), + Some("image/tiff; application=geotiff") + ); + } + + #[test] + fn json_suffix_is_text_case_insensitive() { + assert_eq!( + render( + &[("content-type", "Application/Problem+JSON; charset=utf-8")], + JSON + ), + text(PRETTY_JSON) + ); + assert_eq!( + render(&[("content-type", "application/gml+xml")], b""), + text("") + ); + assert_eq!( + render(&[("content-type", "TEXT/HTML")], b"

"), + text("

") + ); + } + + #[test] + fn decodes_charset() { + assert_eq!( + render( + &[("content-type", "text/plain; charset=\"ISO-8859-1\"")], + b"caf\xe9" + ), + text("café") + ); + } + + #[test] + fn invalid_utf8_text_is_binary() { + assert!(matches!( + render(&[("content-type", "text/plain")], b"caf\xe9"), + LoggedBody::Binary { .. } + )); + assert!(matches!( + render(&[("content-type", "text/plain; charset=bogus")], b"x"), + LoggedBody::Binary { .. } + )); + } + + #[test] + fn without_content_type_sniffs_utf8() { + assert_eq!(render(&[], b"hello"), text("hello")); + assert_eq!( + render(&[], PNG_BYTES), + LoggedBody::Binary { + content_type: None, + len: PNG_BYTES.len(), + sha256: sha256_hex(PNG_BYTES), + } + ); + } + + #[test] + fn decodes_content_encodings() { + let json = [("content-type", "application/json")]; + + let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + gzip.write_all(JSON).unwrap(); + let gzip = gzip.finish().unwrap(); + + let mut zlib = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::fast()); + zlib.write_all(JSON).unwrap(); + let zlib = zlib.finish().unwrap(); + + let mut raw = flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::fast()); + raw.write_all(JSON).unwrap(); + let raw = raw.finish().unwrap(); + + let mut br = Vec::new(); + { + let mut writer = brotli::CompressorWriter::new(&mut br, 4096, 5, 22); + writer.write_all(JSON).unwrap(); + } + + let zstd = + ruzstd::encoding::compress_to_vec(JSON, ruzstd::encoding::CompressionLevel::Fastest); + + for (coding, bytes) in [ + ("gzip", &gzip), + ("x-gzip", &gzip), + ("deflate", &zlib), + ("deflate", &raw), + ("br", &br), + ("zstd", &zstd), + ] { + let mut map = headers(&json); + map.insert(CONTENT_ENCODING, HeaderValue::from_static(coding)); + assert_eq!( + render_body(&map, bytes, true, None), + text(PRETTY_JSON), + "{coding}" + ); + } + + // stacked codings are removed in reverse order + let mut gzip_br = Vec::new(); + { + let mut writer = brotli::CompressorWriter::new(&mut gzip_br, 4096, 5, 22); + writer.write_all(&gzip).unwrap(); + } + assert_eq!( + render( + &[ + ("content-type", "application/json"), + ("content-encoding", "gzip, identity"), + ("content-encoding", "br"), + ], + &gzip_br + ), + text(PRETTY_JSON) + ); + } + + #[test] + fn unknown_or_broken_encoding_is_binary() { + let bytes = b"not compressed"; + assert_eq!( + render( + &[ + ("content-type", "application/json"), + ("content-encoding", "compress") + ], + bytes + ), + LoggedBody::Binary { + content_type: Some("application/json".to_owned()), + len: bytes.len(), + sha256: sha256_hex(bytes), + } + ); + assert!(matches!( + render( + &[("content-type", "text/plain"), ("content-encoding", "gzip")], + bytes + ), + LoggedBody::Binary { .. } + )); + } + + #[test] + fn empty_or_unexpected_body_is_none() { + assert_eq!(render(&[], b""), LoggedBody::None); + assert_eq!( + render_body(&headers(&[]), b"x", false, None), + LoggedBody::None + ); + } + + #[test] + fn response_body_expectation() { + assert!(!response_expects_body(&Method::HEAD, StatusCode::OK)); + assert!(!response_expects_body(&Method::GET, StatusCode::CONTINUE)); + assert!(!response_expects_body(&Method::GET, StatusCode::NO_CONTENT)); + assert!(!response_expects_body( + &Method::GET, + StatusCode::NOT_MODIFIED + )); + assert!(response_expects_body(&Method::GET, StatusCode::OK)); + assert!(response_expects_body(&Method::POST, StatusCode::NOT_FOUND)); + } + + #[tokio::test] + async fn stores_binary_bodies_content_addressed() { + let dir = std::env::temp_dir().join(format!("rlp-test-{}", std::process::id())); + tokio::fs::create_dir_all(&dir).await.unwrap(); + let sha = sha256_hex(PNG_BYTES); + + let path = store_binary(&dir, PNG_BYTES, Some("Image/PNG; foo=bar"), &sha) + .await + .unwrap(); + assert_eq!(path, dir.join(format!("{sha}.png"))); + assert_eq!(tokio::fs::read(&path).await.unwrap(), PNG_BYTES); + + // storing again is a no-op and yields the same path + let again = store_binary(&dir, PNG_BYTES, Some("image/png"), &sha) + .await + .unwrap(); + assert_eq!(again, path); + + let unknown = store_binary(&dir, b"x", None, "abc").await.unwrap(); + assert_eq!(unknown, dir.join("abc.bin")); + + tokio::fs::remove_dir_all(&dir).await.unwrap(); + } + + #[test] + fn truncates_on_char_boundary() { + // "é" is two bytes, so a limit of 2 must not split it + assert_eq!( + render_body( + &headers(&[("content-type", "text/plain")]), + "aébc".as_bytes(), + true, + Some(2) + ), + LoggedBody::Text { + text: "a".to_owned(), + truncated_bytes: 4, + } + ); + assert_eq!( + render_body(&headers(&[]), b"abc", true, Some(3)), + text("abc") + ); + } +} diff --git a/src/config.rs b/src/config.rs index da4394a..5a9b4b4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -20,6 +20,16 @@ pub struct Config { /// Overrides the authorization header and sends a Bearer authentication token (e.g., "TOKEN") #[arg(long, env = "OVERRIDE_BEARER_AUTH")] pub override_bearer_auth: Option, + + /// Maximum number of bytes of a textual body to log; longer bodies are truncated + #[arg(long, env = "MAX_LOGGED_BODY_BYTES")] + pub max_logged_body_bytes: Option, + + /// Directory to store binary bodies in; the log references the files. + /// Relative paths are written as given, so they resolve relative to the log file + /// if the log is written to a file in the current working directory. + #[arg(long, env = "BINARY_BODY_DIR")] + pub binary_body_dir: Option, } #[derive(ValueEnum, Debug, Clone, Copy, PartialEq, Eq, Default)] diff --git a/src/logger.rs b/src/logger.rs index 9f53f17..5d9b01d 100644 --- a/src/logger.rs +++ b/src/logger.rs @@ -1,12 +1,8 @@ -use anyhow::Context; +use crate::body::{LoggedBody, render_body, response_expects_body, store_binary}; use bytes::Bytes; -use flate2::read::GzDecoder; -use http::HeaderValue; -use http::header::CONTENT_ENCODING; use http_body_util::{BodyExt, Full}; -use serde_json::Value; -use std::io::{self, BufWriter}; -use std::io::{Read, Write}; +use std::io::{self, BufWriter, Write}; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Mutex, MutexGuard}; #[derive(Debug)] @@ -42,6 +38,21 @@ impl RequestResponseLogger for DevNullLogger { #[derive(Debug)] pub struct VSCodeRestLogger { writer: LockableWriter, + options: VSCodeRestLoggerOptions, +} + +#[derive(Debug, Default, Clone)] +pub struct VSCodeRestLoggerOptions { + /// Truncate logged text bodies after this many bytes. + pub max_text_body_bytes: Option, + /// Store binary bodies in this directory and link them from the log. + pub binary_body_dir: Option, +} + +/// A body as it is written to the log, with the outcome of storing it to a file. +struct BodyOutput { + body: LoggedBody, + file: Option>, } #[async_trait::async_trait] @@ -51,19 +62,17 @@ impl RequestResponseLogger for VSCodeRestLogger { LogEntry { request, response }: LogEntry, ) -> anyhow::Result<()> { let (request_header, request_body) = request.into_parts(); - let request_body = body_to_string( - request_body, - ContentDecoder::try_from(request_header.headers.get(CONTENT_ENCODING))?, - ) - .await - .context("Failed to decode request body")?; + let request_body = self + .prepare_body(&request_header.headers, collect(request_body).await, true) + .await; let (response_header, response_body) = response.into_parts(); - let response_body = body_to_string( - response_body, - ContentDecoder::try_from(response_header.headers.get(CONTENT_ENCODING))?, - ) - .await - .context("failed to decode response body")?; + let response_body = self + .prepare_body( + &response_header.headers, + collect(response_body).await, + response_expects_body(&request_header.method, response_header.status), + ) + .await; let mut w = self.writer.lock()?; Self::log_request(&mut w, request_header, request_body)?; Self::log_response(&mut w, response_header, response_body)?; @@ -74,30 +83,84 @@ impl RequestResponseLogger for VSCodeRestLogger { } impl VSCodeRestLogger { - pub fn from_stdout(stdout: io::Stdout) -> Self { + pub fn from_stdout(stdout: io::Stdout, options: VSCodeRestLoggerOptions) -> Self { Self { writer: LockableWriter::Stdout(stdout), + options, } } #[cfg(test)] - pub fn from_buf_writer(buf_writer: Arc>>>) -> Self { + pub fn from_buf_writer( + buf_writer: Arc>>>, + options: VSCodeRestLoggerOptions, + ) -> Self { Self { writer: LockableWriter::BufWriter(buf_writer), + options, } } + async fn prepare_body( + &self, + headers: &http::HeaderMap, + bytes: Bytes, + expect_body: bool, + ) -> BodyOutput { + let body = render_body( + headers, + &bytes, + expect_body, + self.options.max_text_body_bytes, + ); + let file = match (&body, &self.options.binary_body_dir) { + ( + LoggedBody::Binary { + content_type, + sha256, + .. + }, + Some(dir), + ) => Some(store_binary(dir, &bytes, content_type.as_deref(), sha256).await), + _ => None, + }; + BodyOutput { body, file } + } + fn log_request( writer: &mut W, header: http::request::Parts, - body: String, + BodyOutput { body, file }: BodyOutput, ) -> anyhow::Result<()> { + writeln!(writer, "\n###\n")?; + + // Comments go before the request line, because inside the body they would be sent as content. + match &body { + LoggedBody::Text { + truncated_bytes, .. + } if *truncated_bytes > 0 => { + writeln!(writer, "# Body: {truncated_bytes} more bytes truncated")?; + } + LoggedBody::Binary { + content_type, + len, + sha256, + } => { + writeln!( + writer, + "# Body: {}", + LoggedBody::binary_summary(content_type.as_deref(), *len, sha256) + )?; + if let Some(Err(e)) = &file { + writeln!(writer, "# Body could not be stored: {e}")?; + } + } + _ => {} + } + writeln!( writer, - "\n\ - ###\n\ - \n\ - {method} {uri}", + "{method} {uri}", method = header.method, uri = header.uri )?; @@ -110,12 +173,13 @@ impl VSCodeRestLogger { )?; } - if !body.is_empty() { - let body_string = serde_json::from_str::(&body) - .and_then(|json| serde_json::to_string_pretty(&json)) - .unwrap_or(body); - - writeln!(writer, "\n{body_string}")?; + match (body, file) { + (LoggedBody::Text { text, .. }, _) => writeln!(writer, "\n{text}")?, + // the request body is read from the file when the request is sent again + (LoggedBody::Binary { .. }, Some(Ok(path))) => { + writeln!(writer, "\n< {}", file_reference(&path))?; + } + _ => {} } Ok(()) @@ -124,7 +188,7 @@ impl VSCodeRestLogger { fn log_response( writer: &mut W, header: http::response::Parts, - body: String, + BodyOutput { body, file }: BodyOutput, ) -> anyhow::Result<()> { writeln!( writer, @@ -145,77 +209,55 @@ impl VSCodeRestLogger { } writeln!(writer, "#")?; - if !body.is_empty() { - let body_string = serde_json::from_str::(&body) - .and_then(|json| serde_json::to_string_pretty(&json)) - .unwrap_or(body) - .replace("\n", "\n# "); // Prefix each line with a comment - writeln!( - writer, - "# Body:\n\ - # {body_string}\n\ - #" - )?; + match body { + LoggedBody::None => {} + LoggedBody::Text { + text, + truncated_bytes, + } => { + let text = text.replace('\n', "\n# "); // Prefix each line with a comment + writeln!(writer, "# Body:\n# {text}")?; + if truncated_bytes > 0 { + writeln!(writer, "# … {truncated_bytes} more bytes truncated")?; + } + writeln!(writer, "#")?; + } + LoggedBody::Binary { + content_type, + len, + sha256, + } => { + writeln!( + writer, + "# Body: {}", + LoggedBody::binary_summary(content_type.as_deref(), len, &sha256) + )?; + match file { + Some(Ok(path)) => writeln!(writer, "# File: {}", file_reference(&path))?, + Some(Err(e)) => writeln!(writer, "# Body could not be stored: {e}")?, + None => {} + } + writeln!(writer, "#")?; + } } Ok(()) } } -async fn body_to_string( - body: Full, - content_encoding: ContentDecoder, -) -> anyhow::Result { - let collected = body - .collect() - .await - .map_err(|e| io::Error::other(format!("Failed to collect body: {e}")))?; - let bytes = collected.to_bytes(); - - content_encoding - .decode(&bytes) - .with_context(|| format!("Failed to decode body: {}", String::from_utf8_lossy(&bytes))) -} - -enum ContentDecoder { - None, - Gzip, -} - -impl TryFrom> for ContentDecoder { - type Error = anyhow::Error; - - fn try_from(value: Option<&HeaderValue>) -> Result { - match value { - Some(header_value) => match header_value.to_str() { - Ok("gzip") => Ok(ContentDecoder::Gzip), - _ => anyhow::bail!("Unsupported content encoding: {header_value:?}"), - }, - None => Ok(ContentDecoder::None), - } +/// Format a path for the log. Relative paths get a `./` prefix, as in the HTTP client examples. +fn file_reference(path: &Path) -> String { + if path.is_relative() && !path.starts_with(".") && !path.starts_with("..") { + format!("./{}", path.display()) + } else { + path.display().to_string() } } -impl ContentDecoder { - fn decode(&self, data: &[u8]) -> anyhow::Result { - if data.is_empty() { - return Ok(String::new()); - } - - match self { - ContentDecoder::None => { - String::from_utf8(data.to_vec()).context("Failed to convert body to string") - } - ContentDecoder::Gzip => { - let mut decoder = GzDecoder::new(data); - let mut decoded_data = String::new(); - decoder - .read_to_string(&mut decoded_data) - .context("Failed to decode gzip data")?; - Ok(decoded_data) - } - } - } +/// Collect a fully buffered body. This cannot fail. +async fn collect(body: Full) -> Bytes { + let Ok(collected) = body.collect().await; + collected.to_bytes() } #[derive(Debug)] @@ -268,7 +310,7 @@ mod tests { #[tokio::test] async fn it_logs_requests_and_responses() { let buf_writer = Arc::new(Mutex::new(BufWriter::new(Vec::new()))); - let mut logger = VSCodeRestLogger::from_buf_writer(buf_writer.clone()); + let mut logger = VSCodeRestLogger::from_buf_writer(buf_writer.clone(), Default::default()); let request = hyper::Request::builder() .method("GET") @@ -295,4 +337,128 @@ mod tests { assert_eq!(logged_string, include_str!("../test-data/logged.http")); } + + async fn log( + request: hyper::Request>, + response: hyper::Response>, + ) -> String { + log_with(Default::default(), request, response).await + } + + async fn log_with( + options: VSCodeRestLoggerOptions, + request: hyper::Request>, + response: hyper::Response>, + ) -> String { + let buf_writer = Arc::new(Mutex::new(BufWriter::new(Vec::new()))); + let mut logger = VSCodeRestLogger::from_buf_writer(buf_writer.clone(), options); + logger + .log_request_response(LogEntry { request, response }) + .await + .unwrap(); + let logged_output = buf_writer.lock().unwrap().get_ref().clone(); + String::from_utf8(logged_output).unwrap() + } + + #[tokio::test] + async fn it_logs_binary_responses_as_placeholder() { + let request = hyper::Request::builder() + .method("GET") + .uri("http://example.com/tile.png") + .body(Full::default()) + .unwrap(); + let response = hyper::Response::builder() + .status(200) + .header("Content-Type", "image/png") + .body(Full::from(Bytes::from_static(b"abc"))) + .unwrap(); + + assert_eq!( + log(request, response).await, + "\n###\n\nGET http://example.com/tile.png\n\n### Response ###\n#\n# Status Code: 200 OK\n\ + # Response Headers:\n# content-type: image/png\n#\n\ + # Body: image/png, 3 bytes, sha256 ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad\n#\n" + ); + } + + #[tokio::test] + async fn it_logs_no_body_for_head_responses() { + let request = hyper::Request::builder() + .method("HEAD") + .uri("http://example.com/test") + .body(Full::default()) + .unwrap(); + // a body that must be ignored + let response = hyper::Response::builder() + .status(200) + .header("Content-Type", "application/json") + .body(Full::from(Bytes::from_static(b"{}"))) + .unwrap(); + + assert_eq!( + log(request, response).await, + "\n###\n\nHEAD http://example.com/test\n\n### Response ###\n#\n# Status Code: 200 OK\n\ + # Response Headers:\n# content-type: application/json\n#\n" + ); + } + + #[tokio::test] + async fn it_stores_binary_bodies_in_files() { + const SHA_ABC: &str = "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"; + let dir = std::env::temp_dir().join(format!("rlp-logger-test-{}", std::process::id())); + tokio::fs::create_dir_all(&dir).await.unwrap(); + + let request = hyper::Request::builder() + .method("PUT") + .uri("http://example.com/upload") + .header("Content-Type", "image/tiff; application=geotiff") + .body(Full::from(Bytes::from_static(b"abc"))) + .unwrap(); + let response = hyper::Response::builder() + .status(200) + .header("Content-Type", "image/png") + .body(Full::from(Bytes::from_static(b"abc"))) + .unwrap(); + + let logged = log_with( + VSCodeRestLoggerOptions { + binary_body_dir: Some(dir.clone()), + ..Default::default() + }, + request, + response, + ) + .await; + + let tif = dir.join(format!("{SHA_ABC}.tif")); + let png = dir.join(format!("{SHA_ABC}.png")); + assert_eq!( + logged, + format!( + "\n###\n\n\ + # Body: image/tiff; application=geotiff, 3 bytes, sha256 {SHA_ABC}\n\ + PUT http://example.com/upload\n\ + content-type: image/tiff; application=geotiff\n\ + \n< {tif}\n\ + \n### Response ###\n#\n# Status Code: 200 OK\n\ + # Response Headers:\n# content-type: image/png\n#\n\ + # Body: image/png, 3 bytes, sha256 {SHA_ABC}\n\ + # File: {png}\n#\n", + tif = tif.display(), + png = png.display(), + ) + ); + assert_eq!(tokio::fs::read(&tif).await.unwrap(), b"abc"); + assert_eq!(tokio::fs::read(&png).await.unwrap(), b"abc"); + + tokio::fs::remove_dir_all(&dir).await.unwrap(); + } + + #[test] + fn it_prefixes_relative_file_references() { + assert_eq!(file_reference(Path::new("bodies/a.png")), "./bodies/a.png"); + assert_eq!(file_reference(Path::new("./a.png")), "./a.png"); + assert_eq!(file_reference(Path::new("../a.png")), "../a.png"); + assert_eq!(file_reference(Path::new("/tmp/a.png")), "/tmp/a.png"); + } } diff --git a/src/main.rs b/src/main.rs index 7798ca1..c5d8f9b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -9,12 +9,13 @@ use tokio::{ sync::mpsc::{Sender, channel}, }; -use crate::logger::{DevNullLogger, LogEntry, VSCodeRestLogger}; +use crate::logger::{DevNullLogger, LogEntry, VSCodeRestLogger, VSCodeRestLoggerOptions}; use crate::{ config::{Config, LoggerType}, logger::RequestResponseLogger, }; +mod body; mod config; mod logger; @@ -44,7 +45,18 @@ async fn main() { let log_sender = match config.logger { LoggerType::DevNull => log_channel(DevNullLogger), - LoggerType::Vscode => log_channel(VSCodeRestLogger::from_stdout(io::stdout())), + LoggerType::Vscode => { + if let Some(dir) = &config.binary_body_dir { + std::fs::create_dir_all(dir).expect("Unable to create binary body directory"); + } + log_channel(VSCodeRestLogger::from_stdout( + io::stdout(), + VSCodeRestLoggerOptions { + max_text_body_bytes: config.max_logged_body_bytes, + binary_body_dir: config.binary_body_dir, + }, + )) + } }; let logger = ReplayLogger::new(log_sender, target_address, config.override_bearer_auth); From 34a883af1f705add36a9aa5a04bef52c1d049ffc Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Wed, 11 Nov 2026 16:21:58 +0100 Subject: [PATCH 2/3] - **Graceful shutdown.** On SIGINT or SIGTERM the proxy waits up to 30 s for open connections, then writes all queued log entries before exiting. A second signal skips the wait. --- Cargo.toml | 2 +- examples/demo.sh | 6 ++- src/main.rs | 101 +++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 93 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 51ccd04..d6b6dc8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -64,7 +64,7 @@ hyper-util = { version = "0.1", features = [ ruzstd = "0.9.1" serde_json = "1.0" sha2 = "0.11.1" -tokio = { version = "1.53", features = ["fs", "macros", "net", "rt-multi-thread", "signal"] } +tokio = { version = "1.53", features = ["fs", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } url = { version = "2.5", features = [] } [dev-dependencies] diff --git a/examples/demo.sh b/examples/demo.sh index 54ad370..a726751 100755 --- a/examples/demo.sh +++ b/examples/demo.sh @@ -66,8 +66,10 @@ send -I "$PROXY/collections" # HEAD: no body send "$PROXY/cached" # 304: no body rm -f "$OUT/upload.tif" -# Give the logger time to write the last entries before shutting down. -sleep 0.5 +# The proxy writes all pending log entries before it exits. +proxy_pid="${pids[1]}" +kill -TERM "$proxy_pid" +wait "$proxy_pid" || true cleanup wait 2>/dev/null || true diff --git a/src/main.rs b/src/main.rs index c5d8f9b..e398353 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,10 +3,14 @@ use http::Method; use http_body_util::{BodyExt, Empty, Full, combinators::BoxBody}; use hyper::{Request, Response, body::Incoming, service::Service, upgrade::Upgraded}; use hyper_util::rt::TokioIo; -use std::{io, net::SocketAddr, pin::Pin}; +use std::{io, net::SocketAddr, pin::Pin, time::Duration}; use tokio::{ net::{TcpListener, TcpStream}, - sync::mpsc::{Sender, channel}, + sync::{ + mpsc::{Sender, channel}, + oneshot, + }, + task::JoinHandle, }; use crate::logger::{DevNullLogger, LogEntry, VSCodeRestLogger, VSCodeRestLoggerOptions}; @@ -23,6 +27,8 @@ type ClientBuilder = hyper::client::conn::http1::Builder; const FALLBACK_TARGET_PORT: u16 = 80; const CHANNEL_BUFFER_SIZE: usize = 64; +/// How long to wait for open connections to finish on shutdown. +const SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(30); /// The main entry point of the Request Logging Proxy application. /// It sets up the proxy server, handles incoming connections, @@ -43,7 +49,7 @@ async fn main() { .socket_addrs(|| Some(FALLBACK_TARGET_PORT)) .expect("Unable to resolve target address")[0]; - let log_sender = match config.logger { + let log_channel = match config.logger { LoggerType::DevNull => log_channel(DevNullLogger), LoggerType::Vscode => { if let Some(dir) = &config.binary_body_dir { @@ -58,7 +64,11 @@ async fn main() { )) } }; - let logger = ReplayLogger::new(log_sender, target_address, config.override_bearer_auth); + let logger = ReplayLogger::new( + log_channel.sender.clone(), + target_address, + config.override_bearer_auth, + ); let server = hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()) .preserve_header_case(true) @@ -89,24 +99,85 @@ async fn main() { } } } + + eprintln!("Waiting for open connections to finish (press CTRL+C again to skip)..."); + tokio::select! { + () = graceful.shutdown() => {} + () = tokio::time::sleep(SHUTDOWN_TIMEOUT) => { + eprintln!("Timed out after {SHUTDOWN_TIMEOUT:?} waiting for open connections"); + } + signal_str = shutdown_signal() => { + eprintln!("Second shutdown signal received: {signal_str}"); + } + } + + drop(logger); + log_channel.close().await; + eprintln!("All log entries written"); +} + +/// A channel to a background task that writes log entries. +struct LogChannel { + sender: Sender, + close: oneshot::Sender<()>, + task: JoinHandle<()>, +} + +impl LogChannel { + /// Stop accepting new entries and wait until all queued entries are written. + async fn close(self) { + let Self { + sender, + close, + task, + } = self; + drop(sender); + // the task may already have finished if all senders are gone + let _ = close.send(()); + if let Err(e) = task.await { + eprintln!("Logger task failed: {e}"); + } + } } /// Create a logging channel that processes request/response pairs sequentially. /// This makes sure that log entries are not interleaved. /// The channel spawns a background task that receives request/response pairs /// and calls the provided logger's `log_request_response` method. -fn log_channel(mut logger: L) -> Sender { - let (tx, mut rx) = channel::(CHANNEL_BUFFER_SIZE); +fn log_channel(mut logger: L) -> LogChannel { + let (sender, mut rx) = channel::(CHANNEL_BUFFER_SIZE); + let (close, mut close_rx) = oneshot::channel::<()>(); + + let task = tokio::spawn(async move { + loop { + tokio::select! { + entry = rx.recv() => match entry { + Some(entry) => log_entry(&mut logger, entry).await, + // all senders are gone, so nothing is queued anymore + None => return, + }, + _ = &mut close_rx => break, + } + } - tokio::spawn(async move { + // Reject new entries, but write all entries that are already queued. + rx.close(); while let Some(entry) = rx.recv().await { - if let Err(e) = logger.log_request_response(entry).await { - eprintln!("Error logging request/response: {e:?}"); - } + log_entry(&mut logger, entry).await; } }); - tx + LogChannel { + sender, + close, + task, + } +} + +async fn log_entry(logger: &mut L, entry: LogEntry) { + if let Err(e) = logger.log_request_response(entry).await { + eprintln!("Error logging request/response: {e:?}"); + } } /// A service that proxies requests to a target address @@ -165,13 +236,17 @@ impl ReplayLogger { Response::from_parts(parts, Full::from(full_body_bytes.clone())) }; - self.logger + if self + .logger .send(LogEntry { request, response: response.clone(), }) .await - .unwrap(); + .is_err() + { + eprintln!("Logger is shut down, dropping log entry"); + } Ok(response) } From 90c328bb55a1b9e42e73d02a439e7c07f5723cb8 Mon Sep 17 00:00:00 2001 From: Christian Beilschmidt Date: Wed, 11 Nov 2026 16:22:38 +0100 Subject: [PATCH 3/3] chore: bump version to 0.4.0 --- Cargo.lock | 2 +- Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fd012b8..6b5390f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -686,7 +686,7 @@ dependencies = [ [[package]] name = "request-logging-proxy" -version = "0.3.0" +version = "0.4.0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index d6b6dc8..c6f354b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "request-logging-proxy" -version = "0.3.0" +version = "0.4.0" edition = "2024" authors = ["Christian Beilschmidt "] description = "A simple HTTP proxy that logs requests and responses"