From ca7def79c781d52f17effb6ab4b3f2bcacecb334 Mon Sep 17 00:00:00 2001 From: Aaron Gibson Date: Tue, 17 Nov 2026 12:47:08 +0000 Subject: [PATCH] fastpath: deploy 20261117-27f2b28c, fall back to clickhouse2 Update fastpath1 and fastpath2 to ooni/fastpath:20261117-27f2b28c which adds the following: - drains its queue into ClickHouse on docker stop (#1326) - retries failed inserts with a backoff, and splits a batch to drop only the rows the server rejects (#1327, #1332) - answers 503 at once when its queue is full (#1330) clickhouse_url: clickhouse1, then clickhouse2 as alt_hosts clickhouse2's performance under oonimeasurements load isn't known and until upgrade it does not have the per-query thread limits, so it only takes writes when clickhouse1 can't be reached. clickhouse-driver stays on the alt host once it has moved there: restart fastpath after clickhouse1 is back. Future work: detect when alt_host is being used and periodically retry and switch back to primary. fastpath role: set fastpath_stop_timeout to 120s so docker stop leaves time to drain FASTPATH_QUEUE_PUT_TIMEOUT 0 instead of -1: refuse a measurement at once when the queue is full, so ooniprobe sends it to another fastpath or to S3 instead of holding its thread until the request times out. --- ansible/host_vars/fastpath.dev.ooni.io/vars.yml | 4 +++- ansible/host_vars/fastpath.prod.ooni.io/vars.yml | 7 +++++-- ansible/host_vars/fastpath2.prod.ooni.io/vars.yml | 7 +++++-- ansible/roles/fastpath/tasks/main.yml | 11 ++++++++++- 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml index ed30f067..3f090b81 100644 --- a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml @@ -4,4 +4,6 @@ bucket_name: "ooni-data-eu-fra-test" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "3" env: "dev" -fastpath_image: "ooni/fastpath:20260903-7c51eb5d" +# This image drains its queue on docker stop, give it 120s to drain (docker default 10s) +fastpath_image: "ooni/fastpath:20261117-27f2b28c" +fastpath_stop_timeout: 120 diff --git a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml index 986a62f6..305740d1 100644 --- a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml @@ -1,7 +1,10 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni" +# clickhouse-driver moves to an alt_host only when it can't connect, and stays there: restart fastpath to mvove back +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni?alt_hosts=clickhouse2.prod.ooni.io" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "1" env: "prod" -fastpath_image: "ooni/fastpath:20260921-ac774cb4" +# This image drains its queue on docker stop, give it 120s to drain (docker default 10s) +fastpath_image: "ooni/fastpath:20261117-27f2b28c" +fastpath_stop_timeout: 120 diff --git a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml index 26123dcc..1830fe89 100644 --- a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml @@ -1,7 +1,10 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni" +# clickhouse-driver moves to an alt_host only when it can't connect, and stays there: restart fastpath to mvove back +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni?alt_hosts=clickhouse2.prod.ooni.io" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "4" env: "prod" -fastpath_image: "ooni/fastpath:20260921-ac774cb4" +# This image drains its queue on docker stop, give it 120s to drain (docker default 10s) +fastpath_image: "ooni/fastpath:20261117-27f2b28c" +fastpath_stop_timeout: 120 diff --git a/ansible/roles/fastpath/tasks/main.yml b/ansible/roles/fastpath/tasks/main.yml index 8b381b65..164bd0df 100644 --- a/ansible/roles/fastpath/tasks/main.yml +++ b/ansible/roles/fastpath/tasks/main.yml @@ -152,6 +152,12 @@ image: "{{ fastpath_image }}" state: started restart_policy: always + # Seconds docker stop waits before SIGKILL. With ooni/backend's + # fastpath_drain_on_shutdown, fastpath drains its queue into ClickHouse on + # SIGTERM; set this per host once its image has that fix. Without the fix, + # a stop hangs until this timeout and the queue is lost anyway. Unset, it's + # left out, so the container keeps Docker's 10 s and isn't recreated. + stop_timeout: "{{ fastpath_stop_timeout | default(omit) }}" user: "{{user_uid.stdout}}:{{user_gid.stdout}}" # use network mode = host to allow traffic from fastpath to the statsd exporter without # creating a network with redirection rules to match the ports @@ -164,7 +170,10 @@ - /home/fastpath/app/run_fastpath - --write-to-disk env: - FASTPATH_QUEUE_PUT_TIMEOUT: "-1" + # Don't wait for room in a full queue: refuse the measurement at once + # so ooniprobe sends it to another fastpath or to S3. Blocking holds + # ooniprobe's threads until its request times out. + FASTPATH_QUEUE_PUT_TIMEOUT: "0" FASTPATH_NUM_WORKERS: "10" volumes: - /opt/{{fastpath_user}}/backend/fastpath/fastpath.conf:/etc/ooni/fastpath.conf