diff --git a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml index ed30f067..3f090b81 100644 --- a/ansible/host_vars/fastpath.dev.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.dev.ooni.io/vars.yml @@ -4,4 +4,6 @@ bucket_name: "ooni-data-eu-fra-test" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "3" env: "dev" -fastpath_image: "ooni/fastpath:20260903-7c51eb5d" +# This image drains its queue on docker stop, give it 120s to drain (docker default 10s) +fastpath_image: "ooni/fastpath:20261117-27f2b28c" +fastpath_stop_timeout: 120 diff --git a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml index 986a62f6..305740d1 100644 --- a/ansible/host_vars/fastpath.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath.prod.ooni.io/vars.yml @@ -1,7 +1,10 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni" +# clickhouse-driver moves to an alt_host only when it can't connect, and stays there: restart fastpath to mvove back +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni?alt_hosts=clickhouse2.prod.ooni.io" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "1" env: "prod" -fastpath_image: "ooni/fastpath:20260921-ac774cb4" +# This image drains its queue on docker stop, give it 120s to drain (docker default 10s) +fastpath_image: "ooni/fastpath:20261117-27f2b28c" +fastpath_stop_timeout: 120 diff --git a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml index 26123dcc..1830fe89 100644 --- a/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml +++ b/ansible/host_vars/fastpath2.prod.ooni.io/vars.yml @@ -1,7 +1,10 @@ s3_ooni_open_data_access_key: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/s3_ooni_open_data_access_key', profile='oonidevops_user_prod') }}" -clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni" +# clickhouse-driver moves to an alt_host only when it can't connect, and stays there: restart fastpath to mvove back +clickhouse_url: "clickhouse://fastpath:{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/clickhouse_fastpath_password', profile='oonidevops_user_prod') }}@clickhouse1.prod.ooni.io/ooni?alt_hosts=clickhouse2.prod.ooni.io" bucket_name: "ooni-data-eu-fra" # COLLECTOR ID SHOULD BE DIFFERENT BETWEEN EACH FASTPATH INSTANCE collector_id: "4" env: "prod" -fastpath_image: "ooni/fastpath:20260921-ac774cb4" +# This image drains its queue on docker stop, give it 120s to drain (docker default 10s) +fastpath_image: "ooni/fastpath:20261117-27f2b28c" +fastpath_stop_timeout: 120 diff --git a/ansible/roles/fastpath/tasks/main.yml b/ansible/roles/fastpath/tasks/main.yml index 8b381b65..164bd0df 100644 --- a/ansible/roles/fastpath/tasks/main.yml +++ b/ansible/roles/fastpath/tasks/main.yml @@ -152,6 +152,12 @@ image: "{{ fastpath_image }}" state: started restart_policy: always + # Seconds docker stop waits before SIGKILL. With ooni/backend's + # fastpath_drain_on_shutdown, fastpath drains its queue into ClickHouse on + # SIGTERM; set this per host once its image has that fix. Without the fix, + # a stop hangs until this timeout and the queue is lost anyway. Unset, it's + # left out, so the container keeps Docker's 10 s and isn't recreated. + stop_timeout: "{{ fastpath_stop_timeout | default(omit) }}" user: "{{user_uid.stdout}}:{{user_gid.stdout}}" # use network mode = host to allow traffic from fastpath to the statsd exporter without # creating a network with redirection rules to match the ports @@ -164,7 +170,10 @@ - /home/fastpath/app/run_fastpath - --write-to-disk env: - FASTPATH_QUEUE_PUT_TIMEOUT: "-1" + # Don't wait for room in a full queue: refuse the measurement at once + # so ooniprobe sends it to another fastpath or to S3. Blocking holds + # ooniprobe's threads until its request times out. + FASTPATH_QUEUE_PUT_TIMEOUT: "0" FASTPATH_NUM_WORKERS: "10" volumes: - /opt/{{fastpath_user}}/backend/fastpath/fastpath.conf:/etc/ooni/fastpath.conf