From 92e933c51bf3900658038258b2424060c183ee25 Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Thu, 8 Oct 2026 17:39:09 +0300 Subject: [PATCH 1/4] fetch the latest OpenID Conformance Suite release --- .github/workflows/test.yaml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 79156c9ef..d4bdb48e7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -214,7 +214,6 @@ jobs: ssp-version: ["v2.5.0"] env: SUITE_BASE_URL: https://localhost.emobix.co.uk:8443 - VERSION: release-v5.1.39 steps: - uses: actions/checkout@v6 with: @@ -223,6 +222,18 @@ jobs: run: | pip install --upgrade pip pip install httpx + - name: Install jq + run: | + sudo apt-get update + sudo apt-get install --no-install-recommends -y jq + - name: Resolve latest Conformance Suite release + run: | + set -euo pipefail + VERSION="$(curl --location --fail --silent --show-error \ + https://gitlab.com/api/v4/projects/openid%2Fconformance-suite/releases/permalink/latest \ + | jq --raw-output --exit-status '.tag_name')" + echo "Using Conformance Suite release: $VERSION" + echo "VERSION=$VERSION" >> "$GITHUB_ENV" - name: Conformance Suite Checkout if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: git clone --depth 1 --single-branch --branch $VERSION https://gitlab.com/openid/conformance-suite.git From 2aad0fa615eb793debc1fced0a225d00130b7e20 Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Thu, 8 Oct 2026 18:34:27 +0300 Subject: [PATCH 2/4] add failure diagnostic logging add failure diagnostic logging --- .github/workflows/test.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d4bdb48e7..a09fbe65c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -270,6 +270,14 @@ jobs: - name: Run RP backchannel run: | ./conformance-suite/scripts/run-test-plan.py "oidcc-backchannel-rp-initiated-logout-certification-test-plan[response_type=code][client_registration=static_client]" ./main/conformance-tests/conformance-back-channel-logout-ci.json + - name: Dump Incomplete Test Log to Console + if: failure() + run: | + echo "=== SimpleSAMLphp OP Container Log (Last 100 lines) ===" + docker compose -f ./main/docker/docker-compose.yml --project-directory ./main logs --tail=100 || true + + echo "=== Conformance Runner Log Entries ===" + curl -sk "https://localhost.emobix.co.uk:8443/api/log" | jq 'if type == "array" then .[] elif type == "object" and has("data") and (.data | type == "array") then .data[] else empty end | select(.result != "PASSED") | {testId: ._id, testName: .testName, status: .status, result: .result}' || true - name: Stop SSP working-directory: ./main run: | From a669d3d76ad6127d38b2aa749905e55b07e957f1 Mon Sep 17 00:00:00 2001 From: Ioannis Igoumenos Date: Thu, 8 Oct 2026 19:46:59 +0300 Subject: [PATCH 3/4] Updated docker/conformance.sql to replace httpd:8443 with nginx:8443 for back-channel logout client endpoints in conformance testing fixtures. --- docker/conformance.sql | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/conformance.sql b/docker/conformance.sql index 4194f9e85..8fe1728ca 100644 --- a/docker/conformance.sql +++ b/docker/conformance.sql @@ -45,13 +45,13 @@ CREATE TABLE oidc_client ( expires_at TIMESTAMP NULL DEFAULT NULL, is_federated BOOLEAN NOT NULL DEFAULT false ); --- Used 'httpd' host for back-channel logout url (https://httpd:8443/test/a/simplesamlphp-module-oidc/backchannel_logout) +-- Used 'nginx' host for back-channel logout url (https://nginx:8443/test/a/simplesamlphp-module-oidc/backchannel_logout) -- since this is the hostname of conformance server while running in container environment -INSERT INTO oidc_client VALUES('_55a99a1d298da921cb27d700d4604352e51171ebc4','_8967dd97d07cc59db7055e84ac00e79005157c1132','Conformance Client 1',replace('Client 1 for Conformance Testing https://openid.net/certification/connect_op_testing/\n','\n',char(10)),'example-userpass','["https:\/\/localhost.emobix.co.uk:8443\/test\/a\/simplesamlphp-module-oidc\/callback","https:\/\/www.certification.openid.net\/test\/a\/simplesamlphp-module-oidc\/callback"]','["openid","profile","email","address","phone","offline_access"]',1,1,NULL,'["https:\/\/localhost.emobix.co.uk:8443\/test\/a\/simplesamlphp-module-oidc\/post_logout_redirect"]','https://httpd:8443/test/a/simplesamlphp-module-oidc/backchannel_logout',NULL,NULL, NULL, NULL, NULL, NULL, 'manual', NULL, NULL, NULL, false); +INSERT INTO oidc_client VALUES('_55a99a1d298da921cb27d700d4604352e51171ebc4','_8967dd97d07cc59db7055e84ac00e79005157c1132','Conformance Client 1',replace('Client 1 for Conformance Testing https://openid.net/certification/connect_op_testing/\n','\n',char(10)),'example-userpass','["https:\/\/localhost.emobix.co.uk:8443\/test\/a\/simplesamlphp-module-oidc\/callback","https:\/\/www.certification.openid.net\/test\/a\/simplesamlphp-module-oidc\/callback"]','["openid","profile","email","address","phone","offline_access"]',1,1,NULL,'["https:\/\/localhost.emobix.co.uk:8443\/test\/a\/simplesamlphp-module-oidc\/post_logout_redirect"]','https://nginx:8443/test/a/simplesamlphp-module-oidc/backchannel_logout',NULL,NULL, NULL, NULL, NULL, NULL, 'manual', NULL, NULL, NULL, false); INSERT INTO oidc_client VALUES('_34efb61060172a11d62101bc804db789f8f9100b0e','_91a4607a1c10ba801268929b961b3f6c067ff82d21','Conformance Client 2','','example-userpass','["https:\/\/localhost.emobix.co.uk:8443\/test\/a\/simplesamlphp-module-oidc\/callback","https:\/\/www.certification.openid.net\/test\/a\/simplesamlphp-module-oidc\/callback"]','["openid","profile","email","offline_access"]',1,1,NULL,NULL,NULL,NULL,NULL, NULL, NULL, NULL, NULL, 'manual', NULL, NULL, NULL, false); INSERT INTO oidc_client VALUES('_0afb7d18e54b2de8205a93e38ca119e62ee321d031','_944e73bbeec7850d32b68f1b5c780562c955967e4e','Conformance Client 3','Client for client_secret_post','example-userpass','["https:\/\/localhost.emobix.co.uk:8443\/test\/a\/simplesamlphp-module-oidc\/callback","https:\/\/www.certification.openid.net\/test\/a\/simplesamlphp-module-oidc\/callback"]','["openid","profile","email"]',1,1,NULL,NULL,NULL,NULL,NULL, NULL, NULL, NULL, NULL, 'manual', NULL, NULL, NULL, false); INSERT INTO oidc_client VALUES('_8957eda35234902ba8343c0cdacac040310f17dfca','_322d16999f9da8b5abc9e9c0c08e853f60f4dc4804','RP-Initiated Logout Client','Client for testing RP-Initiated Logout','example-userpass','["https:\/\/localhost.emobix.co.uk:8443\/test\/a\/simplesamlphp-module-oidc\/callback","https:\/\/www.certification.openid.net\/test\/a\/simplesamlphp-module-oidc\/callback"]','["openid","profile","email","address","phone"]',1,1,NULL,'["https:\/\/localhost.emobix.co.uk:8443\/test\/a\/simplesamlphp-module-oidc\/post_logout_redirect"]',NULL,NULL,NULL, NULL, NULL, NULL, NULL, 'manual', NULL, NULL, NULL, false); -INSERT INTO oidc_client VALUES('_9fe2f7589ece1b71f5ef75a91847d71bc5125ec2a6','_3c0beb20194179c01d7796c6836f62801e9ed4b368','Back-Channel Logout Client','Client for testing Back-Channel Logout','example-userpass','["https:\/\/localhost.emobix.co.uk:8443\/test\/a\/simplesamlphp-module-oidc\/callback","https:\/\/www.certification.openid.net\/test\/a\/simplesamlphp-module-oidc\/callback"]','["openid","profile","email","address","phone"]',1,1,NULL,'["https:\/\/localhost.emobix.co.uk:8443\/test\/a\/simplesamlphp-module-oidc\/post_logout_redirect"]','https://httpd:8443/test/a/simplesamlphp-module-oidc/backchannel_logout',NULL,NULL, NULL, NULL, NULL, NULL, 'manual', NULL, NULL, NULL, false); +INSERT INTO oidc_client VALUES('_9fe2f7589ece1b71f5ef75a91847d71bc5125ec2a6','_3c0beb20194179c01d7796c6836f62801e9ed4b368','Back-Channel Logout Client','Client for testing Back-Channel Logout','example-userpass','["https:\/\/localhost.emobix.co.uk:8443\/test\/a\/simplesamlphp-module-oidc\/callback","https:\/\/www.certification.openid.net\/test\/a\/simplesamlphp-module-oidc\/callback"]','["openid","profile","email","address","phone"]',1,1,NULL,'["https:\/\/localhost.emobix.co.uk:8443\/test\/a\/simplesamlphp-module-oidc\/post_logout_redirect"]','https://nginx:8443/test/a/simplesamlphp-module-oidc/backchannel_logout',NULL,NULL, NULL, NULL, NULL, NULL, 'manual', NULL, NULL, NULL, false); CREATE TABLE oidc_access_token ( id VARCHAR(191) PRIMARY KEY NOT NULL, scopes TEXT, From 83a5f8c316a15be33d3cc25abfdfb1e312eb6b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ivan=C4=8Di=C4=87?= Date: Wed, 4 Nov 2026 10:52:49 +0100 Subject: [PATCH 4/4] Pin the conformance suite at release-v5.3.4 in CI Following the suite's latest release at run time lets a new release turn CI red with no change here, and makes the same commit pass or fail depending on when it runs. The suite had eight releases between August and October 2026. Pin it instead, as wip-version-7 does, so moving to a new release is a deliberate commit. This removes the step resolving the latest release and the jq install. The runner image already has jq. The failure dump now uses the step from wip-version-7, which shows the last 2000 lines of the OP's log rather than 100 lines per container. The /api/log part is gone: it printed nothing in a failed run. The local instructions in docs/5-oidc-conformance.md name the same release. Assisted-by: Claude Code (claude-opus-5-5, implementation) --- .github/workflows/test.yaml | 26 ++++++++------------------ docs/5-oidc-conformance.md | 2 +- 2 files changed, 9 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a09fbe65c..1bd08d09b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -214,6 +214,7 @@ jobs: ssp-version: ["v2.5.0"] env: SUITE_BASE_URL: https://localhost.emobix.co.uk:8443 + VERSION: release-v5.3.4 steps: - uses: actions/checkout@v6 with: @@ -222,18 +223,6 @@ jobs: run: | pip install --upgrade pip pip install httpx - - name: Install jq - run: | - sudo apt-get update - sudo apt-get install --no-install-recommends -y jq - - name: Resolve latest Conformance Suite release - run: | - set -euo pipefail - VERSION="$(curl --location --fail --silent --show-error \ - https://gitlab.com/api/v4/projects/openid%2Fconformance-suite/releases/permalink/latest \ - | jq --raw-output --exit-status '.tag_name')" - echo "Using Conformance Suite release: $VERSION" - echo "VERSION=$VERSION" >> "$GITHUB_ENV" - name: Conformance Suite Checkout if: ${{ steps.cache.outputs.cache-hit != 'true' }} run: git clone --depth 1 --single-branch --branch $VERSION https://gitlab.com/openid/conformance-suite.git @@ -270,14 +259,15 @@ jobs: - name: Run RP backchannel run: | ./conformance-suite/scripts/run-test-plan.py "oidcc-backchannel-rp-initiated-logout-certification-test-plan[response_type=code][client_registration=static_client]" ./main/conformance-tests/conformance-back-channel-logout-ci.json - - name: Dump Incomplete Test Log to Console + - name: Dump OP logs if: failure() + working-directory: ./main + # What the suite reports is only its own side of the exchange, and some failures say nothing there: + # a request the OP never sent reads as a test waiting for a callback until it times out, with no + # failed check anywhere. The reason is in the OP's log, which otherwise leaves with the container. + # Same compose invocation as "Start SSP docker", so this addresses the project that step created. run: | - echo "=== SimpleSAMLphp OP Container Log (Last 100 lines) ===" - docker compose -f ./main/docker/docker-compose.yml --project-directory ./main logs --tail=100 || true - - echo "=== Conformance Runner Log Entries ===" - curl -sk "https://localhost.emobix.co.uk:8443/api/log" | jq 'if type == "array" then .[] elif type == "object" and has("data") and (.data | type == "array") then .data[] else empty end | select(.result != "PASSED") | {testId: ._id, testName: .testName, status: .status, result: .result}' || true + docker compose -f docker/docker-compose.yml --project-directory . logs --no-color --tail 2000 oidc-op || true - name: Stop SSP working-directory: ./main run: | diff --git a/docs/5-oidc-conformance.md b/docs/5-oidc-conformance.md index 420e8c44c..3b2a24445 100644 --- a/docs/5-oidc-conformance.md +++ b/docs/5-oidc-conformance.md @@ -17,7 +17,7 @@ Clone, build, and run the conformance test suite: ```bash git clone https://gitlab.com/openid/conformance-suite.git cd conformance-suite -git checkout release-v5.1.39 +git checkout release-v5.3.4 MAVEN_CACHE=./m2 docker-compose -f builder-compose.yml run builder docker-compose up ```