From 1accb3e08baea653577e72f23d3c637dbccf58ef Mon Sep 17 00:00:00 2001 From: serge-rider Date: Sat, 7 Nov 2026 19:45:45 +0100 Subject: [PATCH 1/7] dbeaver/pro#11574 Resolve project.deps recursively --- clone-repositories/action.yml | 29 ++++---------------- clone-repositories/clone.sh | 41 +++++++++++++++++++++++++++++ clone-repositories/test/mock-git.sh | 20 ++++++++++++++ clone-repositories/test/test.sh | 35 ++++++++++++++++++++++++ 4 files changed, 101 insertions(+), 24 deletions(-) create mode 100644 clone-repositories/clone.sh create mode 100644 clone-repositories/test/mock-git.sh create mode 100644 clone-repositories/test/test.sh diff --git a/clone-repositories/action.yml b/clone-repositories/action.yml index c3930e3..4e615d8 100644 --- a/clone-repositories/action.yml +++ b/clone-repositories/action.yml @@ -15,7 +15,7 @@ inputs: [Learn more about creating and using encrypted secrets](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets) default: ${{ github.token }} project_deps_path: - description: "The content of the project.deps file with repository names" + description: "Path to the project.deps file with repository names" required: true runs: @@ -35,32 +35,13 @@ runs: shell: bash env: GH_TOKEN: ${{ inputs.token }} + PROJECT_DEPS_PATH: ${{ inputs.project_deps_path }} + HEAD_BRANCH: ${{ env.GITHUB_HEAD_REF }} + BASE_BRANCH: ${{ env.GITHUB_BASE_REF }} GIT_CONFIG_COUNT: '2' GIT_CONFIG_KEY_0: credential.helper GIT_CONFIG_VALUE_0: '' GIT_CONFIG_KEY_1: credential.https://github.com.helper GIT_CONFIG_VALUE_1: >- !f() { if [ "$1" = get ]; then printf '%s\n' 'username=x-access-token' "password=$GH_TOKEN"; fi; }; f - run: | - exit_code=0 - while IFS= read -r repo || [[ -n $repo ]]; do - echo "Cloning repository: $repo" - { - default_branch=$(git ls-remote --symref "https://github.com/dbeaver/$repo.git" HEAD | grep '^ref:' | awk '{print $2}' | sed 's|refs/heads/||') - - # Try to clone the head branch, then the base branch, and finally the default branch - git clone --depth 1 -b "${{ env.GITHUB_HEAD_REF }}" "https://github.com/dbeaver/$repo.git" 2>/dev/null || \ - git clone --depth 1 -b "${{ env.GITHUB_BASE_REF }}" "https://github.com/dbeaver/$repo.git" 2>/dev/null || \ - git clone --depth 1 -b "$default_branch" "https://github.com/dbeaver/$repo.git" - - # Capture the exit code of the last command - if [[ $? -ne 0 ]]; then - exit_code=1 # Set exit_code to 1 if any clone command fails - fi - } & - done < "${{ inputs.project_deps_path }}" - - wait - - # Exit with the appropriate exit code - exit $exit_code + run: bash "${{ github.action_path }}/clone.sh" diff --git a/clone-repositories/clone.sh b/clone-repositories/clone.sh new file mode 100644 index 0000000..0566450 --- /dev/null +++ b/clone-repositories/clone.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set -euo pipefail + +declare -A visited=() +queue=() + +enqueue_dependencies() { + local file=$1 repo + [[ -f $file ]] || return 0 + while IFS= read -r repo || [[ -n $repo ]]; do + repo=${repo%$'\r'} + repo=${repo#"${repo%%[![:space:]]*}"} + repo=${repo%"${repo##*[![:space:]]}"} + [[ -z $repo || $repo == \#* ]] && continue + if [[ ! $repo =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]]; then + echo "Invalid repository name in $file: $repo" >&2 + exit 1 + fi + queue+=("$repo") + done < "$file" +} + +if [[ ! -f $PROJECT_DEPS_PATH ]]; then + echo "Dependency file not found: $PROJECT_DEPS_PATH" >&2 + exit 1 +fi +enqueue_dependencies "$PROJECT_DEPS_PATH" +for ((index=0; index<${#queue[@]}; index++)); do + repo=${queue[index]} + [[ -v visited[$repo] ]] && continue + visited[$repo]=1 + if [[ ! -d $repo ]]; then + echo "Cloning repository: $repo" + url="https://github.com/dbeaver/$repo.git" + default_branch=$(git ls-remote --symref "$url" HEAD | awk '/^ref:/ {sub("refs/heads/", "", $2); print $2; exit}') + git clone --depth 1 -b "${HEAD_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ + git clone --depth 1 -b "${BASE_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ + git clone --depth 1 -b "$default_branch" "$url" + fi + enqueue_dependencies "$repo/project.deps" +done diff --git a/clone-repositories/test/mock-git.sh b/clone-repositories/test/mock-git.sh new file mode 100644 index 0000000..e811120 --- /dev/null +++ b/clone-repositories/test/mock-git.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +case "$1" in + ls-remote) + printf 'ref: refs/heads/main\tHEAD\n' + ;; + clone) + branch=$5 + url=$6 + repo=${url##*/} + repo=${repo%.git} + if [[ $repo == second && $branch == feature ]]; then + exit 1 + fi + printf '%s %s\n' "$repo" "$branch" >> "$CLONE_LOG" + cp -R "$FIXTURES/$repo" "$repo" + ;; + *) exit 1 ;; +esac diff --git a/clone-repositories/test/test.sh b/clone-repositories/test/test.sh new file mode 100644 index 0000000..3051b42 --- /dev/null +++ b/clone-repositories/test/test.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +action_dir=$(cd "$(dirname "$0")/.." && pwd) +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +mkdir -p "$tmp/bin" "$tmp/fixtures/first" "$tmp/fixtures/second" "$tmp/fixtures/third" "$tmp/work" +cp "$action_dir/test/mock-git.sh" "$tmp/bin/git" +printf 'second\nthird\n' > "$tmp/fixtures/first/project.deps" +printf 'first\nthird\n' > "$tmp/fixtures/second/project.deps" +printf '# no dependencies\n' > "$tmp/fixtures/third/project.deps" +printf ' first\r\nfirst\n# comment\n\n' > "$tmp/work/project.deps" + +export PATH="$tmp/bin:$PATH" FIXTURES="$tmp/fixtures" CLONE_LOG="$tmp/clones" +export PROJECT_DEPS_PATH=project.deps HEAD_BRANCH=refs/heads/feature BASE_BRANCH=refs/heads/main +( + cd "$tmp/work" + bash "$action_dir/clone.sh" +) + +[[ $(wc -l < "$CLONE_LOG") -eq 3 ]] +grep -qx 'first feature' "$CLONE_LOG" +grep -qx 'second main' "$CLONE_LOG" +grep -qx 'third feature' "$CLONE_LOG" +printf 'missing\n' > "$tmp/work/project.deps" +if (cd "$tmp/work" && bash "$action_dir/clone.sh" >/dev/null 2>&1); then + echo 'Missing repository was accepted' >&2 + exit 1 +fi +PROJECT_DEPS_PATH=does-not-exist +if (cd "$tmp/work" && bash "$action_dir/clone.sh" >/dev/null 2>&1); then + echo 'Missing dependency file was accepted' >&2 + exit 1 +fi +printf 'Recursive clone test passed\n' From 55acef57a30e82c4ed4bca55b2b685c1a6b8e2ce Mon Sep 17 00:00:00 2001 From: serge-rider Date: Sat, 7 Nov 2026 19:58:37 +0100 Subject: [PATCH 2/7] dbeaver/pro#11574 Remove clone action test scripts --- clone-repositories/test/mock-git.sh | 20 ----------------- clone-repositories/test/test.sh | 35 ----------------------------- 2 files changed, 55 deletions(-) delete mode 100644 clone-repositories/test/mock-git.sh delete mode 100644 clone-repositories/test/test.sh diff --git a/clone-repositories/test/mock-git.sh b/clone-repositories/test/mock-git.sh deleted file mode 100644 index e811120..0000000 --- a/clone-repositories/test/mock-git.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -case "$1" in - ls-remote) - printf 'ref: refs/heads/main\tHEAD\n' - ;; - clone) - branch=$5 - url=$6 - repo=${url##*/} - repo=${repo%.git} - if [[ $repo == second && $branch == feature ]]; then - exit 1 - fi - printf '%s %s\n' "$repo" "$branch" >> "$CLONE_LOG" - cp -R "$FIXTURES/$repo" "$repo" - ;; - *) exit 1 ;; -esac diff --git a/clone-repositories/test/test.sh b/clone-repositories/test/test.sh deleted file mode 100644 index 3051b42..0000000 --- a/clone-repositories/test/test.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -action_dir=$(cd "$(dirname "$0")/.." && pwd) -tmp=$(mktemp -d) -trap 'rm -rf "$tmp"' EXIT -mkdir -p "$tmp/bin" "$tmp/fixtures/first" "$tmp/fixtures/second" "$tmp/fixtures/third" "$tmp/work" -cp "$action_dir/test/mock-git.sh" "$tmp/bin/git" -printf 'second\nthird\n' > "$tmp/fixtures/first/project.deps" -printf 'first\nthird\n' > "$tmp/fixtures/second/project.deps" -printf '# no dependencies\n' > "$tmp/fixtures/third/project.deps" -printf ' first\r\nfirst\n# comment\n\n' > "$tmp/work/project.deps" - -export PATH="$tmp/bin:$PATH" FIXTURES="$tmp/fixtures" CLONE_LOG="$tmp/clones" -export PROJECT_DEPS_PATH=project.deps HEAD_BRANCH=refs/heads/feature BASE_BRANCH=refs/heads/main -( - cd "$tmp/work" - bash "$action_dir/clone.sh" -) - -[[ $(wc -l < "$CLONE_LOG") -eq 3 ]] -grep -qx 'first feature' "$CLONE_LOG" -grep -qx 'second main' "$CLONE_LOG" -grep -qx 'third feature' "$CLONE_LOG" -printf 'missing\n' > "$tmp/work/project.deps" -if (cd "$tmp/work" && bash "$action_dir/clone.sh" >/dev/null 2>&1); then - echo 'Missing repository was accepted' >&2 - exit 1 -fi -PROJECT_DEPS_PATH=does-not-exist -if (cd "$tmp/work" && bash "$action_dir/clone.sh" >/dev/null 2>&1); then - echo 'Missing dependency file was accepted' >&2 - exit 1 -fi -printf 'Recursive clone test passed\n' From 35389c666e359088bf5fedb7e54b8ac679b64549 Mon Sep 17 00:00:00 2001 From: serge-rider Date: Sat, 7 Nov 2026 20:00:38 +0100 Subject: [PATCH 3/7] dbeaver/pro#11574 Keep recursive cloning in composite action --- clone-repositories/action.yml | 42 ++++++++++++++++++++++++++++++++++- clone-repositories/clone.sh | 41 ---------------------------------- 2 files changed, 41 insertions(+), 42 deletions(-) delete mode 100644 clone-repositories/clone.sh diff --git a/clone-repositories/action.yml b/clone-repositories/action.yml index 4e615d8..c438237 100644 --- a/clone-repositories/action.yml +++ b/clone-repositories/action.yml @@ -44,4 +44,44 @@ runs: GIT_CONFIG_KEY_1: credential.https://github.com.helper GIT_CONFIG_VALUE_1: >- !f() { if [ "$1" = get ]; then printf '%s\n' 'username=x-access-token' "password=$GH_TOKEN"; fi; }; f - run: bash "${{ github.action_path }}/clone.sh" + run: | + set -euo pipefail + + declare -A visited=() + queue=() + + enqueue_dependencies() { + local file=$1 repo + [[ -f $file ]] || return 0 + while IFS= read -r repo || [[ -n $repo ]]; do + repo=${repo%$'\r'} + repo=${repo#"${repo%%[![:space:]]*}"} + repo=${repo%"${repo##*[![:space:]]}"} + [[ -z $repo || $repo == \#* ]] && continue + if [[ ! $repo =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]]; then + echo "Invalid repository name in $file: $repo" >&2 + exit 1 + fi + queue+=("$repo") + done < "$file" + } + + if [[ ! -f $PROJECT_DEPS_PATH ]]; then + echo "Dependency file not found: $PROJECT_DEPS_PATH" >&2 + exit 1 + fi + enqueue_dependencies "$PROJECT_DEPS_PATH" + for ((index=0; index<${#queue[@]}; index++)); do + repo=${queue[index]} + [[ -v visited[$repo] ]] && continue + visited[$repo]=1 + if [[ ! -d $repo ]]; then + echo "Cloning repository: $repo" + url="https://github.com/dbeaver/$repo.git" + default_branch=$(git ls-remote --symref "$url" HEAD | awk '/^ref:/ {sub("refs/heads/", "", $2); print $2; exit}') + git clone --depth 1 -b "${HEAD_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ + git clone --depth 1 -b "${BASE_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ + git clone --depth 1 -b "$default_branch" "$url" + fi + enqueue_dependencies "$repo/project.deps" + done diff --git a/clone-repositories/clone.sh b/clone-repositories/clone.sh deleted file mode 100644 index 0566450..0000000 --- a/clone-repositories/clone.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -declare -A visited=() -queue=() - -enqueue_dependencies() { - local file=$1 repo - [[ -f $file ]] || return 0 - while IFS= read -r repo || [[ -n $repo ]]; do - repo=${repo%$'\r'} - repo=${repo#"${repo%%[![:space:]]*}"} - repo=${repo%"${repo##*[![:space:]]}"} - [[ -z $repo || $repo == \#* ]] && continue - if [[ ! $repo =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]]; then - echo "Invalid repository name in $file: $repo" >&2 - exit 1 - fi - queue+=("$repo") - done < "$file" -} - -if [[ ! -f $PROJECT_DEPS_PATH ]]; then - echo "Dependency file not found: $PROJECT_DEPS_PATH" >&2 - exit 1 -fi -enqueue_dependencies "$PROJECT_DEPS_PATH" -for ((index=0; index<${#queue[@]}; index++)); do - repo=${queue[index]} - [[ -v visited[$repo] ]] && continue - visited[$repo]=1 - if [[ ! -d $repo ]]; then - echo "Cloning repository: $repo" - url="https://github.com/dbeaver/$repo.git" - default_branch=$(git ls-remote --symref "$url" HEAD | awk '/^ref:/ {sub("refs/heads/", "", $2); print $2; exit}') - git clone --depth 1 -b "${HEAD_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ - git clone --depth 1 -b "${BASE_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ - git clone --depth 1 -b "$default_branch" "$url" - fi - enqueue_dependencies "$repo/project.deps" -done From 8cfa488113a4df800db70cd6f94f5affba8f1f02 Mon Sep 17 00:00:00 2001 From: serge-rider Date: Sat, 7 Nov 2026 20:01:20 +0100 Subject: [PATCH 4/7] dbeaver/pro#11574 Document recursive clone traversal --- clone-repositories/action.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clone-repositories/action.yml b/clone-repositories/action.yml index c438237..e8d10db 100644 --- a/clone-repositories/action.yml +++ b/clone-repositories/action.yml @@ -47,6 +47,8 @@ runs: run: | set -euo pipefail + # Walk the dependency graph breadth-first. The queue grows as each cloned + # repository's project.deps is read; visited prevents duplicates and cycles. declare -A visited=() queue=() @@ -79,9 +81,11 @@ runs: echo "Cloning repository: $repo" url="https://github.com/dbeaver/$repo.git" default_branch=$(git ls-remote --symref "$url" HEAD | awk '/^ref:/ {sub("refs/heads/", "", $2); print $2; exit}') + # Prefer the PR branch, then its base branch, then the remote default. git clone --depth 1 -b "${HEAD_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ git clone --depth 1 -b "${BASE_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ git clone --depth 1 -b "$default_branch" "$url" fi + # Also traverse dependencies of repositories already present in the workspace. enqueue_dependencies "$repo/project.deps" done From 1c1f7856bf73438332a2362aca961d79e6db0017 Mon Sep 17 00:00:00 2001 From: serge-rider Date: Sat, 7 Nov 2026 20:09:26 +0100 Subject: [PATCH 5/7] dbeaver/pro#11574 Clone dependency levels in parallel --- clone-repositories/action.yml | 59 +++++++++++++++++++++++++---------- 1 file changed, 43 insertions(+), 16 deletions(-) diff --git a/clone-repositories/action.yml b/clone-repositories/action.yml index e8d10db..531b253 100644 --- a/clone-repositories/action.yml +++ b/clone-repositories/action.yml @@ -47,10 +47,11 @@ runs: run: | set -euo pipefail - # Walk the dependency graph breadth-first. The queue grows as each cloned - # repository's project.deps is read; visited prevents duplicates and cycles. + # Walk the dependency graph breadth-first. Clone each level in parallel, + # then read its project.deps files to discover the next level. declare -A visited=() queue=() + max_parallel=8 enqueue_dependencies() { local file=$1 repo @@ -68,24 +69,50 @@ runs: done < "$file" } + clone_repository() { + local repo=$1 url default_branch + echo "Cloning repository: $repo" + url="https://github.com/dbeaver/$repo.git" + default_branch=$(git ls-remote --symref "$url" HEAD | awk '/^ref:/ {sub("refs/heads/", "", $2); print $2; exit}') + # Prefer the PR branch, then its base branch, then the remote default. + git clone --depth 1 -b "${HEAD_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ + git clone --depth 1 -b "${BASE_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ + git clone --depth 1 -b "$default_branch" "$url" + } + if [[ ! -f $PROJECT_DEPS_PATH ]]; then echo "Dependency file not found: $PROJECT_DEPS_PATH" >&2 exit 1 fi enqueue_dependencies "$PROJECT_DEPS_PATH" - for ((index=0; index<${#queue[@]}; index++)); do - repo=${queue[index]} - [[ -v visited[$repo] ]] && continue - visited[$repo]=1 - if [[ ! -d $repo ]]; then - echo "Cloning repository: $repo" - url="https://github.com/dbeaver/$repo.git" - default_branch=$(git ls-remote --symref "$url" HEAD | awk '/^ref:/ {sub("refs/heads/", "", $2); print $2; exit}') - # Prefer the PR branch, then its base branch, then the remote default. - git clone --depth 1 -b "${HEAD_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ - git clone --depth 1 -b "${BASE_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ - git clone --depth 1 -b "$default_branch" "$url" + level_start=0 + while ((level_start < ${#queue[@]})); do + level_end=${#queue[@]} + pending_pids=() + failed=0 + for ((index=level_start; index= max_parallel)); then + if ! wait "${pending_pids[0]}"; then failed=1; fi + pending_pids=("${pending_pids[@]:1}") + fi + fi + done + for pid in "${pending_pids[@]}"; do + if ! wait "$pid"; then failed=1; fi + done + if ((failed)); then + echo "One or more repository clones failed" >&2 + exit 1 fi - # Also traverse dependencies of repositories already present in the workspace. - enqueue_dependencies "$repo/project.deps" + # Include dependencies of existing checkouts, but never revisit a repo. + for ((index=level_start; index Date: Sat, 7 Nov 2026 20:13:29 +0100 Subject: [PATCH 6/7] dbeaver/pro#11574 Support macOS Bash in recursive clone action --- clone-repositories/action.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/clone-repositories/action.yml b/clone-repositories/action.yml index 531b253..dc5d7e8 100644 --- a/clone-repositories/action.yml +++ b/clone-repositories/action.yml @@ -45,11 +45,11 @@ runs: GIT_CONFIG_VALUE_1: >- !f() { if [ "$1" = get ]; then printf '%s\n' 'username=x-access-token' "password=$GH_TOKEN"; fi; }; f run: | - set -euo pipefail + set -eo pipefail # Walk the dependency graph breadth-first. Clone each level in parallel, # then read its project.deps files to discover the next level. - declare -A visited=() + visited=' ' queue=() max_parallel=8 @@ -92,8 +92,10 @@ runs: failed=0 for ((index=level_start; index Date: Sat, 7 Nov 2026 20:19:03 +0100 Subject: [PATCH 7/7] dbeaver/pro#11574 Scan each dependency only once --- clone-repositories/action.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/clone-repositories/action.yml b/clone-repositories/action.yml index dc5d7e8..12102ef 100644 --- a/clone-repositories/action.yml +++ b/clone-repositories/action.yml @@ -89,6 +89,7 @@ runs: while ((level_start < ${#queue[@]})); do level_end=${#queue[@]} pending_pids=() + to_scan=() failed=0 for ((index=level_start; index&2 exit 1 fi - # Include dependencies of existing checkouts, but never revisit a repo. - for ((index=level_start; index