dbeaver/pro#11574 Resolve project.deps recursively

clone-repositories/action.yml

@@ -15,7 +15,7 @@ inputs:
       [Learn more about creating and using encrypted secrets](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets)
     default: ${{ github.token }}
   project_deps_path:
-    description: "The content of the project.deps file with repository names"
+    description: "Path to the project.deps file with repository names"
     required: true
 
 runs:
@@ -35,32 +35,88 @@ runs:
       shell: bash
       env:
         GH_TOKEN: ${{ inputs.token }}
+        PROJECT_DEPS_PATH: ${{ inputs.project_deps_path }}
+        HEAD_BRANCH: ${{ env.GITHUB_HEAD_REF }}
+        BASE_BRANCH: ${{ env.GITHUB_BASE_REF }}
         GIT_CONFIG_COUNT: '2'
         GIT_CONFIG_KEY_0: credential.helper
         GIT_CONFIG_VALUE_0: ''
         GIT_CONFIG_KEY_1: credential.https://github.com.helper
         GIT_CONFIG_VALUE_1: >-
           !f() { if [ "$1" = get ]; then printf '%s\n' 'username=x-access-token' "password=$GH_TOKEN"; fi; }; f
       run: |
-        exit_code=0
-        while IFS= read -r repo || [[ -n $repo ]]; do
-          echo "Cloning repository: $repo"
-          {
-            default_branch=$(git ls-remote --symref "https://github.com/dbeaver/$repo.git" HEAD | grep '^ref:' | awk '{print $2}' | sed 's|refs/heads/||')
+        set -eo pipefail
 
-            # Try to clone the head branch, then the base branch, and finally the default branch
-            git clone --depth 1 -b "${{ env.GITHUB_HEAD_REF }}" "https://github.com/dbeaver/$repo.git" 2>/dev/null || \
-            git clone --depth 1 -b "${{ env.GITHUB_BASE_REF }}" "https://github.com/dbeaver/$repo.git" 2>/dev/null || \
-            git clone --depth 1 -b "$default_branch" "https://github.com/dbeaver/$repo.git"
+        # Walk the dependency graph breadth-first. Clone each level in parallel,
+        # then read its project.deps files to discover the next level.
+        visited=' '
+        queue=()
+        max_parallel=8
 
-            # Capture the exit code of the last command
-            if [[ $? -ne 0 ]]; then
-              exit_code=1  # Set exit_code to 1 if any clone command fails
+        enqueue_dependencies() {
+          local file=$1 repo
+          [[ -f $file ]] || return 0
+          while IFS= read -r repo || [[ -n $repo ]]; do
+            repo=${repo%$'\r'}
+            repo=${repo#"${repo%%[![:space:]]*}"}
+            repo=${repo%"${repo##*[![:space:]]}"}
+            [[ -z $repo || $repo == \#* ]] && continue
+            if [[ ! $repo =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]]; then
+              echo "Invalid repository name in $file: $repo" >&2
+              exit 1
             fi
-          } &
-        done < "${{ inputs.project_deps_path }}"
+            queue+=("$repo")
+          done < "$file"
+        }
 
-        wait
+        clone_repository() {
+          local repo=$1 url default_branch
+          echo "Cloning repository: $repo"
+          url="https://github.com/dbeaver/$repo.git"
+          default_branch=$(git ls-remote --symref "$url" HEAD | awk '/^ref:/ {sub("refs/heads/", "", $2); print $2; exit}')
+          # Prefer the PR branch, then its base branch, then the remote default.
+          git clone --depth 1 -b "${HEAD_BRANCH#refs/heads/}" "$url" 2>/dev/null || \
+            git clone --depth 1 -b "${BASE_BRANCH#refs/heads/}" "$url" 2>/dev/null || \
+            git clone --depth 1 -b "$default_branch" "$url"
+        }
 
-        # Exit with the appropriate exit code
-        exit $exit_code
+        if [[ ! -f $PROJECT_DEPS_PATH ]]; then
+          echo "Dependency file not found: $PROJECT_DEPS_PATH" >&2
+          exit 1
+        fi
+        enqueue_dependencies "$PROJECT_DEPS_PATH"
+        level_start=0
+        while ((level_start < ${#queue[@]})); do
+          level_end=${#queue[@]}
+          pending_pids=()
+          to_scan=()
+          failed=0
+          for ((index=level_start; index<level_end; index++)); do
+            repo=${queue[index]}
+            case "$visited" in
+              *" $repo "*) continue ;;
+            esac
+            visited="$visited$repo "
+            to_scan+=("$repo")
+            if [[ ! -d $repo ]]; then
+              clone_repository "$repo" &
+              pending_pids+=("$!")
+              if ((${#pending_pids[@]} >= max_parallel)); then
+                if ! wait "${pending_pids[0]}"; then failed=1; fi
+                pending_pids=("${pending_pids[@]:1}")
+              fi
+            fi
+          done
+          for pid in "${pending_pids[@]}"; do
+            if ! wait "$pid"; then failed=1; fi
+          done
+          if ((failed)); then
+            echo "One or more repository clones failed" >&2
+            exit 1
+          fi
+          # Scan newly visited repos, including checkouts already in the workspace.
+          for repo in "${to_scan[@]}"; do
+            enqueue_dependencies "$repo/project.deps"
+          done
+          level_start=$level_end
+        done