diff --git a/clone-repositories/action.yml b/clone-repositories/action.yml index c3930e3..12102ef 100644 --- a/clone-repositories/action.yml +++ b/clone-repositories/action.yml @@ -15,7 +15,7 @@ inputs: [Learn more about creating and using encrypted secrets](https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets) default: ${{ github.token }} project_deps_path: - description: "The content of the project.deps file with repository names" + description: "Path to the project.deps file with repository names" required: true runs: @@ -35,6 +35,9 @@ runs: shell: bash env: GH_TOKEN: ${{ inputs.token }} + PROJECT_DEPS_PATH: ${{ inputs.project_deps_path }} + HEAD_BRANCH: ${{ env.GITHUB_HEAD_REF }} + BASE_BRANCH: ${{ env.GITHUB_BASE_REF }} GIT_CONFIG_COUNT: '2' GIT_CONFIG_KEY_0: credential.helper GIT_CONFIG_VALUE_0: '' @@ -42,25 +45,78 @@ runs: GIT_CONFIG_VALUE_1: >- !f() { if [ "$1" = get ]; then printf '%s\n' 'username=x-access-token' "password=$GH_TOKEN"; fi; }; f run: | - exit_code=0 - while IFS= read -r repo || [[ -n $repo ]]; do - echo "Cloning repository: $repo" - { - default_branch=$(git ls-remote --symref "https://github.com/dbeaver/$repo.git" HEAD | grep '^ref:' | awk '{print $2}' | sed 's|refs/heads/||') + set -eo pipefail - # Try to clone the head branch, then the base branch, and finally the default branch - git clone --depth 1 -b "${{ env.GITHUB_HEAD_REF }}" "https://github.com/dbeaver/$repo.git" 2>/dev/null || \ - git clone --depth 1 -b "${{ env.GITHUB_BASE_REF }}" "https://github.com/dbeaver/$repo.git" 2>/dev/null || \ - git clone --depth 1 -b "$default_branch" "https://github.com/dbeaver/$repo.git" + # Walk the dependency graph breadth-first. Clone each level in parallel, + # then read its project.deps files to discover the next level. + visited=' ' + queue=() + max_parallel=8 - # Capture the exit code of the last command - if [[ $? -ne 0 ]]; then - exit_code=1 # Set exit_code to 1 if any clone command fails + enqueue_dependencies() { + local file=$1 repo + [[ -f $file ]] || return 0 + while IFS= read -r repo || [[ -n $repo ]]; do + repo=${repo%$'\r'} + repo=${repo#"${repo%%[![:space:]]*}"} + repo=${repo%"${repo##*[![:space:]]}"} + [[ -z $repo || $repo == \#* ]] && continue + if [[ ! $repo =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]]; then + echo "Invalid repository name in $file: $repo" >&2 + exit 1 fi - } & - done < "${{ inputs.project_deps_path }}" + queue+=("$repo") + done < "$file" + } - wait + clone_repository() { + local repo=$1 url default_branch + echo "Cloning repository: $repo" + url="https://github.com/dbeaver/$repo.git" + default_branch=$(git ls-remote --symref "$url" HEAD | awk '/^ref:/ {sub("refs/heads/", "", $2); print $2; exit}') + # Prefer the PR branch, then its base branch, then the remote default. + git clone --depth 1 -b "${HEAD_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ + git clone --depth 1 -b "${BASE_BRANCH#refs/heads/}" "$url" 2>/dev/null || \ + git clone --depth 1 -b "$default_branch" "$url" + } - # Exit with the appropriate exit code - exit $exit_code + if [[ ! -f $PROJECT_DEPS_PATH ]]; then + echo "Dependency file not found: $PROJECT_DEPS_PATH" >&2 + exit 1 + fi + enqueue_dependencies "$PROJECT_DEPS_PATH" + level_start=0 + while ((level_start < ${#queue[@]})); do + level_end=${#queue[@]} + pending_pids=() + to_scan=() + failed=0 + for ((index=level_start; index= max_parallel)); then + if ! wait "${pending_pids[0]}"; then failed=1; fi + pending_pids=("${pending_pids[@]:1}") + fi + fi + done + for pid in "${pending_pids[@]}"; do + if ! wait "$pid"; then failed=1; fi + done + if ((failed)); then + echo "One or more repository clones failed" >&2 + exit 1 + fi + # Scan newly visited repos, including checkouts already in the workspace. + for repo in "${to_scan[@]}"; do + enqueue_dependencies "$repo/project.deps" + done + level_start=$level_end + done