diff --git a/commands/curation/curationaudit.go b/commands/curation/curationaudit.go index 37f040eb5..ab0c9ebde 100644 --- a/commands/curation/curationaudit.go +++ b/commands/curation/curationaudit.go @@ -18,6 +18,8 @@ import ( "sync/atomic" "github.com/google/uuid" + "github.com/gookit/color" + "github.com/jedib0t/go-pretty/v6/text" "github.com/jfrog/gofrog/datastructures" "github.com/jfrog/gofrog/parallel" rtUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" @@ -90,6 +92,12 @@ const ( // "All versions blocked - {policy:openssf-jfca,condition:openssf}". allVersionsBlockedMsgPattern = `(?i)all versions blocked\s*-\s*\{\s*policy:([^,}]+),condition:([^}]+)\}` + // allVersionsOfPattern matches Xray's whole-package-block message with its lines joined by spaces: + // "All versions of 'X' (repo) are blocked from download by JFrog Curation. Blocked by one or more of + // these policies: 1. P (C) 2. ... Note: ... Action: ... Details: ...". Any other text after the + // first sentence matches with empty groups. + allVersionsOfPattern = `(?i)All versions of '[^']+' \([^)]*\) are blocked from download by JFrog Curation\.(?:(?: Blocked by one or more of these policies: (.*?))?(?: Note: ([^.]*\.))?(?: Action: (.*?))?(?: Details: (\S+))?$|.*)` + errorTemplateHeadRequest = "failed sending HEAD request to %s for package '%s:%s'. Status-code: %v. Cause: %v" errorTemplateUnsupportedTech = "It looks like this project uses '%s' to download its dependencies. " + @@ -130,7 +138,16 @@ const ( var CurationOutputFormats = []string{string(outFormat.Table), string(outFormat.Json)} var osGetwd = os.Getwd var npmGetConfigValue = npmtech.GetNpmConfigValue -var allVersionsBlockedMsgRegex = regexp.MustCompile(allVersionsBlockedMsgPattern) +var ( + allVersionsBlockedMsgRegex = regexp.MustCompile(allVersionsBlockedMsgPattern) + allVersionsOfMsgRegex = regexp.MustCompile(allVersionsOfPattern) + // catalogLinkRegex matches the Catalog link of a single-version block message. + catalogLinkRegex = regexp.MustCompile(`For details and alternatives, visit: (\S+)`) + // allVersionsOfPolicyNumberRegex matches the "1. " / " 2. " numbering between the items of the policy list. + allVersionsOfPolicyNumberRegex = regexp.MustCompile(`(?:^| )\d+\. `) + // allVersionsOfPolicyItemRegex matches one policy list item: P (C), where C may itself contain parentheses. + allVersionsOfPolicyItemRegex = regexp.MustCompile(`^(.+?) \((.*)\)$`) +) var supportedTech = map[techutils.Technology]func(ca *CurationAuditCommand) (bool, error){ techutils.Npm: func(ca *CurationAuditCommand) (bool, error) { return true, nil }, @@ -248,6 +265,7 @@ type PackageStatus struct { ParentName string `json:"direct_dependency_package_name"` ParentVersion string `json:"direct_dependency_package_version"` BlockedPackageUrl string `json:"blocked_package_url,omitempty"` + CatalogURL string `json:"catalog_url,omitempty"` PackageName string `json:"blocked_package_name"` PackageVersion string `json:"blocked_package_version"` BlockingReason string `json:"blocking_reason"` @@ -273,8 +291,9 @@ type PackageStatusTable struct { PkgType string `col-name:"Package\nType" auto-merge:"true"` Policy string `col-name:"Violated\nPolicy\nName"` Condition string `col-name:"Violated Condition\nName"` - Explanation string `col-name:"Explanation"` - Recommendation string `col-name:"Recommendation"` + Explanation string `col-name:"Explanation" auto-merge:"true"` + Recommendation string `col-name:"Recommendation" auto-merge:"true"` + Link string `col-name:"Blocked\nPackage\nLink" auto-merge:"true" omitempty:"true"` } type treeAnalyzer struct { @@ -1564,6 +1583,33 @@ func (ca *CurationAuditCommand) requestWaiver(blockedPackages []*PackageStatus) return coreutils.PrintTable(pkgStatusTable, "Waiver request submitted!", "Requested 0 waivers", true) } +// catalogLinkStyle is the style of a package name that is a terminal hyperlink to its Catalog page. +var catalogLinkStyle = color.New(color.FgBlue, color.OpUnderscore) + +// catalogLinks lists one JFrog Catalog page per blocked package name, the first one found. +func catalogLinks(packagesStatus []*PackageStatus) string { + var names, urls []string + seen := map[string]bool{} + width := 0 + for _, ps := range packagesStatus { + if ps.CatalogURL == "" || seen[ps.PackageName] { + continue + } + seen[ps.PackageName] = true + names, urls = append(names, ps.PackageName), append(urls, ps.CatalogURL) + width = max(width, len(ps.PackageName)) + } + if len(urls) == 0 { + return "" + } + var sb strings.Builder + sb.WriteString("\nFor more details, visit:") + for i := range urls { + fmt.Fprintf(&sb, "\n %-*s %s", width, names[i], urls[i]) + } + return sb.String() +} + func printResult(format outFormat.OutputFormat, projectPath string, packagesStatus []*PackageStatus) error { if format == "" { format = outFormat.Table @@ -1578,17 +1624,23 @@ func printResult(format outFormat.OutputFormat, projectPath string, packagesStat } } case outFormat.Table: - pkgStatusTable := convertToPackageStatusTable(packagesStatus) + withLinks := output.TerminalSupportsHyperlinks() + pkgStatusTable := convertToPackageStatusTable(packagesStatus, withLinks) err := coreutils.PrintTable(pkgStatusTable, "Curation", "Found 0 blocked packages", true) if err != nil { return err } + if links := catalogLinks(packagesStatus); !withLinks && links != "" { + log.Output(links) + } } log.Output("\n") return nil } -func convertToPackageStatusTable(packagesStatus []*PackageStatus) []PackageStatusTable { +// convertToPackageStatusTable builds the table rows. With withLinks, a package with a Catalog link gets a +// styled "Open ↗" terminal hyperlink to it in the Link column; the column is omitted when no row has one. +func convertToPackageStatusTable(packagesStatus []*PackageStatus, withLinks bool) []PackageStatusTable { var pkgStatusTable []PackageStatusTable for index, pkgStatus := range packagesStatus { // We use auto-merge supported by the 'go-pretty' library. It doesn't have an option to merge lines by a group of unique fields. @@ -1597,6 +1649,10 @@ func convertToPackageStatusTable(packagesStatus []*PackageStatus) []PackageStatu if index%2 == 0 { uniqLineSep = " " } + var link string + if withLinks && pkgStatus.CatalogURL != "" { + link = catalogLinkStyle.Render(text.Hyperlink(pkgStatus.CatalogURL, "Open ↗")) + uniqLineSep + } pkgTable := PackageStatusTable{ ID: fmt.Sprintf("%d%s", index+1, uniqLineSep), ParentName: pkgStatus.ParentName + uniqLineSep, @@ -1604,6 +1660,7 @@ func convertToPackageStatusTable(packagesStatus []*PackageStatus) []PackageStatu PackageName: pkgStatus.PackageName + uniqLineSep, PackageVersion: pkgStatus.PackageVersion + uniqLineSep, PkgType: pkgStatus.PkgType + uniqLineSep, + Link: link, } if len(pkgStatus.Policy) == 0 { pkgStatusTable = append(pkgStatusTable, pkgTable) @@ -1611,8 +1668,8 @@ func convertToPackageStatusTable(packagesStatus []*PackageStatus) []PackageStatu } for _, policyCond := range pkgStatus.Policy { pkgTable.Policy = policyCond.Policy - pkgTable.Explanation = policyCond.Explanation - pkgTable.Recommendation = policyCond.Recommendation + pkgTable.Explanation = policyCond.Explanation + uniqLineSep + pkgTable.Recommendation = policyCond.Recommendation + uniqLineSep pkgTable.Condition = policyCond.Condition pkgStatusTable = append(pkgStatusTable, pkgTable) } @@ -2650,7 +2707,7 @@ func (ca *CurationAuditCommand) runNpmLogFallback(logsDir string, tech techutils const allVersionsBlockedText = "All versions blocked" // wholePackageBlockedStatus synthesizes a *PackageStatus straight from the log's notice+403 -// evidence — no HEAD-check needed. Recommendation is filled in later by +// evidence — no HEAD-check needed. Explanation and Recommendation are finished later by // applyTransitiveAwareRecommendation, once the graph walk knows the parent. func wholePackageBlockedStatus(entry npmLogEntry, info npmBlockedInfo, tech techutils.Technology) *PackageStatus { return &PackageStatus{ @@ -2659,19 +2716,14 @@ func wholePackageBlockedStatus(entry npmLogEntry, info npmBlockedInfo, tech tech Action: blocked, BlockingReason: BlockingReasonPolicy, PkgType: string(tech), - Policy: []Policy{{ - Policy: info.Policy, - Condition: info.Condition, - Explanation: allVersionsBlockedText, - // Recommendation is filled in by applyTransitiveAwareRecommendation. - }}, + Policy: slices.Clone(info.Policies), + CatalogURL: info.CatalogURL, } } -// applyTransitiveAwareRecommendation normalizes every ps.Policy entry to generic, -// direct/transitive-aware text for a whole-package block (matches npm). Only safe when -// all entries are already whole-package (e.g. Step 0). For a per-version probe result, -// use normalizeMatchingWholePackagePolicies instead. +// applyTransitiveAwareRecommendation applies normalizeWholePackagePolicy to every ps.Policy entry of a +// whole-package block. Only safe when all entries are already whole-package (e.g. Step 0). For a +// per-version probe result, use normalizeMatchingWholePackagePolicies instead. func applyTransitiveAwareRecommendation(ps *PackageStatus, isDirectDependency bool) { policies := make([]Policy, len(ps.Policy)) copy(policies, ps.Policy) @@ -2681,38 +2733,46 @@ func applyTransitiveAwareRecommendation(ps *PackageStatus, isDirectDependency bo ps.Policy = policies } -// normalizeMatchingWholePackagePolicies normalizes only entries in ps.Policy matching -// aggregatePolicies by Policy+Condition — the same block Step 0 confirmed. A genuinely -// different per-version policy is left untouched. -func normalizeMatchingWholePackagePolicies(ps *PackageStatus, aggregatePolicies []Policy, isDirectDependency bool) { +// normalizeMatchingWholePackagePolicies copies the Explanation/Recommendation of the already +// normalized aggregatePolicies onto the entries in ps.Policy matching them by Policy+Condition — +// the same block Step 0 confirmed. A genuinely different per-version policy is left untouched. +func normalizeMatchingWholePackagePolicies(ps *PackageStatus, aggregatePolicies []Policy) { policies := make([]Policy, len(ps.Policy)) copy(policies, ps.Policy) for i := range policies { - if !matchesAnyPolicy(policies[i], aggregatePolicies) { - continue + if c, ok := matchingPolicy(policies[i], aggregatePolicies); ok { + policies[i].Explanation, policies[i].Recommendation = c.Explanation, c.Recommendation } - normalizeWholePackagePolicy(&policies[i], ps, isDirectDependency) } ps.Policy = policies } -// matchesAnyPolicy reports whether p shares the same Policy+Condition as any of candidates. -func matchesAnyPolicy(p Policy, candidates []Policy) bool { +// matchingPolicy returns the first of candidates sharing the same Policy+Condition as p. +func matchingPolicy(p Policy, candidates []Policy) (Policy, bool) { for _, c := range candidates { if p.Policy == c.Policy && p.Condition == c.Condition { - return true + return c, true } } - return false + return Policy{}, false } -// normalizeWholePackagePolicy rewrites a single Policy entry's Explanation/Recommendation -// to the generic, direct/transitive-aware text for a whole-package block. +// normalizeWholePackagePolicy sets the Explanation of a whole-package Policy entry to allVersionsBlockedText. +// A Recommendation already on p holds the Note and Action lines of Xray's message and is kept; for a +// transitive dependency it follows a sentence naming the parent. Without one, a generic +// direct/transitive-aware text is used. func normalizeWholePackagePolicy(p *Policy, ps *PackageStatus, isDirectDependency bool) { p.Explanation = allVersionsBlockedText - if isDirectDependency { + action := p.Recommendation + switch { + case isDirectDependency && action != "": + // Xray's Note and Action lines are used as is. + case isDirectDependency: p.Recommendation = "Remove this package from your project and replace with an alternate package" - } else { + case action != "": + p.Recommendation = fmt.Sprintf("%s is a transitive dependency of %s and cannot be removed directly. %s", + ps.PackageName, ps.ParentName, action) + default: p.Recommendation = fmt.Sprintf( "%s is a transitive dependency of %s and cannot be removed directly. Apply a waiver if acceptable, or replace %s with an alternative that doesn't depend on %s.", ps.PackageName, ps.ParentName, ps.ParentName, ps.PackageName) @@ -3003,7 +3063,10 @@ func (nc *treeAnalyzer) fetchCvsBlockedStatus(pins []python.PinnedRequirement) [ // Normalize only the entries that match the aggregate's policy+condition — a // second, genuinely different real policy in the same probe must be preserved. if wholePackageBlocked && wholePackageFallback != nil && len(wholePackageFallback.Policy) > 0 { - normalizeMatchingWholePackagePolicies(pkStatus, wholePackageFallback.Policy, depRelation == directRelation) + normalizeMatchingWholePackagePolicies(pkStatus, wholePackageFallback.Policy) + } + if wholePackageBlocked && wholePackageFallback != nil && wholePackageFallback.CatalogURL != "" { + pkStatus.CatalogURL = wholePackageFallback.CatalogURL } statuses = append(statuses, pkStatus) } @@ -3098,6 +3161,10 @@ func (nc *treeAnalyzer) getBlockedPackageDetails(packageUrl string, name string, if strings.Contains(strings.ToLower(respError.Errors[0].Message), BlockMessageKey) { blockingReason := BlockingReasonPolicy policies := nc.extractPoliciesFromMsg(respError) + var catalogURL string + if m := catalogLinkRegex.FindStringSubmatch(respError.Errors[0].Message); m != nil { + catalogURL = m[1] + } if len(policies) == 0 { // a server predating per-policy pending behavior sends no {policy,...} groups, only these substrings lowerMsg := strings.ToLower(respError.Errors[0].Message) @@ -3121,6 +3188,25 @@ func (nc *treeAnalyzer) getBlockedPackageDetails(packageUrl string, name string, BlockingReason: blockingReason, WaiverAllowed: strings.Contains(respError.Errors[0].Message, "[waivers allowed]"), PkgType: string(nc.tech), + CatalogURL: catalogURL, + }, nil + } + if m := allVersionsOfMsgRegex.FindStringSubmatch(strings.ReplaceAll(respError.Errors[0].Message, "\n", " ")); m != nil { + policies := allVersionsOfPolicies(m) + blockingReason := BlockingReasonPolicy + if m[1] == "" && m[3] == "" { + log.Debug(fmt.Sprintf("curation: could not parse the policies of the all-versions block message for %s: %s", name, respError.Errors[0].Message)) + policies, blockingReason = nil, BlockingReasonUnknown + } + return &PackageStatus{ + PackageName: name, + PackageVersion: version, + BlockedPackageUrl: packageUrl, + Action: blocked, + BlockingReason: blockingReason, + Policy: policies, + PkgType: string(nc.tech), + CatalogURL: m[4], }, nil } if m := allVersionsBlockedMsgRegex.FindStringSubmatch(respError.Errors[0].Message); m != nil { diff --git a/commands/curation/curationaudit_test.go b/commands/curation/curationaudit_test.go index cac0d2e31..6a3b1acd7 100644 --- a/commands/curation/curationaudit_test.go +++ b/commands/curation/curationaudit_test.go @@ -27,6 +27,7 @@ import ( "github.com/jfrog/jfrog-cli-security/utils/formats" "github.com/google/uuid" + "github.com/jedib0t/go-pretty/v6/text" biutils "github.com/jfrog/build-info-go/utils" "github.com/jfrog/gofrog/datastructures" rtUtils "github.com/jfrog/jfrog-cli-core/v2/artifactory/utils" @@ -2604,6 +2605,7 @@ func TestGetBlockedPackageDetails_PolicyGroupsPreemptPendingSubstrings(t *testin wantReason string wantPolicies []Policy wantWaiver bool + wantCatalog string }{ { name: "old server: package not found in catalog", @@ -2627,7 +2629,8 @@ func TestGetBlockedPackageDetails_PolicyGroupsPreemptPendingSubstrings(t *testin Explanation: "The JFrog Catalog has no data for this package yet so the policy cannot be evaluated against it. This policy is set to block while a package is pending Catalog analysis.", Recommendation: "Request a waiver for this package or wait for the JFrog Catalog to analyze it.", }}, - wantWaiver: true, + wantWaiver: true, + wantCatalog: "http://localhost:8083/ui/catalog/packages/details/npm/@milkio%2Fstargate-worker?ecosystem=generic&showVersions=true", }, { name: "new server: policy group wins over on-demand", @@ -2639,12 +2642,19 @@ func TestGetBlockedPackageDetails_PolicyGroupsPreemptPendingSubstrings(t *testin Explanation: "This package is still being scanned so the policy cannot be evaluated against it. This policy is set to block until the scan finishes.", Recommendation: "Retry the download once the scan completes. No waiver is needed.", }}, + wantCatalog: "http://localhost:8083/ui/catalog/onDemand/packages/details/npm/@milkio%2Fstargate-worker/1.3.65?ecosystem=generic", }, { - name: "curation block with neither policy groups nor a pending substring", - message: `package @milkio/stargate-worker:1.3.65 download was blocked by jfrog packages curation service due to the following policies violated . For details and alternatives, visit: http://localhost:8083/ui/catalog/packages/details/npm/@milkio%2Fstargate-worker?ecosystem=generic&showVersions=true [waivers allowed]`, + name: "curation block with neither policy groups nor a pending substring", + message: `package @milkio/stargate-worker:1.3.65 download was blocked by jfrog packages curation service due to the following policies violated . For details and alternatives, visit: http://localhost:8083/ui/catalog/packages/details/npm/@milkio%2Fstargate-worker?ecosystem=generic&showVersions=true [waivers allowed]`, + wantReason: BlockingReasonUnknown, + wantWaiver: true, + wantCatalog: "http://localhost:8083/ui/catalog/packages/details/npm/@milkio%2Fstargate-worker?ecosystem=generic&showVersions=true", + }, + { + name: "all-versions message with a reworded tail", + message: "All versions of '@milkio/stargate-worker' (npm-remote) are blocked from download by JFrog Curation.\nBlocked by these policies:\n1. P (C)", wantReason: BlockingReasonUnknown, - wantWaiver: true, }, } @@ -2679,6 +2689,7 @@ func TestGetBlockedPackageDetails_PolicyGroupsPreemptPendingSubstrings(t *testin assert.Equal(t, tt.wantReason, got.BlockingReason) assert.Equal(t, tt.wantPolicies, got.Policy) assert.Equal(t, tt.wantWaiver, got.WaiverAllowed) + assert.Equal(t, tt.wantCatalog, got.CatalogURL) }) } } @@ -3693,6 +3704,49 @@ func TestRunNpmLogFallback_MissingPackageJsonStillRecoversReport(t *testing.T) { assert.True(t, directDepNamesContains(report, "blockedpkg"), "blockedpkg is a direct dependency of the real (log-recovered) root, so its recommendation must say so, not treat it as transitive") } +func TestRunNpmLogFallback_AllVersionsOfNotice(t *testing.T) { + serverMock, serverDetails, _ := coreCommonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { + t.Errorf("unexpected HEAD-check for a whole-package-blocked entry: %s", r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + }) + defer serverMock.Close() + ca := &CurationAuditCommand{ + PackageManagerConfig: (&project.RepositoryConfig{}).SetTargetRepo("my-npm-remote").SetServerDetails(serverDetails), + extractPoliciesRegex: regexp.MustCompile(extractPoliciesRegexTemplate), + } + const action = "Waivers are allowed for some versions only. Please use a different package, or request a waiver for a version whose policies allow it." + logsDir := filepath.Join(t.TempDir(), "_logs") + require.NoError(t, os.MkdirAll(logsDir, 0755)) + logContent := "0 verbose cli node npm\n" + + "43 notice All versions of 'blockedpkg' (my-npm-remote) are blocked from download by JFrog Curation. " + + "Blocked by one or more of these policies: 1. Block Immature Packages (immature-7) 2. Block Malicious Packages (Malicious package) " + + "Note: Some versions are pending JFrog Catalog analysis. Action: " + action + " Details: https://example.jfrog.io/ui/catalog/blockedpkg\n" + + "44 http fetch GET 403 https://z0test.jfrogdev.org/artifactory/api/npm/my-npm-remote/blockedpkg 100ms (cache skip)\n" + + "45 silly placeDep ROOT blockedpkg@ OK for: realproj@9.9.9 want: ^1.0.0\n" + require.NoError(t, os.WriteFile(filepath.Join(logsDir, "2026-01-01T00_00_00_000Z-debug-0.log"), []byte(logContent), 0644)) + projectDir := t.TempDir() + orig := osGetwd + osGetwd = func() (string, error) { return projectDir, nil } + t.Cleanup(func() { osGetwd = orig }) + results := map[string]*CurationReport{} + + err := ca.runNpmLogFallback(logsDir, techutils.Npm, results, errors.New("error while running 'npm install': exit status 1"), false, 0) + + require.NoError(t, err) + require.Len(t, results, 1) + for _, report := range results { + require.Len(t, report.packagesStatus, 1) + got := report.packagesStatus[0] + assert.Equal(t, "blockedpkg", got.PackageName) + assert.Equal(t, allVersionsBlockedText, got.PackageVersion) + assert.Equal(t, "https://example.jfrog.io/ui/catalog/blockedpkg", got.CatalogURL) + assert.Equal(t, []Policy{ + {Policy: "Block Immature Packages", Condition: "immature-7", Explanation: allVersionsBlockedText, Recommendation: "Some versions are pending JFrog Catalog analysis. " + action}, + {Policy: "Block Malicious Packages", Condition: "Malicious package", Explanation: allVersionsBlockedText, Recommendation: "Some versions are pending JFrog Catalog analysis. " + action}, + }, got.Policy) + } +} + // directDepNamesContains checks pkgName's recommendation uses the direct, not transitive, wording. func directDepNamesContains(report *CurationReport, pkgName string) bool { for _, ps := range report.packagesStatus { @@ -5944,6 +5998,55 @@ func TestFetchCvsBlockedStatusRangedWholePackageBlocked(t *testing.T) { assert.Equal(t, expectedRecTemplate, s.Policy[0].Recommendation, "direct dependency must get the 'remove and replace' recommendation") } +func TestFetchCvsBlockedStatusRangedWholePackageBlockedAllVersionsOfMessage(t *testing.T) { + const action = "Please use a different package or request a waiver to use this one." + blockResponse := `{"errors":[{"status":403,"message":"All versions of 'urllib3' (test-pip-repo) are blocked from download by JFrog Curation.\n` + + `Blocked by one or more of these policies:\n1. p1 (c1 (fix version available))\n2. p2 (package pending JFrog Catalog analysis)\n` + + `Note: Some versions are pending JFrog Catalog analysis.\nAction: ` + action + `\nDetails: https://example.jfrog.io/ui/catalog"}]}` + serverMock, serverDetails, _ := coreCommonTests.CreateRtRestsMockServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(blockResponse)) + }) + defer serverMock.Close() + rtManager, err := rtUtils.CreateServiceManager(serverDetails, 0, 0, false) + require.NoError(t, err) + rtAuth := rtManager.GetConfig().GetServiceDetails() + tests := []struct { + name string + parent string + wantRecommendation string + }{ + {name: "direct", parent: "urllib3", wantRecommendation: "Some versions are pending JFrog Catalog analysis. " + action}, + {name: "transitive", parent: "requests", + wantRecommendation: "urllib3 is a transitive dependency of requests and cannot be removed directly. Some versions are pending JFrog Catalog analysis. " + action}, + } + for _, tech := range []techutils.Technology{techutils.Pip, techutils.Poetry, techutils.Pipenv, techutils.Uv} { + for _, tt := range tests { + t.Run(tech.String()+"/"+tt.name, func(t *testing.T) { + analyzer := treeAnalyzer{ + rtManager: rtManager, + extractPoliciesRegex: regexp.MustCompile(extractPoliciesRegexTemplate), + rtAuth: rtAuth, + httpClientDetails: rtAuth.CreateHttpClientDetails(), + url: rtAuth.GetUrl(), + repo: "test-pip-repo", + tech: tech, + } + statuses := analyzer.fetchCvsBlockedStatus([]python.PinnedRequirement{{Name: "urllib3", VersionRange: ">=2.0.0", ParentName: tt.parent}}) + + require.Len(t, statuses, 1) + assert.Equal(t, allVersionsBlockedText, statuses[0].PackageVersion) + assert.Equal(t, BlockingReasonPolicy, statuses[0].BlockingReason) + assert.Equal(t, []Policy{ + {Policy: "p1", Condition: "c1 (fix version available)", Explanation: allVersionsBlockedText, Recommendation: tt.wantRecommendation}, + {Policy: "p2", Condition: "package pending JFrog Catalog analysis", Explanation: allVersionsBlockedText, Recommendation: tt.wantRecommendation}, + }, statuses[0].Policy) + assert.Equal(t, "https://example.jfrog.io/ui/catalog", statuses[0].CatalogURL) + }) + } + } +} + // TestFetchCvsBlockedStatusPinnedWholePackageBlockedDifferentPolicyThanAggregate covers a // pinned version blocked by a DIFFERENT policy than Step 0's aggregate check. The // per-version detail must be preserved, not overwritten with the unrelated generic text. @@ -6037,14 +6140,15 @@ func TestFetchCvsBlockedStatusPinnedWholePackageBlockedSamePolicyAsAggregate(t * whlRelativePath = "packages/ca/25/urllib3-2.0.0-py3-none-any.whl" ) // Step 0's aggregate probe: confirmed whole-package block under policy openssf-jfca-test/openssf. - wholePackageBlockResponse := fmt.Sprintf( - `{"errors":[{"status":403,"message":"All versions blocked - {policy:%s,condition:%s}"}]}`, - policyName, condName, + wholePackageBlockMsg := fmt.Sprintf( + "All versions of '%s' (%s) are blocked from download by JFrog Curation.\nBlocked by one or more of these policies:\n1. %s (%s)\nDetails: https://x/pkg", + blockedPkg, repo, policyName, condName, ) + wholePackageBlockResponse := fmt.Sprintf(`{"errors":[{"status":403,"message":%q}]}`, wholePackageBlockMsg) // Steps 1-3's per-version probe: the SAME policy+condition, but with the raw, // inconsistently-worded backend message (as opposed to Step 0's generic phrasing). blockMsg := fmt.Sprintf( - "Package %s:%s download was blocked by JFrog Packages Curation service due to the following policies violated {%s, %s, %s, %s}.", + "Package %s:%s download was blocked by JFrog Packages Curation service due to the following policies violated {%s, %s, %s, %s}. For details and alternatives, visit: https://x/version", blockedPkg, blockedVer, policyName, condName, rawExpl, rawRec, ) versionBlockResponse := fmt.Sprintf(`{"errors":[{"status":403,"message":%q}]}`, blockMsg) @@ -6099,6 +6203,7 @@ func TestFetchCvsBlockedStatusPinnedWholePackageBlockedSamePolicyAsAggregate(t * "same policy as aggregate must be normalized to the generic text, not leak the raw inconsistent backend message") assert.Equal(t, "Remove this package from your project and replace with an alternate package", s.Policy[0].Recommendation, "direct pinned dependency, same policy as aggregate, must get the generic npm-parity recommendation") + assert.Equal(t, "https://x/pkg", s.CatalogURL) } // TestFetchCvsBlockedStatusPinnedWholePackageBlockedMultiPolicyPartialMatch verifies that @@ -7721,3 +7826,83 @@ func TestCredentialFallbackServerDetailsHostCheck(t *testing.T) { }) } } + +func TestCatalogLinks(t *testing.T) { + tests := []struct { + name string + statuses []*PackageStatus + want string + }{ + {name: "no links", statuses: []*PackageStatus{{PackageName: "axios"}}}, + { + name: "versions of one package", + statuses: []*PackageStatus{ + {PackageName: "vm2", PackageVersion: "3.9.18", CatalogURL: "https://v18"}, + {PackageName: "vm2", PackageVersion: "3.9.19", CatalogURL: "https://v19"}, + {PackageName: "axios", PackageVersion: allVersionsBlockedText, CatalogURL: "https://a"}, + }, + want: "\nFor more details, visit:\n vm2 https://v18\n axios https://a", + }, + { + name: "one line per link, names aligned", + statuses: []*PackageStatus{ + {PackageName: "axios", CatalogURL: "https://a"}, + {PackageName: "axios", CatalogURL: "https://a"}, + {PackageName: "left-pad"}, + {PackageName: "urllib3", CatalogURL: "https://u"}, + }, + want: "\nFor more details, visit:\n axios https://a\n urllib3 https://u", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, catalogLinks(tt.statuses)) + }) + } +} + +func TestPackageStatusTableMergesSharedText(t *testing.T) { + policies := []Policy{{Policy: "p1", Explanation: "expl", Recommendation: "rec"}, {Policy: "p2", Explanation: "expl", Recommendation: "rec"}} + statuses := []*PackageStatus{{PackageName: "a", Policy: policies}, {PackageName: "b", Policy: policies}} + tw, err := coreutils.PrepareTable(convertToPackageStatusTable(statuses, false), "", false) + require.NoError(t, err) + got := tw.Render() + if n := strings.Count(got, "rec"); n != 2 { + t.Errorf("PrepareTable(two packages with two policies each) shows the recommendation %d times, want 2:\n%s", n, got) + } +} + +func TestConvertToPackageStatusTableLinkColumn(t *testing.T) { + statuses := []*PackageStatus{ + {PackageName: "axios", CatalogURL: "https://a"}, + {PackageName: "left-pad"}, + } + tests := []struct { + name string + withLinks bool + want []string + }{ + {name: "terminal", withLinks: true, want: []string{catalogLinkStyle.Render(text.Hyperlink("https://a", "Open ↗")) + " ", ""}}, + {name: "not a terminal", withLinks: false, want: []string{"", ""}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got []string + for _, row := range convertToPackageStatusTable(statuses, tt.withLinks) { + got = append(got, row.Link) + if row.PackageName != "axios " && row.PackageName != "left-pad" { + t.Errorf("PackageName = %q, want the plain name", row.PackageName) + } + } + assert.Equal(t, tt.want, got) + }) + } +} + +func TestPackageStatusTableOmitsEmptyLinkColumn(t *testing.T) { + tw, err := coreutils.PrepareTable(convertToPackageStatusTable([]*PackageStatus{{PackageName: "axios", CatalogURL: "https://a"}}, false), "", false) + require.NoError(t, err) + if got := tw.Render(); strings.Contains(got, "LINK") { + t.Errorf("PrepareTable(rows without links) has a link column:\n%s", got) + } +} diff --git a/commands/curation/npmlogparser.go b/commands/curation/npmlogparser.go index 98a0c61b9..6fadc3793 100644 --- a/commands/curation/npmlogparser.go +++ b/commands/curation/npmlogparser.go @@ -50,15 +50,17 @@ const ( npmEntryETARGET ) -// npmBlockedInfo carries the policy/condition parsed from a whole-package-block notice line. +// npmBlockedInfo carries the policies and Catalog link parsed from a whole-package-block notice line. type npmBlockedInfo struct { - Policy string - Condition string + Policies []Policy + CatalogURL string } var ( // Matches: 42 notice All versions blocked - {policy:X,condition:Y} npmNoticeBlockedRegex = regexp.MustCompile(`^\d+\s+notice\s+All versions blocked - \{policy:([^,]+),condition:([^}]+)\}`) + // Matches: 42 notice All versions of 'X' (repo) are blocked from download by JFrog Curation. Blocked by ... + npmNoticeAllVersionsOfRegex = regexp.MustCompile(`^\d+\s+notice\s+` + allVersionsOfPattern) // Matches: 43 http fetch GET 403 https://.../api/npm// ... — excludes tarball fetches (path has /-/). npmFetch403Regex = regexp.MustCompile(`^\d+\s+http fetch GET 403\s+\S*/api/npm/[^/\s]+/((?:@[^/\s]+/)?[^/\s]+)\s`) // Matches: 110 silly placeDep ROOT accepts@2.0.0 OK for: express@5.2.1 want: ^2.0.0 @@ -96,7 +98,37 @@ func parseNpmDebugLog(logsDir string, afterKey int64) (entries []npmLogEntry, bl // npmUnknownBlockedInfo replaces a guessed policy/condition when the notice→403 FIFO pairing // is ambiguous — a wrong guess is worse than an honest "unknown". -var npmUnknownBlockedInfo = npmBlockedInfo{Policy: "unknown", Condition: "unknown"} +var npmUnknownBlockedInfo = npmBlockedInfo{Policies: []Policy{{Policy: "unknown", Condition: "unknown"}}} + +// parseNpmBlockedNotice returns the policies of a whole-package-block notice line, in either message format. +func parseNpmBlockedNotice(line string) (npmBlockedInfo, bool) { + if m := npmNoticeBlockedRegex.FindStringSubmatch(line); m != nil { + return npmBlockedInfo{Policies: []Policy{{Policy: strings.TrimSpace(m[1]), Condition: strings.TrimSpace(m[2])}}}, true + } + if m := npmNoticeAllVersionsOfRegex.FindStringSubmatch(line); m != nil { + return npmBlockedInfo{Policies: allVersionsOfPolicies(m), CatalogURL: m[4]}, true + } + return npmBlockedInfo{}, false +} + +// allVersionsOfPolicies returns the policies of a match of allVersionsOfPattern. Every policy gets the +// Action line, preceded by the Note line, as its Recommendation. +func allVersionsOfPolicies(m []string) []Policy { + recommendation := m[3] + if recommendation != "" && m[2] != "" { + recommendation = m[2] + " " + recommendation + } + var policies []Policy + for _, item := range allVersionsOfPolicyNumberRegex.Split(m[1], -1) { + if p := allVersionsOfPolicyItemRegex.FindStringSubmatch(item); p != nil { + policies = append(policies, Policy{Policy: p[1], Condition: p[2], Recommendation: recommendation}) + } + } + if len(policies) == 0 { + return []Policy{{Recommendation: recommendation}} + } + return policies +} // pendingNotice is a queued "notice" awaiting its matching 403. ambiguous is set once it has // ever overlapped with another unresolved notice, since FIFO order can no longer be trusted @@ -123,8 +155,8 @@ func scanNpmDebugLogChunk(logFile string, entries []npmLogEntry, blockedPackages for scanner.Scan() { line := scanner.Text() - if m := npmNoticeBlockedRegex.FindStringSubmatch(line); m != nil { - next := pendingNotice{info: npmBlockedInfo{Policy: strings.TrimSpace(m[1]), Condition: strings.TrimSpace(m[2])}} + if info, ok := parseNpmBlockedNotice(line); ok { + next := pendingNotice{info: info} if len(pendingNotices) > 0 { // A prior notice is still unresolved — both it and this one become ambiguous, // since same-relative-order with their eventual 403s is no longer provable. diff --git a/commands/curation/npmlogparser_test.go b/commands/curation/npmlogparser_test.go index fdeb70018..ea26a628f 100644 --- a/commands/curation/npmlogparser_test.go +++ b/commands/curation/npmlogparser_test.go @@ -21,8 +21,8 @@ func TestParseNpmDebugLog(t *testing.T) { // Whole-package block: notice immediately followed by the matching 403 fetch line. require.Contains(t, blockedPackages, "depd") - assert.Equal(t, "blocks open ssf", blockedPackages["depd"].Policy) - assert.Equal(t, "open ssf", blockedPackages["depd"].Condition) + assert.Equal(t, "blocks open ssf", blockedPackages["depd"].Policies[0].Policy) + assert.Equal(t, "open ssf", blockedPackages["depd"].Policies[0].Condition) byName := map[string]npmLogEntry{} for _, e := range entries { @@ -162,8 +162,8 @@ func TestParseNpmDebugLog_OverlappingPendingNoticesDegradeToUnknownPolicy(t *tes require.NoError(t, err) require.Contains(t, blockedPackages, "pkg-a", "still reported as blocked — just with unknown policy, not dropped entirely") require.Contains(t, blockedPackages, "pkg-b") - assert.Equal(t, "unknown", blockedPackages["pkg-a"].Policy) - assert.Equal(t, "unknown", blockedPackages["pkg-b"].Policy) + assert.Equal(t, "unknown", blockedPackages["pkg-a"].Policies[0].Policy) + assert.Equal(t, "unknown", blockedPackages["pkg-b"].Policies[0].Policy) } // A single notice resolved before the next one ever queues is unambiguous — FIFO order is @@ -183,8 +183,8 @@ func TestParseNpmDebugLog_SequentialNonOverlappingNoticesKeepRealPolicy(t *testi require.NoError(t, err) require.Contains(t, blockedPackages, "pkg-a") require.Contains(t, blockedPackages, "pkg-b") - assert.Equal(t, "blocks-cve", blockedPackages["pkg-a"].Policy) - assert.Equal(t, "blocks-license", blockedPackages["pkg-b"].Policy) + assert.Equal(t, "blocks-cve", blockedPackages["pkg-a"].Policies[0].Policy) + assert.Equal(t, "blocks-license", blockedPackages["pkg-b"].Policies[0].Policy) } // npm splits one run's debug log into multiple numbered chunk files past 50,000 log lines, @@ -221,7 +221,7 @@ func TestParseNpmDebugLog_MultipleChunksOfSameRunAreAllRead(t *testing.T) { require.Contains(t, blockedPackages, "depd", "the notice from chunk 0 must still pair with its matching 403 in chunk 1") - assert.Equal(t, "blocks open ssf", blockedPackages["depd"].Policy) + assert.Equal(t, "blocks open ssf", blockedPackages["depd"].Policies[0].Policy) } func TestParseNpmDebugLog_NoLogsDir(t *testing.T) { @@ -232,6 +232,75 @@ func TestParseNpmDebugLog_NoLogsDir(t *testing.T) { assert.Empty(t, logFilePaths) } +func TestParseNpmBlockedNotice(t *testing.T) { + const header = "42 notice All versions of '@hono/node-server' (npm-remote) are blocked from download by JFrog Curation." + const waiverAction = "Please use a different package or request a waiver to use this one." + const withNote = "Some versions are pending JFrog Catalog analysis." + tests := []struct { + name string + line string + want npmBlockedInfo + wantOk bool + }{ + { + name: "every listed policy, conditions with parentheses", + line: header + " Blocked by one or more of these policies: 1. BlockHighCVEs (CVE with CVSS score between 7.0 and 8.9 (fix version available)) " + + "2. Block Immature (package pending JFrog Catalog analysis) Note: Some versions are pending JFrog Catalog analysis. " + + "Action: " + waiverAction + " Details: https://example.jfrog.io/ui/catalog", + want: npmBlockedInfo{CatalogURL: "https://example.jfrog.io/ui/catalog", Policies: []Policy{ + {Policy: "BlockHighCVEs", Condition: "CVE with CVSS score between 7.0 and 8.9 (fix version available)", Recommendation: withNote + " " + waiverAction}, + {Policy: "Block Immature", Condition: "package pending JFrog Catalog analysis", Recommendation: withNote + " " + waiverAction}, + }}, + wantOk: true, + }, + { + name: "no policy list", + line: header + " Action: " + waiverAction, + want: npmBlockedInfo{Policies: []Policy{{Recommendation: waiverAction}}}, + wantOk: true, + }, + { + name: "different capitalization", + line: "42 notice ALL VERSIONS OF 'x' (r) ARE BLOCKED FROM DOWNLOAD BY JFROG CURATION. Action: " + waiverAction, + want: npmBlockedInfo{Policies: []Policy{{Recommendation: waiverAction}}}, + wantOk: true, + }, + { + name: "reworded tail", + line: header + " Blocked by these policies: 1. P (C)", + want: npmBlockedInfo{Policies: []Policy{{}}}, + wantOk: true, + }, + { + name: "old format", + line: "42 notice All versions blocked - {policy:blocks open ssf,condition:open ssf}", + want: npmBlockedInfo{Policies: []Policy{{Policy: "blocks open ssf", Condition: "open ssf"}}}, + wantOk: true, + }, + { + name: "unrelated notice", + line: "42 notice New major version of npm available!", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok := parseNpmBlockedNotice(tt.line) + assert.Equal(t, tt.wantOk, ok) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestWholePackageBlockedStatusKeepsEveryPolicy(t *testing.T) { + info := npmBlockedInfo{CatalogURL: "u", Policies: []Policy{{Policy: "p1", Condition: "c1", Explanation: "e", Recommendation: "a"}, {Policy: "p2", Condition: "c2", Recommendation: "a"}}} + got := wholePackageBlockedStatus(npmLogEntry{Name: "axios"}, info, techutils.Npm) + assert.Equal(t, []Policy{ + {Policy: "p1", Condition: "c1", Explanation: "e", Recommendation: "a"}, + {Policy: "p2", Condition: "c2", Recommendation: "a"}, + }, got.Policy) + assert.Equal(t, "u", got.CatalogURL) +} + // Git host shorthand (github:/gitlab:/bitbucket:/gist:) must not regress — real evidence // (github:jonschlinkert/is-thirteen) depends on it. func TestClassifyNpmSpecifier(t *testing.T) { @@ -276,7 +345,7 @@ func TestClassifyNpmSpecifier(t *testing.T) { } func TestClassifyBlankVersionEntry(t *testing.T) { - blockedPackages := map[string]npmBlockedInfo{"depd": {Policy: "blocks open ssf", Condition: "open ssf"}} + blockedPackages := map[string]npmBlockedInfo{"depd": {Policies: []Policy{{Policy: "blocks open ssf", Condition: "open ssf"}}}} assert.Equal(t, npmEntryResolved, classifyBlankVersionEntry( npmLogEntry{Name: "accepts", Version: "2.0.0"}, blockedPackages)) @@ -369,7 +438,7 @@ func TestResolveNpmAliasEntriesLeavesMalformedAliasUntouched(t *testing.T) { // of npmEntryNonRegistrySpecifier — the exact misclassification reported live against a // "blocks open ssf" policy (RTECO-1882 / XRAY-157605). func TestClassifyBlankVersionEntryAfterAliasResolution(t *testing.T) { - blockedPackages := map[string]npmBlockedInfo{"strip-ansi": {Policy: "blocks open ssf", Condition: "open ssf"}} + blockedPackages := map[string]npmBlockedInfo{"strip-ansi": {Policies: []Policy{{Policy: "blocks open ssf", Condition: "open ssf"}}}} entries := resolveNpmAliasEntries([]npmLogEntry{ {Name: "strip-ansi-cjs", Version: "", ParentName: "myproj", ParentVersion: "1.0.0", Specifier: "npm:strip-ansi@6.0.1"}, }) diff --git a/sca/bom/buildinfo/technologies/python/python_cvs_fallback.go b/sca/bom/buildinfo/technologies/python/python_cvs_fallback.go index eb50e827f..2adabd7e4 100644 --- a/sca/bom/buildinfo/technologies/python/python_cvs_fallback.go +++ b/sca/bom/buildinfo/technologies/python/python_cvs_fallback.go @@ -1,6 +1,7 @@ package python import ( + "cmp" "errors" "fmt" "regexp" @@ -50,6 +51,14 @@ var pipFailedRangeReqRegex = regexp.MustCompile( `([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?((?:(?:~=|!=|>=|<=|>|<)[^\s,(]+(?:,(?:~=|!=|>=|<=|>|<)[^\s,(]+)*))` + `(?:\s+\(from\s+([A-Za-z0-9][A-Za-z0-9._-]*)(?:==([^\s)]+))?\))?`) +// pipFailedBareReqRegex matches a failed requirement with no version specifier in pip's error output. +// Groups: (1) name, (2) the first "(from ...)" clause: "versions: ..." for a direct requirement, else the parent. +var pipFailedBareReqRegex = regexp.MustCompile( + `satisfies the requirement\s+([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?\s+\(from\s+([^)]*)\)`) + +// pipFromParentRegex matches the parent in a "(from ...)" clause. Groups: (1) name, (2) version (optional). +var pipFromParentRegex = regexp.MustCompile(`^([A-Za-z0-9][A-Za-z0-9._-]*)(?:==(\S+))?$`) + // pipCollectingRegex matches packages pip collected before failing; used to recover parent version omitted in "(from )". // Groups: (1) name, (2) version. var pipCollectingRegex = regexp.MustCompile( @@ -185,6 +194,19 @@ func parseCvsFailedPackages(pipOutput string) []PinnedRequirement { failed = append(failed, pr) } + for _, m := range pipFailedBareReqRegex.FindAllStringSubmatch(pipOutput, -1) { + name := normalizePyPIName(m[1]) + if seen[name] { + continue + } + seen[name] = true + pr := PinnedRequirement{Name: name, ParentName: name, ConfirmedDirect: true} + if p := pipFromParentRegex.FindStringSubmatch(m[2]); p != nil { + pr.ParentName, pr.ParentVersion, pr.ConfirmedDirect = normalizePyPIName(p[1]), cmp.Or(p[2], collected[normalizePyPIName(p[1])]), false + } + failed = append(failed, pr) + } + // ResolutionImpossible: deep transitive deps stripped by CVS never appear in a // "No matching distribution found" line, so add them as name-only entries. if m := pipResolutionImpossibleRegex.FindStringSubmatch(pipOutput); len(m) > 1 { diff --git a/sca/bom/buildinfo/technologies/python/python_cvs_fallback_test.go b/sca/bom/buildinfo/technologies/python/python_cvs_fallback_test.go index e2b34761f..ca6717147 100644 --- a/sca/bom/buildinfo/technologies/python/python_cvs_fallback_test.go +++ b/sca/bom/buildinfo/technologies/python/python_cvs_fallback_test.go @@ -35,6 +35,16 @@ func TestParseCvsFailedPackages(t *testing.T) { {Name: "deepagents", Version: "0.5.5", ParentName: "deepagents", ParentVersion: "0.5.5"}, }, }, + { + name: "no version specifier", + output: "ERROR: Could not find a version that satisfies the requirement sccp (from versions: none)\nERROR: No matching distribution found for sccp\n", + want: []PinnedRequirement{{Name: "sccp", ParentName: "sccp", ConfirmedDirect: true}}, + }, + { + name: "no version specifier, transitive", + output: "ERROR: Could not find a version that satisfies the requirement sccp (from parentpkg==1.0) (from versions: none)\nERROR: No matching distribution found for sccp\n", + want: []PinnedRequirement{{Name: "sccp", ParentName: "parentpkg", ParentVersion: "1.0"}}, + }, { name: "range spec without parent", output: "ERROR: No matching distribution found for langchain-core>=1.4.0,<2.0.0",