diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b9d2cb48..72d39c19b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - Maven dependencies with missing or invalid POMs are reported in every module that uses them, `system`-scope jars reach reachability analysis, and a failing dependency no longer fails scopes that `excludeConfigs` leaves out or the build when it lost a version conflict. - Reachability scans that hit unresolved JVM dependencies now point out that ignoring them also needs `--reach-continue-on-install-errors`. - A Gradle dependency whose variant is ambiguous in every configuration of its project now fails the run instead of quietly dropping out of the scan. +- Gradle dependencies that fail dependency verification, or whose jars cannot be fetched, are reported instead of passed on as healthy. +- Failed Gradle project dependencies are reported instead of silently dropping everything they pull in, and failures name the version that actually failed. ## [1.10.0](https://github.com/SocketDev/socket-cli/releases/tag/v1.10.0) - 2026-10-22 diff --git a/src/commands/manifest/scripts/resolution-report-gradle.mts b/src/commands/manifest/scripts/resolution-report-gradle.mts index 1b181956d..b2ff74a4a 100644 --- a/src/commands/manifest/scripts/resolution-report-gradle.mts +++ b/src/commands/manifest/scripts/resolution-report-gradle.mts @@ -21,9 +21,26 @@ export function classifyGradleFailure(detail: string): FailureCategory { ) { return 'no-matching-variant' } - if (t.includes('cannot choose between')) { + // The artifact-level form: several transforms produce the requested variant + // ("transformation chains" from Gradle 8.12, "transforms" before it). + if ( + t.includes('cannot choose between') || + t.includes('found multiple transformation chains') || + t.includes('found multiple transforms that can produce a variant') + ) { return 'variant-ambiguity' } + // One line per problem of the artifact; a checksum missing from the + // metadata is no evidence against it, unlike any other problem. + const problems = t.split('\n').filter(l => l.startsWith('on artifact ')) + if ( + problems.length && + problems.every(l => + l.endsWith(': checksum is missing from verification metadata.'), + ) + ) { + return 'checksum-missing' + } if ( t.includes('could not get') || t.includes('could not head') || @@ -46,7 +63,8 @@ export function classifyGradleFailure(detail: string): FailureCategory { // Every kind blocks except variant-ambiguity: the module demonstrably exists // and is almost always captured via another configuration, so it's benign for // the SBOM (non-blocking, count only). The renderer blocks it anyway when the -// module resolved in no configuration of its project. +// module resolved in no configuration of its project. Nor does checksum-missing: +// teams often record checksums only for what their own tasks resolve. export const GRADLE_DIALECT: ResolutionDialect = { label: 'Gradle', classify: classifyGradleFailure, @@ -95,5 +113,11 @@ export const GRADLE_DIALECT: ResolutionDialect = { notice: (n, depCount, configCount) => `Skipped ${depCount} dependency(ies) with ambiguous variant selection in ${configCount} configuration(s) — re-run with --verbose for ${n}'s messages.`, }, + { + key: 'checksum-missing', + blocking: false, + notice: (n, depCount, configCount) => + `Left ${depCount} dependency(ies) uninstalled in ${configCount} configuration(s): ${n}'s verification metadata has no checksum for their artifacts — re-run with --verbose for ${n}'s messages.`, + }, ], } diff --git a/src/commands/manifest/scripts/resolution-report-render.mts b/src/commands/manifest/scripts/resolution-report-render.mts index 2fccc81f9..0abcff773 100644 --- a/src/commands/manifest/scripts/resolution-report-render.mts +++ b/src/commands/manifest/scripts/resolution-report-render.mts @@ -15,6 +15,7 @@ export type FailureCategory = | 'no-matching-variant' | 'capability-conflict' | 'variant-ambiguity' + | 'checksum-missing' | 'repository-or-network' | 'config-problem' | 'other' diff --git a/src/commands/manifest/scripts/resolution-report-render.test.mts b/src/commands/manifest/scripts/resolution-report-render.test.mts index cfb79bc9a..df3f42fd1 100644 --- a/src/commands/manifest/scripts/resolution-report-render.test.mts +++ b/src/commands/manifest/scripts/resolution-report-render.test.mts @@ -5,6 +5,7 @@ import { REACH_CONTINUE_ON_INSTALL_ERRORS_FLAG, renderResolutionErrorReport, } from './resolution-report-render.mts' +import { moduleKey } from './resolution-report.mts' import type { BuildOptionHints } from './resolution-report-render.mts' import type { ResolutionFailure } from './resolution-report.mts' @@ -49,6 +50,63 @@ describe('resolution failure classification', () => { expect(r.nonBlockingNotice).toContain('ambiguous variant') }) + it.each([ + 'Found multiple transformation chains that produce a variant of com.example:amb:1.0', + 'Found multiple transforms that can produce a variant of com.example:amb:1.0', + ])( + 'classifies an ambiguous Gradle artifact transform as variant ambiguity: %s', + detail => { + const r = renderResolutionErrorReport( + [f('com.example:amb:1.0', detail)], + ['runtimeClasspath'], + 'gradle', + ) + expect(r.hasBlockingFailures).toBe(false) + expect(r.nonBlockingNotice).toContain('ambiguous variant') + }, + ) + + it('treats a checksum missing from Gradle verification metadata as non-blocking, unlike a mismatch', () => { + const header = + "Dependency verification failed for configuration ':runtimeClasspath':" + const missing = renderResolutionErrorReport( + [ + f( + 'com.example:lib:1.0', + `${header}\nOn artifact lib-1.0.jar (com.example:lib:1.0) in repository 'maven': checksum is missing from verification metadata.`, + ), + ], + ['runtimeClasspath'], + 'gradle', + { resolvedModules: new Set([moduleKey(':', 'com.example:lib')]) }, + ) + expect(missing.hasBlockingFailures).toBe(false) + expect(missing.nonBlockingNotice).toContain('no checksum') + const mismatch = renderResolutionErrorReport( + [ + f( + 'com.example:lib:1.0', + `${header}\nOn artifact lib-1.0.jar (com.example:lib:1.0) in repository 'maven': expected a 'sha256' checksum of '00' but was '11'`, + ), + ], + ['runtimeClasspath'], + 'gradle', + ) + expect(mismatch.hasBlockingFailures).toBe(true) + const both = renderResolutionErrorReport( + [ + f( + 'com.example:lib:1.0', + `${header}\nOn artifact lib-1.0.jar (com.example:lib:1.0) in repository 'a': checksum is missing from verification metadata.\nOn artifact lib-1.0.jar (com.example:lib:1.0) in repository 'b': expected a 'sha256' checksum of '00' but was '11'`, + ), + ], + ['runtimeClasspath'], + 'gradle', + { resolvedModules: new Set([moduleKey(':', 'com.example:lib')]) }, + ) + expect(both.hasBlockingFailures).toBe(true) + }) + it("classifies Gradle's 'unable to find a matching variant' phrasing as blocking no-matching-variant", () => { const r = renderResolutionErrorReport( [ diff --git a/src/commands/manifest/scripts/run.mts b/src/commands/manifest/scripts/run.mts index bb3ad29a2..23ecb0c32 100644 --- a/src/commands/manifest/scripts/run.mts +++ b/src/commands/manifest/scripts/run.mts @@ -256,6 +256,9 @@ async function invokeGradle( '--init-script', initScript, '-Dorg.gradle.configuration-cache=false', + // Names each artifact's own verification problem, so a checksum missing + // from the metadata can be told apart from a mismatch. + '-Porg.gradle.dependency.verification.console=verbose', `-Psocket.recordsFile=${recordsFile}`, ...commonProps(opts, '-P'), ...(opts.toolOpts ?? []), diff --git a/src/commands/manifest/scripts/socket-facts.init.gradle b/src/commands/manifest/scripts/socket-facts.init.gradle index e7b64340d..730f4698b 100644 --- a/src/commands/manifest/scripts/socket-facts.init.gradle +++ b/src/commands/manifest/scripts/socket-facts.init.gradle @@ -53,7 +53,6 @@ gradle.ext.socketFactsState = [ // Whole configs whose resolution threw (vs a single unresolved dep in `failures`). unscannable : Collections.synchronizedList([]), scannedConfigs : Collections.synchronizedSet([] as Set), - projectKeys : Collections.synchronizedSet([] as Set), // Only populated with `-Psocket.withFiles=true`: reading `artifact.file` downloads the artifact. paths : Collections.synchronizedMap([:]), perSub : Collections.synchronizedMap([:]), @@ -63,10 +62,12 @@ gradle.ext.socketFactsState = [ projectArtifactExt : Collections.synchronizedMap([:]), // Project path -> "group:name", to tell this build's projects from an included build's. projectGaByPath : Collections.synchronizedMap([:]), + // Project path -> "group:name:version", the coordinate a failed project dependency is reported by. + projectGavByPath : Collections.synchronizedMap([:]), ] -// Capture every project's (group:name) before collectors run so they can filter intra-project -// deps without an ordering dependency on other subprojects. +// Capture every project's identity before collectors run so they can recognize intra-project deps +// without an ordering dependency on other subprojects. gradle.projectsEvaluated { g -> def rootPath = g.rootProject.projectDir.toPath() // '/'-normalized so values are stable across OSes and portable to another machine. @@ -75,7 +76,6 @@ gradle.projectsEvaluated { g -> r.isEmpty() ? '.' : r } g.rootProject.allprojects.each { p -> - g.socketFactsState.projectKeys.add("${p.group ?: ''}:${p.name}") // The module's real artifact extension (jar for java; '' for a project that builds no archive, // e.g. a pure aggregator). Used to stamp intra-project deps that resolve without an artifact. def artExt = '' @@ -90,9 +90,10 @@ gradle.projectsEvaluated { g -> } catch (Exception ignore) {} g.socketFactsState.projectArtifactExt[p.path] = artExt g.socketFactsState.projectGaByPath[p.path] = "${p.group ?: ''}:${p.name}".toString() + g.socketFactsState.projectGavByPath[p.path] = "${p.group ?: ''}:${p.name}:${p.version ?: ''}".toString() // A wholly excluded subproject emits no project record and its configs are never resolved (see the - // collector). projectKeys/projectArtifactExt above stay populated so a KEPT project depending on it - // still recognizes it as a first-party module. + // collector). The project maps above stay populated so a KEPT project depending on it still + // recognizes it as a first-party module. if (g.ext.socketIsExcluded.call(rel(p.projectDir))) { return } @@ -157,13 +158,15 @@ allprojects { project -> // visit/upsertNode closures capture them); `failures`/`scannedConfigs` stay shared. def nodes = [:] def directIds = [] as Set + def configName = '' // "group:name:version" -> path of the build project this config resolved it to. def projectByGav = [:] def failures = state.failures def scannedConfigs = state.scannedConfigs - def projectKeys = state.projectKeys def projectArtifactExt = state.projectArtifactExt def paths = [:] + // Ids of nodes whose requested artifact yielded no file. + def unfetched = [] as Set def withFiles = gradle.socketProp.call(project, 'socket.withFiles')?.toString()?.toLowerCase() == 'true' // Optional `-Psocket.populateFilesFor=` (newline-delimited GAVs): scope `--with-files` // materialization to those GAVs. This run walks EVERY resolvable config (not told the SBOM's @@ -203,10 +206,6 @@ allprojects { project -> [coord.groupId, coord.artifactId, coord.ext, coord.classifier, coord.version].findAll { it }.join(':') } - def isIntraProject = { String group, String name -> - projectKeys.contains("${group ?: ''}:${name}") - } - // A first-party module dep can resolve with no published artifact (classes-dir variant, or // variant-ambiguous moduleArtifacts), leaving it ext-less. Stamp the module's real artifact // extension so it keeps a full, stable coordinate instead of an ext-less one. @@ -234,6 +233,24 @@ allprojects { project -> id } + def causeMessages = { problem -> + def out = [] + def t = problem + int guard = 0 + while (t != null && guard++ < 12) { out << t.message; t = t.cause } + out + } + + // The deepest cause's full message: the TS report classifies it, so nothing is categorized here. + def rawDetail = { problem -> + def messages = causeMessages(problem).findAll { it } + return (messages ? messages[-1] : 'unknown resolution failure').toString().trim() + } + + def recordFailure = { String coord, String detail -> + failures << [coord: coord, detail: detail, config: configName, project: project.path] + } + // Walk a resolved dependency and its transitive closure. We never touch `artifact.file` here — // that forces Gradle to download the file (catastrophic on builds declaring distribution // archives as deps); `artifact.extension`/`artifact.classifier` read already-resolved metadata. @@ -251,15 +268,23 @@ allprojects { project -> // `moduleArtifacts` forces ARTIFACT-level variant selection, which can throw on a // variant-ambiguous dep even when the dependency GRAPH resolves fine. Guard per-dep: on - // failure emit at GAV level and keep walking so one ambiguous artifact can't sink the config. + // failure keep walking so one ambiguous artifact can't sink the config. + def gav = "${dep.moduleGroup ?: ''}:${dep.moduleName}:${dep.moduleVersion ?: ''}".toString() + def shouldMaterialize = withFiles && (populateGavs == null || populateGavs.contains(gav)) + def projPath = projectByGav[gav] def artifacts + def artifactsError = null try { artifacts = dep.moduleArtifacts } catch (Exception e) { artifacts = [] as Set + artifactsError = e } - def projPath = projectByGav["${dep.moduleGroup ?: ''}:${dep.moduleName}:${dep.moduleVersion ?: ''}".toString()] - if (artifacts.isEmpty()) { + if (artifactsError != null && projPath == null) { + // No artifact was selected, so there is no coordinate to emit: report the module and + // bubble its children up (below), as for a skipped first-party module. + recordFailure(gav, rawDetail(artifactsError)) + } else if (artifacts.isEmpty()) { def ext = effExt(projPath, '') // Skip a no-artifact first-party module (aggregator / build root): it builds no archive, so // it's fully described by `projects` and is never a real artifact dependency. @@ -273,17 +298,6 @@ allprojects { project -> ], isProd, projPath) } } else { - // Build the GAV scope key only when a scope is set: the no-scope path never reads it and - // this is the hottest line under --parallel. - def shouldMaterialize - if (!withFiles) { - shouldMaterialize = false - } else if (populateGavs == null) { - shouldMaterialize = true - } else { - def gav = "${dep.moduleGroup ?: ''}:${dep.moduleName}:${dep.moduleVersion ?: ''}".toString() - shouldMaterialize = populateGavs.contains(gav) - } artifacts.each { a -> // Directory variants (java-classes-directory etc.) carry no extension; for a first-party // module fall back to its real artifact ext (never artifact.type, a Gradle variant attr). @@ -310,7 +324,13 @@ allprojects { project -> set << path } } catch (Exception e) { - println "[socket-facts] could not materialize ${aid}: ${e.message?.readLines()?.first()}" + println "[socket-facts] could not materialize ${aid}: ${rawDetail(e).readLines().find()}" + // A requested artifact that yields no file is a failure: without one, the dependency + // would read as resolved with nothing to analyse. + if (projPath == null) { + recordFailure(gav, rawDetail(e)) + unfetched << aid + } } } } @@ -320,9 +340,9 @@ allprojects { project -> dep.children.each { child -> childIds.addAll(visit(child, isProd, cache)) } - // A skipped no-artifact first-party module produced no node, so bubble its resolved children - // up to the parent (return childIds) — otherwise the module's transitives orphan (attached to - // nothing, not bubbled). The module itself is described by `projects`. + // A skipped no-artifact first-party module, or a module whose artifacts could not be + // selected, produced no node, so bubble its resolved children up to the parent (return + // childIds); otherwise the module's transitives would be attached to nothing. if (producedIds.isEmpty()) { cache[dep] = childIds return childIds @@ -408,13 +428,168 @@ allprojects { project -> return true } - // Deepest cause's message, in FULL — Coana classifies/templates it, so we don't categorize here. - def rawDetail = { problem -> - def deepest = null - def t = problem - int guard = 0 - while (t != null && guard++ < 12) { if (t.message) { deepest = t.message }; t = t.cause } - return (deepest ?: 'unknown resolution failure').toString().trim() + // A failed edge is reported by the selector Gradle ATTEMPTED (after forces and substitutions). + // A project of this build is named by its own coordinate, as its resolved node would be. + def selectorCoord = { sel -> + if (sel instanceof org.gradle.api.artifacts.component.ModuleComponentSelector) { + return "${sel.group}:${sel.module}${sel.version ? ':' + sel.version : ''}".toString() + } + if (sel instanceof org.gradle.api.artifacts.component.ProjectComponentSelector) { + def build = sel.metaClass.respondsTo(sel, 'getBuildPath') ? sel.buildPath : + sel.metaClass.respondsTo(sel, 'getBuildName') ? sel.buildName : ':' + def gav = build == ':' ? state.projectGavByPath[sel.projectPath] : null + if (gav != null) { + return gav + } + } + sel.displayName.toString() + } + + // Gradle verifies only artifact-set reads, never the walk's `ResolvedArtifact.file`, and fails + // them with every fatal failure so far; only STRICT throws, from 6.2, with the root build's metadata. + def rootBuild = gradle + while (rootBuild.parent != null) { + rootBuild = rootBuild.parent + } + def startParameter = gradle.startParameter + def verifyArtifacts = withFiles && + new File(rootBuild.rootProject.rootDir, 'gradle/verification-metadata.xml').isFile() && + startParameter.metaClass.respondsTo(startParameter, 'getDependencyVerificationMode') && + startParameter.dependencyVerificationMode.toString() == 'STRICT' + def gavOf = { Map c -> "${c.groupId}:${c.artifactId}:${c.version}".toString() } + def verificationMessage = { Throwable e -> + causeMessages(e).find { it?.startsWith('Dependency verification failed') } + } + // Old Gradle splits a module into one ResolvedDependency per target configuration, of which only + // some carry its artifacts: an artifact-less one is the same component, so fold it into them. + def foldArtifactless = { + if (!nodes.values().any { n -> !n.coord.ext && n.project == null }) { + return + } + def withExt = [:] + nodes.each { id, n -> + if (n.coord.ext) { + withExt.get(gavOf(n.coord), []) << id + } + } + def folds = [:] + nodes.each { id, n -> + def siblings = (!n.coord.ext && n.project == null) ? withExt[gavOf(n.coord)] : null + if (siblings) { + folds[id] = siblings + } + } + if (folds.isEmpty()) { + return + } + def replaced = { Collection ids -> ids.collectMany { c -> folds[c] ?: [c] } } + folds.each { id, siblings -> + def folded = nodes.remove(id) + siblings.each { sid -> + nodes[sid].children.addAll(replaced(folded.children).findAll { it != sid }) + if (folded.prod) { + nodes[sid].prod = true + } + } + } + nodes.each { nid, n -> + if (n.children.any { folds.containsKey(it) }) { + def children = replaced(n.children).findAll { it != nid } + n.children.clear() + n.children.addAll(children) + } + } + def direct = replaced(directIds) + directIds.clear() + directIds.addAll(direct) + } + + // Drops the files of this config's artifacts that failed verification and reports them. + def checkVerification = { cfg -> + def external = paths.keySet().findAll { id -> nodes[id].project == null } + if (external.isEmpty()) { + return + } + def fetched = external.collect { id -> gavOf(nodes[id].coord) } as Set + def message = null + try { + cfg.incoming.artifactView { + lenient(true) + componentFilter { id -> + id instanceof org.gradle.api.artifacts.component.ModuleComponentIdentifier && + fetched.contains("${id.group}:${id.module}:${id.version}".toString()) + } + }.artifacts.each { it.file } + } catch (Exception e) { + message = verificationMessage(e) + if (message == null) { + // Any other failure concerns files the walk already read without it. + println "[socket-facts] could not check artifact verification for ${cfg.name}: ${rawDetail(e).readLines().find()}" + } + } + if (message == null) { + return + } + // Each failed artifact is named by its identifier's display name, " (::)", + // whose file part is `-[-].` rather than the cached file's own name. + def artifactRef = /(\S+) \(([^\s():]+):([^\s():]+):([^\s()]+)\)/ + def lines = message.readLines() + def header = lines ? lines[0] : '' + // The verbose report (the Socket CLI sets org.gradle.dependency.verification.console=verbose) gives + // each artifact's problems: "On artifact in repository '': ", or "On artifact + // multiple problems reported:" followed by one "in repository '': " line each. + def failed = [] + def current = null + lines.each { raw -> + def line = raw.trim().replaceFirst(/^- /, '') + def m = line =~ ('^On artifact ' + artifactRef + ' (.*)$') + if (m.matches()) { + current = [gav: "${m.group(2)}:${m.group(3)}:${m.group(4)}".toString(), file: m.group(1), problems: []] + failed << current + if (m.group(5) != 'multiple problems reported:') { + current.problems << m.group(5) + } + } else if (current != null && line.startsWith("in repository '")) { + current.problems << line + } else { + current = null + } + } + failed.each { f -> + f.detail = ([header] + f.problems.collect { "On artifact ${f.file} (${f.gav}) ${it}" }).join('\n') + } + if (failed.isEmpty()) { + // The summary report names only the artifacts. + (message =~ artifactRef).each { m -> + failed << [gav: "${m[2]}:${m[3]}:${m[4]}".toString(), file: m[1], detail: message] + } + } + def markUnfetched = { Collection ids -> + ids.each { id -> + paths.remove(id) + unfetched << id + } + } + if (failed.isEmpty()) { + // No artifact can be told apart: the walked graph stands, without any of its external files. + state.unscannable << [config: cfg.name, detail: message, project: project.path] + markUnfetched(external) + return + } + failed.each { f -> + def ids = external.findAll { id -> gavOf(nodes[id].coord) == f.gav } + // A failed metadata file (pom, module) names none of the module's artifacts: all are suspect. + // A failed artifact this config did not fetch leaves it untouched. + def hit = (f.file.endsWith('.pom') || f.file.endsWith('.module')) ? ids : ids.findAll { id -> + def c = nodes[id].coord + def named = "${c.artifactId}-${c.version}${c.classifier ? '-' + c.classifier : ''}${c.ext ? '.' + c.ext : ''}" + f.file == named.toString() || paths[id]?.any { p -> f.file == new File(p).name } + } + if (!hit.isEmpty()) { + recordFailure(f.gav, f.detail) + markUnfetched(hit) + } + } } targetConfigs.each { cfg -> @@ -426,10 +601,13 @@ allprojects { project -> // Fresh tree for THIS resolution root — captured by visit/upsertNode, so no union with siblings. nodes = [:] directIds = [] as Set + configName = cfg.name paths = [:] + unfetched = [] as Set projectByGav = [:] // Per-config try/catch: AGP-style configs can fail variant ambiguity from an init-script // context lacking the consumer attributes AGP sets internally. + def walked = false try { def lenient = cfg.resolvedConfiguration.lenientConfiguration // The resolution graph names a project dependency by project path; within one graph a @@ -453,20 +631,25 @@ allprojects { project -> firstLevel.each { dep -> directIds.addAll(visit(dep, isProd, cache)) } + foldArtifactless() // Unresolved deps drive the root aggregator's abort/warn decision but are NOT emitted as // nodes — their selector-only coordinates would surface as half-formed phantom artifacts. - lenient.unresolvedModuleDependencies.each { dep -> - def sel = dep.selector - if (isIntraProject(sel.group, sel.name)) { - return + // hasError() covers every unresolved dependency, project ones included. + if (cfg.resolvedConfiguration.hasError()) { + def reported = [] as Set + cfg.incoming.resolutionResult.allDependencies.each { d -> + if (d instanceof org.gradle.api.artifacts.result.UnresolvedDependencyResult) { + def coord = selectorCoord(d.attempted ?: d.requested) + if (reported.add(coord)) { + recordFailure(coord, rawDetail(d.failure)) + } + } } - failures << [ - coord : "${sel.group}:${sel.name}${sel.version ? ':' + sel.version : ''}".toString(), - detail: rawDetail(dep.problem), - config: cfg.name, - project: project.path, - ] } + if (verifyArtifacts) { + checkVerification(cfg) + } + walked = true } catch (Exception e) { // A config-LEVEL resolution failure (a resolutionStrategy/beforeResolve rule that throws, // an artifact-transform failure, a broken repository, ...) rather than a per-dep failure. @@ -474,11 +657,11 @@ allprojects { project -> state.unscannable << [config: cfg.name, detail: rawDetail(e), project: project.path] } // Stash this resolution root's tree; the key only has to be unique per root. Skip empty - // configs — they'd contribute nothing. - if (!nodes.isEmpty()) { + // configs, which contribute nothing, and a partial walk, which can orphan components. + if (walked && !nodes.isEmpty()) { synchronized (state.perSub) { state.perSub["${project.path}::${cfg.name}".toString()] = - [nodes: nodes, direct: directIds, paths: paths, projectPath: project.path, config: cfg.name, prod: isProd] + [nodes: nodes, direct: directIds, paths: paths, unfetched: unfetched, projectPath: project.path, config: cfg.name, prod: isProd] } } } @@ -541,6 +724,7 @@ rootProject { rp -> node.children.each { childId -> rec(['edge', rootId, coordId, childId]) } def fs = tree.paths[coordId] if (fs) { (fs as List).sort().each { p -> rec(['file', rootId, coordId, p]) } } + if (tree.unfetched.contains(coordId)) rec(['unfetched', rootId, coordId]) } } diff --git a/src/commands/manifest/scripts/test/gradle-compat/.gitignore b/src/commands/manifest/scripts/test/gradle-compat/.gitignore index 45d17de21..176f55a41 100644 --- a/src/commands/manifest/scripts/test/gradle-compat/.gitignore +++ b/src/commands/manifest/scripts/test/gradle-compat/.gitignore @@ -6,3 +6,10 @@ project/.socket.facts.json project/.gradle/ project/build/ .populate-for.txt +failures/localrepo/ +failures/records.tsv +failures/.gradle/ +failures/build/ +failures/lib/build/ +failures/app/build/ +failures/categories.tsv diff --git a/src/commands/manifest/scripts/test/gradle-compat/failures/app/build.gradle b/src/commands/manifest/scripts/test/gradle-compat/failures/app/build.gradle new file mode 100644 index 000000000..6dc323bca --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/failures/app/build.gradle @@ -0,0 +1,8 @@ +// Kept out of the root project: on some old Gradle, a project dependency on a missing configuration +// fails resolving the declaring project's other configurations too. +configurations { + projectFail +} +dependencies { + projectFail project(path: ':lib', configuration: 'nonexistent') +} diff --git a/src/commands/manifest/scripts/test/gradle-compat/failures/build.gradle b/src/commands/manifest/scripts/test/gradle-compat/failures/build.gradle new file mode 100644 index 000000000..cdc64bb0f --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/failures/build.gradle @@ -0,0 +1,64 @@ +// Hermetic fixture for ../smoke-test-failures.sh: each configuration fails to resolve in one way, +// against a generated local Maven repo (see ../../make-stub-repo.sh). +allprojects { + repositories { + maven { url rootProject.file('localrepo').toURI() } + } +} + +configurations { + forced + shadowing + verified + walkAborted + quiet + unparsed +} +configurations.forced.resolutionStrategy.force 'demo:c:9.9' + +dependencies { + forced 'demo:a:1.0' + // Shares the group:name of project :lib, but is an external module. + shadowing 'demo:lib:7.7' + verified 'demo:a:1.0' + verified 'demo:tampered:1.0' + verified 'demo:unlisted:1.0' + walkAborted 'demo:a:1.0' + walkAborted 'demo:boom:1.0' + quiet 'demo:quiet:1.0' + unparsed 'demo:a:1.0' +} + +// Artifact transforms and dependency verification exist only on newer Gradle. +if (org.gradle.util.GradleVersion.current() >= org.gradle.util.GradleVersion.version('6.2')) { + apply from: 'transforms.gradle' +} + +// No build throws from the walk's own reads on demand, so make two of them throw for one module +// each: boom's children, and quiet's file with an empty message. unparsed's verification read +// fails with a message that names no artifact. +tasks.matching { it.name == 'socketFactsCollect' }.all { t -> + t.doFirst { + def boom = configurations.walkAborted.resolvedConfiguration.firstLevelModuleDependencies.find { it.moduleName == 'boom' } + def depMeta = boom.getClass().metaClass + def children = depMeta.getMetaMethod('getChildren', [] as Object[]) + depMeta.getChildren = { -> + if (delegate.moduleName == 'boom') { + throw new IllegalStateException('walk aborted by the fixture') + } + children.invoke(delegate) + } + def quiet = configurations.quiet.resolvedConfiguration.resolvedArtifacts.find { it.name == 'quiet' } + def artifactMeta = quiet.getClass().metaClass + def file = artifactMeta.getMetaMethod('getFile', [] as Object[]) + artifactMeta.getFile = { -> + if (delegate.name == 'quiet') { + throw new RuntimeException('') + } + file.invoke(delegate) + } + configurations.unparsed.incoming.metaClass.artifactView = { Closure spec -> + throw new GradleException("Dependency verification failed for configuration ':unparsed'") + } + } +} diff --git a/src/commands/manifest/scripts/test/gradle-compat/failures/gradle/verification-metadata.xml b/src/commands/manifest/scripts/test/gradle-compat/failures/gradle/verification-metadata.xml new file mode 100644 index 000000000..e79d509a0 --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/failures/gradle/verification-metadata.xml @@ -0,0 +1,18 @@ + + + + + false + false + + + + + + + + + + + + diff --git a/src/commands/manifest/scripts/test/gradle-compat/failures/lib/build.gradle b/src/commands/manifest/scripts/test/gradle-compat/failures/lib/build.gradle new file mode 100644 index 000000000..82fbf7fc9 --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/failures/lib/build.gradle @@ -0,0 +1,3 @@ +apply plugin: 'java' +group = 'demo' +version = '3.0' diff --git a/src/commands/manifest/scripts/test/gradle-compat/failures/settings.gradle b/src/commands/manifest/scripts/test/gradle-compat/failures/settings.gradle new file mode 100644 index 000000000..53f191a18 --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/failures/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'gradle-compat-failures' +include 'lib', 'app' diff --git a/src/commands/manifest/scripts/test/gradle-compat/failures/transforms.gradle b/src/commands/manifest/scripts/test/gradle-compat/failures/transforms.gradle new file mode 100644 index 000000000..e688dc60e --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/failures/transforms.gradle @@ -0,0 +1,25 @@ +// Two transforms producing the same artifact type, so selecting a `foo` artifact is ambiguous even +// though the graph resolves. +abstract class IdentityOne implements org.gradle.api.artifacts.transform.TransformAction { + @org.gradle.api.artifacts.transform.InputArtifact + abstract Provider getInputArtifact() + void transform(org.gradle.api.artifacts.transform.TransformOutputs outputs) { outputs.file(inputArtifact.get().asFile) } +} +abstract class IdentityTwo implements org.gradle.api.artifacts.transform.TransformAction { + @org.gradle.api.artifacts.transform.InputArtifact + abstract Provider getInputArtifact() + void transform(org.gradle.api.artifacts.transform.TransformOutputs outputs) { outputs.file(inputArtifact.get().asFile) } +} +def artifactType = Attribute.of('artifactType', String) +dependencies { + registerTransform(IdentityOne) { from.attribute(artifactType, 'jar'); to.attribute(artifactType, 'foo') } + registerTransform(IdentityTwo) { from.attribute(artifactType, 'jar'); to.attribute(artifactType, 'foo') } +} +configurations { + ambiguous { + attributes { attribute(artifactType, 'foo') } + } +} +dependencies { + ambiguous 'demo:a:1.0' +} diff --git a/src/commands/manifest/scripts/test/gradle-compat/smoke-test-failures.sh b/src/commands/manifest/scripts/test/gradle-compat/smoke-test-failures.sh new file mode 100644 index 000000000..fc8aa86d0 --- /dev/null +++ b/src/commands/manifest/scripts/test/gradle-compat/smoke-test-failures.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# Run socket-facts.init.gradle against the `failures` project, whose configurations each fail to +# resolve in one way, and assert every failure is reported and no partial or fabricated component +# is emitted: +# - a forced version that doesn't exist is reported as that version, not the one requested; +# - a failed project dependency, and an external module sharing a project's group:name, are reported; +# - an artifact failing dependency verification (Gradle 6.2+) is reported and has no file, a missing +# checksum as non-blocking and a mismatch as blocking, and one whose report names no artifact +# leaves the configuration unscannable but keeps its graph, without files; +# - a module whose artifact can't be selected is reported with no component, with or without +# --with-files; +# - a walk that throws midway leaves no resolution root, only an unscannable configuration; +# - a file that can't be read with an empty message is reported like any other. +# +# Usage: smoke-test-failures.sh /path/to/gradle +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +GRADLE="${1:?usage: smoke-test-failures.sh }" +INIT="$HERE/../../socket-facts.init.gradle" +CLASSIFY="$HERE/../../resolution-report-gradle.mts" +RECORDS_PARSER="$HERE/../../records.mts" +PROJECT="$HERE/failures" +RECORDS="$PROJECT/records.tsv" +# shellcheck source=SCRIPTDIR/../compat-cache.sh +. "$HERE/../compat-cache.sh" +GUH="$SOCKET_COMPAT_CACHE/gradle-home" + +bash "$HERE/../make-stub-repo.sh" "$PROJECT/localrepo" \ + 'demo:a:1.0+demo:c:1.0' 'demo:c:1.0' 'demo:boom:1.0+demo:c:1.0' 'demo:quiet:1.0' 'demo:tampered:1.0' 'demo:unlisted:1.0' + +run() { + rm -rf "$GUH" "$RECORDS" "$PROJECT/.gradle" "$PROJECT/build" "$PROJECT/lib/build" "$PROJECT/app/build" + ( cd "$PROJECT" && "$GRADLE" --no-daemon --offline -g "$GUH" \ + --init-script "$INIT" -Porg.gradle.dependency.verification.console=verbose \ + -Psocket.recordsFile="$RECORDS" "$@" socketFacts -q ) > /dev/null +} + +# The category the Socket CLI gives each failure record, as "configcoordcategory" lines. +categories() { + node --input-type=module -e ' + import { readFileSync } from "node:fs" + import { parseRecords } from "'"$RECORDS_PARSER"'" + import { classifyGradleFailure } from "'"$CLASSIFY"'" + for (const f of parseRecords(readFileSync(process.argv[1], "utf8")).failures) { + console.log([f.config, f.coord, classifyGradleFailure(f.detail)].join("\t")) + }' "$RECORDS" +} + +echo "+ $("$GRADLE" --version 2>/dev/null | sed -n 's/^Gradle //p' | head -1) (failures)" +for files in true false; do + run -Psocket.withFiles="$files" + categories > "$PROJECT/categories.tsv" + python3 - "$RECORDS" "$files" "$PROJECT/categories.tsv" <<'PY' +import sys +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +with_files = sys.argv[2] == 'true' +categories = {} +for l in open(sys.argv[3]): + if l.strip(): + config, coord, category = l.rstrip('\n').split('\t') + categories.setdefault((config, coord), set()).add(category) +version = next(r[2] for r in rows if r[0] == 'meta') +modern = tuple(int(p) for p in version.split('.')[:2]) >= (6, 2) +roots = {r[1]: r[3] for r in rows if r[0] == 'root'} +nodes = {(roots[r[1]], r[2]): r for r in rows if r[0] == 'node'} +files = {(roots[r[1]], r[2]) for r in rows if r[0] == 'file'} +unfetched = {(roots[r[1]], r[2]) for r in rows if r[0] == 'unfetched'} +failures = {(r[3], r[1]) for r in rows if r[0] == 'failure'} +unscannable = {r[1] for r in rows if r[0] == 'unscannable'} +errors = [] + +def expect(cond, msg): + if not cond: + errors.append(msg) + +expect(('forced', 'demo:c:9.9') in failures, f"forced: missing failure for the forced demo:c:9.9: {sorted(failures)}") +expect(('forced', 'demo:c:1.0') not in failures, "forced: failure names the requested demo:c:1.0, not the attempted 9.9") +# Old Gradle fails the whole configuration instead of the one dependency. +expect(('projectFail', 'demo:lib:3.0') in failures or 'projectFail' in unscannable, + f"projectFail: failed project dependency not reported: {sorted(failures)}") +expect(('shadowing', 'demo:lib:7.7') in failures, f"shadowing: external module sharing :lib's group:name not reported: {sorted(failures)}") +expect(not [n for n in nodes.values() if not n[6]], f"ext-less component emitted: {[n[2] for n in nodes.values() if not n[6]]}") +expect('walkAborted' in unscannable, f"walkAborted: walk that threw not reported unscannable: {sorted(unscannable)}") +expect('walkAborted' not in roots.values(), "walkAborted: partial tree of a walk that threw was emitted") +if modern: + # Only some Gradle versions select legacy-API artifacts through the transforms; the others return + # the untransformed jar. Either way, a module is emitted with an artifact or reported, never both. + ambiguous = [k for k in nodes if k[0] == 'ambiguous'] + reported = ('ambiguous', 'demo:a:1.0') in failures + expect(reported != bool(ambiguous), f"ambiguous: components {ambiguous}, reported={reported}") + if reported: + expect(categories[('ambiguous', 'demo:a:1.0')] == {'variant-ambiguity'}, + f"ambiguous: classified {categories[('ambiguous', 'demo:a:1.0')]}, not variant-ambiguity") + +tampered = ('verified', 'demo:tampered:jar:1.0') +unlisted = ('verified', 'demo:unlisted:jar:1.0') +if with_files: + expect(('quiet', 'demo:quiet:1.0') in failures, f"quiet: unreadable file not reported: {sorted(failures)}") + expect(('quiet', 'demo:quiet:jar:1.0') in unfetched, f"quiet: unreadable file not marked unfetched: {sorted(unfetched)}") + expect('quiet' not in unscannable, "quiet: unreadable file sank the whole configuration") + expect(('verified', 'demo:a:jar:1.0') in files, f"verified: trusted demo:a has no file: {sorted(files)}") + if modern: + expect(('verified', 'demo:tampered:1.0') in failures, f"verified: verification failure not reported: {sorted(failures)}") + expect(tampered in unfetched, f"verified: tampered jar not marked unfetched: {sorted(unfetched)}") + expect(tampered not in files, "verified: tampered jar emitted as a file") + expect(categories.get(('verified', 'demo:tampered:1.0')) == {'other'}, + f"verified: checksum mismatch not blocking: {categories.get(('verified', 'demo:tampered:1.0'))}") + expect(unlisted in unfetched and unlisted not in files, "verified: unlisted jar not marked unfetched") + expect(categories.get(('verified', 'demo:unlisted:1.0')) == {'checksum-missing'}, + f"verified: missing checksum not non-blocking: {categories.get(('verified', 'demo:unlisted:1.0'))}") + expect('unparsed' in unscannable, f"unparsed: unparseable verification failure not reported: {sorted(unscannable)}") + expect(('unparsed', 'demo:a:jar:1.0') in nodes, "unparsed: graph dropped for an unparseable verification failure") + expect(('unparsed', 'demo:a:jar:1.0') in unfetched and ('unparsed', 'demo:a:jar:1.0') not in files, + "unparsed: files kept despite an unparseable verification failure") + else: + expect(tampered in files, "verified: no verification before Gradle 6.2, so the jar must materialize") +else: + expect(not files and not unfetched, "file or unfetched record emitted without --with-files") + +if errors: + print("FAIL:") + for e in errors: print(" -", e) + sys.exit(1) +print(f"PASS (failures, withFiles={sys.argv[2]}): every failure reported, no partial or fabricated component") +PY + python3 "$HERE/../assert-reachability.py" "$RECORDS" +done diff --git a/src/commands/manifest/scripts/test/gradle-compat/smoke-test.sh b/src/commands/manifest/scripts/test/gradle-compat/smoke-test.sh index 7ec20c7e1..9fb864b51 100755 --- a/src/commands/manifest/scripts/test/gradle-compat/smoke-test.sh +++ b/src/commands/manifest/scripts/test/gradle-compat/smoke-test.sh @@ -9,7 +9,8 @@ # - every resolved dependency gets an on-disk jar `file` record under -Psocket.withFiles (incl. the # production dep reached via the legacy `compile` config on old Gradle); # - every component is reachable from a direct dependency of its own root (no orphaned components); -# - -Psocket.populateFilesFor scopes materialization to a single GAV. +# - -Psocket.populateFilesFor scopes materialization to a single GAV; +# - a jar that cannot be fetched is reported as a failure, and the other jars still materialize. # # Usage: smoke-test.sh /path/to/gradle set -euo pipefail @@ -23,11 +24,15 @@ RECORDS="$PROJECT/records.tsv" GUH="$SOCKET_COMPAT_CACHE/gradle-home" # isolated Gradle user home -> hermetic, no global init scripts bash "$HERE/../make-stub-repo.sh" "$PROJECT/localrepo" 'demo.lib:foo:1.0' 'demo.test:bar:1.0' -rm -rf "$GUH" "$RECORDS" "$PROJECT/.gradle" "$PROJECT/build" + +run() { + rm -rf "$GUH" "$RECORDS" "$PROJECT/.gradle" "$PROJECT/build" + ( cd "$PROJECT" && "$GRADLE" --no-daemon --offline -g "$GUH" \ + --init-script "$INIT" -Psocket.withFiles=true -Psocket.recordsFile="$RECORDS" "$@" socketFacts -q ) +} echo "+ $("$GRADLE" --version 2>/dev/null | sed -n 's/^Gradle //p' | head -1)" -( cd "$PROJECT" && "$GRADLE" --no-daemon --offline -g "$GUH" \ - --init-script "$INIT" -Psocket.withFiles=true -Psocket.recordsFile="$RECORDS" socketFacts -q ) +run python3 - "$RECORDS" 'full' <<'PY' import sys @@ -72,9 +77,7 @@ python3 "$HERE/../assert-reachability.py" "$RECORDS" # Second run: scope --with-files to a single GAV and assert ONLY that artifact is materialized. SCOPE="$HERE/.populate-for.txt" printf 'demo.lib:foo:1.0\n' > "$SCOPE" -rm -rf "$GUH" "$RECORDS" "$PROJECT/.gradle" "$PROJECT/build" -( cd "$PROJECT" && "$GRADLE" --no-daemon --offline -g "$GUH" \ - --init-script "$INIT" -Psocket.withFiles=true -Psocket.populateFilesFor="$SCOPE" -Psocket.recordsFile="$RECORDS" socketFacts -q ) +run -Psocket.populateFilesFor="$SCOPE" rm -f "$SCOPE" python3 - "$RECORDS" 'scoped' <<'PY' @@ -96,3 +99,28 @@ print("PASS (populateFilesFor scoping): foo materialized, bar skipped") PY python3 "$HERE/../assert-reachability.py" "$RECORDS" + +# Third run: bar's jar is gone from the repository, so it must be reported rather than emitted as +# resolved with no file. +find "$PROJECT/localrepo/demo/test/bar" -name '*.jar' -delete +run > /dev/null + +python3 - "$RECORDS" <<'PY' +import sys +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +files = {r[2] for r in rows if r[0] == 'file'} +failures = {r[1] for r in rows if r[0] == 'failure'} +unfetched = {r[2] for r in rows if r[0] == 'unfetched'} +errors = [] +if 'demo.lib:foo:jar:1.0' not in files: + errors.append(f"missing jar run: foo must still be materialized: {sorted(files)}") +if 'demo.test:bar:1.0' not in failures: + errors.append(f"missing jar run: bar's jar cannot be fetched, so it must be reported: {sorted(failures)}") +if unfetched != {'demo.test:bar:jar:1.0'}: + errors.append(f"missing jar run: only bar's jar is unfetched: {sorted(unfetched)}") +if errors: + print("FAIL:") + for e in errors: print(" -", e) + sys.exit(1) +print("PASS (missing jar): bar reported, foo materialized") +PY diff --git a/src/commands/manifest/scripts/test/run-compat.sh b/src/commands/manifest/scripts/test/run-compat.sh index 989be5e9e..c28f2b23e 100755 --- a/src/commands/manifest/scripts/test/run-compat.sh +++ b/src/commands/manifest/scripts/test/run-compat.sh @@ -23,7 +23,8 @@ TOOL="${1:-all}" CACHE="$SOCKET_COMPAT_CACHE" # Same matrix as the former CI workflow. Rows: " [scala]". -GRADLE_MATRIX=("1.12 8" "2.14.1 8" "3.3 8" "8.10.2 17" "9.2.1 21") +# 6.2 is the oldest Gradle with dependency verification. +GRADLE_MATRIX=("1.12 8" "2.14.1 8" "3.3 8" "6.2.2 8" "8.10.2 17" "9.2.1 21") # The Maven rows span the range the extension claims to support (see maven-extension/pom.xml): # 3.2.5 is the oldest, 4.x the newest. Resolution goes through Maven's own # ProjectDependenciesResolver, so both ends are load-bearing, not decorative. @@ -62,6 +63,7 @@ run_gradle() { fi bash "$HERE/gradle-compat/smoke-test.sh" "$dir/bin/gradle" bash "$HERE/gradle-compat/smoke-test-workspaces.sh" "$dir/bin/gradle" + bash "$HERE/gradle-compat/smoke-test-failures.sh" "$dir/bin/gradle" done }