diff --git a/CHANGELOG.md b/CHANGELOG.md index df51df5c3..7939ddbd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). - `socket scan create --reach` keeps the private libraries and build modules of Socket facts files in the stored SBOM, with their dependencies under them. - JVM dependencies whose jars cannot be fetched, or are gone from disk, no longer reach reachability analysis looking resolved without code. - One unresolvable Maven dependency no longer strips the jar paths from every other dependency of its build, which made reachability analysis call their vulnerabilities unreachable. Jars that cannot be fetched are reported with their real cause, Maven 4 included. +- Maven dependencies with missing or invalid POMs are reported in every module that uses them, `system`-scope jars reach reachability analysis, and a failing dependency no longer fails scopes that `excludeConfigs` leaves out or the build when it lost a version conflict. ## [1.10.0](https://github.com/SocketDev/socket-cli/releases/tag/v1.10.0) - 2026-10-22 diff --git a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsEventSpy.java b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsEventSpy.java new file mode 100644 index 000000000..089964c75 --- /dev/null +++ b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsEventSpy.java @@ -0,0 +1,31 @@ +package tech.coana.ext; + +import org.apache.maven.eventspy.AbstractEventSpy; +import org.eclipse.aether.RepositoryEvent; +import tech.coana.socket.CollectionObserver; + +import javax.inject.Named; +import javax.inject.Singleton; + +/** + * Relays repository events to the facts engine while it collects. Every Maven chains its event + * spies into the session's repository listener, which an extension cannot replace: the session is + * read-only since Maven 3.1 and immutable in Maven 4. + */ +@Named("coana-facts-events") +@Singleton +public class CoanaFactsEventSpy extends AbstractEventSpy { + + // Static: the container may hand the dispatcher and the participant different instances. + private static volatile CollectionObserver observer; + + static void observe(CollectionObserver o) { + observer = o; + } + + @Override + public void onEvent(Object event) { + CollectionObserver o = observer; + if (o != null && event instanceof RepositoryEvent) o.onEvent((RepositoryEvent) event); + } +} diff --git a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsLifecycleParticipant.java b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsLifecycleParticipant.java index c8fc4a0f1..3ff79ec57 100644 --- a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsLifecycleParticipant.java +++ b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/ext/CoanaFactsLifecycleParticipant.java @@ -8,6 +8,7 @@ import org.eclipse.aether.RepositorySystem; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import tech.coana.socket.CollectionObserver; import tech.coana.socket.SocketFactsRecordsEngine; import javax.inject.Inject; @@ -59,11 +60,16 @@ public void afterSessionEnd(MavenSession session) throws MavenExecutionException opts.excludeConfigs = opt(session, "socket.excludeConfigs"); opts.excludePaths = opt(session, "socket.excludePaths"); File rootDir = new File(session.getExecutionRootDirectory()); + CollectionObserver observer = new CollectionObserver(); + CoanaFactsEventSpy.observe(observer); try { - new SocketFactsRecordsEngine(dependenciesResolver, repositorySystem, runtimeInformation.getMavenVersion(), LOG) + new SocketFactsRecordsEngine( + dependenciesResolver, repositorySystem, runtimeInformation.getMavenVersion(), observer, LOG) .run(session, session.getProjects(), rootDir, opts); } catch (IOException exception) { throw new MavenExecutionException("Cannot write socket facts records", exception); + } finally { + CoanaFactsEventSpy.observe(null); } } diff --git a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/CollectionObserver.java b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/CollectionObserver.java new file mode 100644 index 000000000..f56dcd067 --- /dev/null +++ b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/CollectionObserver.java @@ -0,0 +1,125 @@ +package tech.coana.socket; + +import org.eclipse.aether.RepositoryEvent; +import org.eclipse.aether.RequestTrace; +import org.eclipse.aether.collection.CollectStepData; +import org.eclipse.aether.graph.Dependency; +import org.eclipse.aether.graph.DependencyNode; + +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * What the repository events of dependency collection reveal beyond its result: the POMs Maven's + * default {@code ArtifactDescriptorPolicy} tolerated as missing or invalid (the node stays, its + * dependencies are lost, and no collection error is raised), build-wide, and the graph position of + * each collect step of the current module. + */ +public final class CollectionObserver { + + // CollectStepData is maven-resolver 1.8.1+ (Maven 3.9+); older resolvers trace no collect step. + private static final boolean COLLECT_STEPS = classExists("org.eclipse.aether.collection.CollectStepData"); + + // The resolver caches a tolerated POM build-wide, so only the first module to reach one sees its + // event; that module's detail is kept. + private final Map incompleteDescriptors = new LinkedHashMap<>(); + private final Set incompleteInModule = new HashSet<>(); + private final Map stepParents = new IdentityHashMap<>(); + + public synchronized void onEvent(RepositoryEvent event) { + if (event.getType() == RepositoryEvent.EventType.ARTIFACT_DESCRIPTOR_MISSING) { + incomplete(event, "is missing, no dependency information available"); + } else if (event.getType() == RepositoryEvent.EventType.ARTIFACT_DESCRIPTOR_INVALID) { + incomplete(event, "is invalid, transitive dependencies (if any) will not be available"); + } + // The depth-first collector hands each step its live node stack, so the parent is only right now. + if (COLLECT_STEPS) Steps.recordParent(event.getTrace(), stepParents); + } + + private void incomplete(RepositoryEvent event, String problem) { + String gav = SocketFactsRecordsEngine.gav(event.getArtifact()); + if (incompleteDescriptors.containsKey(gav) && !incompleteInModule.contains(gav)) return; + incompleteInModule.add(gav); + incompleteDescriptors.put( + gav, + "The POM for " + event.getArtifact() + " " + problem + ": " + + SocketFactsRecordsEngine.rootMessage(event.getException())); + } + + /** Starts the next module's collection. */ + synchronized void reset() { + incompleteInModule.clear(); + stepParents.clear(); + } + + /** GAV to failure detail of each POM the collection tolerated as missing or invalid. */ + synchronized Map incompleteDescriptors() { + return new LinkedHashMap<>(incompleteDescriptors); + } + + /** The node whose children the collect step behind {@code trace} was adding, if it was observed. */ + synchronized DependencyNode parentOf(RequestTrace trace) { + return COLLECT_STEPS ? Steps.parentOf(trace, stepParents) : null; + } + + /** The dependency, before management, that the collect step behind {@code trace} was adding. */ + synchronized Dependency dependencyOf(RequestTrace trace) { + return COLLECT_STEPS ? Steps.dependencyOf(trace) : null; + } + + private static boolean classExists(String name) { + try { + Class.forName(name, false, RequestTrace.class.getClassLoader()); + return true; + } catch (ClassNotFoundException | LinkageError e) { + return false; + } + } + + // Only loaded when CollectStepData exists. + private static final class Steps { + static void recordParent(RequestTrace trace, Map parents) { + RequestTrace at = stepTrace(trace); + if (at == null || parents.containsKey(at.getData())) return; + List path = ((CollectStepData) at.getData()).getPath(); + if (!path.isEmpty()) parents.put(at.getData(), path.get(path.size() - 1)); + } + + static DependencyNode parentOf(RequestTrace trace, Map parents) { + RequestTrace at = stepTrace(trace); + if (at == null) return null; + DependencyNode observed = parents.get(at.getData()); + if (observed != null) return observed; + // The depth-first collector hands every step its one live node stack, so an unobserved step's path + // is only trusted when it is checkably its own: the breadth-first collector nests each step's + // trace in its parent's step, whose dependency is then the path's last node. + RequestTrace enclosing = stepTrace(at.getParent()); + List path = ((CollectStepData) at.getData()).getPath(); + if (enclosing == null || path.isEmpty()) return null; + DependencyNode last = path.get(path.size() - 1); + Dependency parent = ((CollectStepData) enclosing.getData()).getNode(); + if (last.getDependency() == null + || !last.getDependency().getArtifact().getGroupId().equals(parent.getArtifact().getGroupId()) + || !last.getDependency().getArtifact().getArtifactId().equals(parent.getArtifact().getArtifactId())) { + return null; + } + return last; + } + + static Dependency dependencyOf(RequestTrace trace) { + RequestTrace at = stepTrace(trace); + return at == null ? null : ((CollectStepData) at.getData()).getNode(); + } + + private static RequestTrace stepTrace(RequestTrace trace) { + for (RequestTrace t = trace; t != null; t = t.getParent()) { + if (t.getData() instanceof CollectStepData) return t; + } + return null; + } + } +} diff --git a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java index 6e29aac61..bffac1783 100644 --- a/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java +++ b/src/commands/manifest/scripts/maven-extension/src/main/java/tech/coana/socket/SocketFactsRecordsEngine.java @@ -1,5 +1,6 @@ package tech.coana.socket; +import org.apache.maven.RepositoryUtils; import org.apache.maven.execution.MavenSession; import org.apache.maven.model.Resource; import org.apache.maven.project.DefaultDependencyResolutionRequest; @@ -9,13 +10,16 @@ import org.apache.maven.project.MavenProject; import org.apache.maven.project.ProjectDependenciesResolver; import org.eclipse.aether.RepositorySystem; +import org.eclipse.aether.RequestTrace; import org.eclipse.aether.artifact.Artifact; import org.eclipse.aether.graph.Dependency; import org.eclipse.aether.graph.DependencyFilter; import org.eclipse.aether.graph.DependencyNode; import org.eclipse.aether.resolution.ArtifactDescriptorException; +import org.eclipse.aether.resolution.ArtifactDescriptorRequest; import org.eclipse.aether.resolution.ArtifactResult; import org.eclipse.aether.resolution.DependencyRequest; +import org.eclipse.aether.resolution.VersionRangeRequest; import org.eclipse.aether.resolution.VersionRangeResolutionException; import org.slf4j.Logger; @@ -77,15 +81,28 @@ public static final class Options { private final ProjectDependenciesResolver dependenciesResolver; private final RepositorySystem repositorySystem; private final String mavenVersion; + private final CollectionObserver observer; private final Logger log; + // Maven 3.2.5's collector (aether 1.0.0) leaves a dependency whose POM failed out of the graph; from + // aether 1.0.2 (Maven 3.3) on it stays, minus its children. + private final boolean collectorDropsFailedNodes; + // Maven 3's JavaScopeDeriver and ClassicDependencyManager, which absentScope models. + private final boolean maven3ScopeRules; + // The resolver caches a failed POM build-wide, keyed by artifact alone, so only the first module to + // reach one sees the failure; every later module gets the node without children and no error. + // Keyed by GAV. + private final Map brokenDescriptors = new LinkedHashMap<>(); public SocketFactsRecordsEngine( ProjectDependenciesResolver dependenciesResolver, RepositorySystem repositorySystem, String mavenVersion, - Logger log) { + CollectionObserver observer, Logger log) { this.dependenciesResolver = dependenciesResolver; this.repositorySystem = repositorySystem; this.mavenVersion = mavenVersion; + this.observer = observer; this.log = log; + this.collectorDropsFailedNodes = mavenVersion != null && mavenVersion.startsWith("3.2."); + this.maven3ScopeRules = mavenVersion == null || mavenVersion.startsWith("3."); } public void run(MavenSession session, List reactor, File rootDir, Options opts) @@ -173,6 +190,7 @@ private void collectModule( DependencyResolutionResult result; DependencyResolutionException thrown = null; DependencyFilter resolutionFilter = materializationFilter(opts, passingScopes, reactorGavs, populateGavs); + observer.reset(); try { DependencyResolutionRequest request = new DefaultDependencyResolutionRequest(module, session.getRepositorySession()); @@ -191,27 +209,68 @@ private void collectModule( } boolean reported = false; DependencyNode graph = result.getDependencyGraph(); - // Dependencies whose own resolution failed are reported, not emitted, as Gradle and sbt do. - Set unresolved = new HashSet<>(); - Map> scopesByGav = null; + List collectionFailures = new ArrayList<>(); for (Exception e : result.getCollectionErrors()) { - String failed = failedArtifact(e); + FailedRequest failed = failedRequest(e); if (failed == null) { unscannable.add(new Unscannable(rootMessage(e), moduleCoord)); } else { - // Maven keeps a dependency whose POM failed in the graph, minus its children (Maven 3.2 drops it), so - // the failure belongs to the scopes it has there. One absent from the graph has none. - if (scopesByGav == null) scopesByGav = scopesByGav(graph); - Set scopes = scopesByGav.getOrDefault(failed, Collections.emptySet()); - if (scopes.isEmpty()) failures.add(new Failure(failed, rootMessage(e), "", moduleCoord)); - for (String scope : scopes) { - if (passingScopes.contains(scope)) failures.add(new Failure(failed, rootMessage(e), scope, moduleCoord)); + collectionFailures.add(failed); + if (failed.descriptor) { + brokenDescriptors.put(gav(failed.artifact), failed); } - unresolved.add(failed); } log.warn("[socket-facts] could not build dependency graph for " + moduleCoord + ": " + rootMessage(e)); reported = true; } + Map incompleteDescriptors = observer.incompleteDescriptors(); + // Dependencies whose own resolution failed are reported, not emitted, as Gradle and sbt do. One + // whose POM Maven tolerated as missing or invalid is still emitted: its own artifact may be fine. + Set unresolved = new HashSet<>(); + Map> scopesByGav = + brokenDescriptors.isEmpty() && incompleteDescriptors.isEmpty() && collectionFailures.isEmpty() + ? Collections.>emptyMap() + : scopesByGav(graph); + for (Map.Entry e : brokenDescriptors.entrySet()) { + Set scopes = scopesByGav.get(e.getKey()); + if (scopes == null) continue; + unresolved.add(e.getKey()); + reported |= addFailures(failures, e.getKey(), e.getValue().detail, scopes, passingScopes, moduleCoord); + } + for (Map.Entry e : incompleteDescriptors.entrySet()) { + // A strict descriptor policy also throws for it: already reported above. + if (brokenDescriptors.containsKey(e.getKey())) continue; + Set scopes = scopesByGav.get(e.getKey()); + if (scopes != null) reported |= addFailures(failures, e.getKey(), e.getValue(), scopes, passingScopes, moduleCoord); + } + Set placed = new HashSet<>(); + Set onClasspath = null; + for (FailedRequest failed : collectionFailures) { + String coord = gav(failed.artifact); + if (scopesByGav.containsKey(coord)) continue; + if (onClasspath == null) onClasspath = onClasspath(graph); + String scope = absentScope(session, module, failed, onClasspath); + placed.add(coord); + if (scope == null) { + log.debug("[socket-facts] " + coord + " failed but sits on no classpath of " + moduleCoord); + } else if (scope.isEmpty() || passingScopes.contains(scope)) { + failures.add(new Failure(coord, failed.detail, scope, moduleCoord)); + } + } + if (collectorDropsFailedNodes) { + // An earlier module's cached failure leaves no trace in this graph; only a direct dependency + // can still be placed. One still among the direct children resolved (a ranged declaration to + // another version), since a direct dependency never loses a conflict. + for (Map.Entry e : brokenDescriptors.entrySet()) { + if (scopesByGav.containsKey(e.getKey()) || placed.contains(e.getKey())) continue; + if (isDirectChild(graph, e.getValue().artifact)) continue; + String scope = declaredScope(session, module, e.getValue().artifact); + if (scope != null && passingScopes.contains(scope)) { + failures.add(new Failure(e.getKey(), e.getValue().detail, scope, moduleCoord)); + reported = true; + } + } + } // Maven throws on a collection error before it fetches any artifact, which would leave every // healthy node of the partial graph without its file. Set unfetched = new HashSet<>(); @@ -278,10 +337,7 @@ public boolean accept(DependencyNode node, List parents) { Dependency dep = node == null ? null : node.getDependency(); Artifact artifact = dep == null ? null : dep.getArtifact(); if (artifact == null) return false; - String scope = scopeOf(dep); - // A system-scope artifact carries its systemPath on the model and has no repository to be - // fetched from; visit() reads the file straight off the node instead. - if ("system".equals(scope) || !passingScopes.contains(scope)) return false; + if (!passingScopes.contains(scopeOf(dep))) return false; String gav = gav(artifact); if (reactorGavs.contains(gav)) return false; return populateGavs == null || populateGavs.contains(gav); @@ -391,7 +447,7 @@ private static String externalId(Artifact artifact) { artifact.getBaseVersion()); } - private static String gav(Artifact artifact) { + static String gav(Artifact artifact) { return artifact.getGroupId() + ":" + artifact.getArtifactId() + ":" + artifact.getBaseVersion(); } @@ -576,7 +632,7 @@ private void write(String recordsFile, List lines) throws IOException { log.info("[socket-facts] records written to: " + out.getAbsolutePath()); } - private static String rootMessage(Throwable t) { + static String rootMessage(Throwable t) { Throwable cur = t; String msg = null; int guard = 0; @@ -607,29 +663,162 @@ private static Map> scopesByGav(DependencyNode root) { return scopes; } - // The artifact a collection error is about, when the resolver names one. - private static String failedArtifact(Throwable t) { - Throwable cur = t; + // The request a collection error is about, when the resolver names one. + private static FailedRequest failedRequest(Exception e) { + Throwable cur = e; int guard = 0; while (cur != null && guard++ < 12) { - Artifact artifact = null; if (cur instanceof ArtifactDescriptorException) { - ArtifactDescriptorException e = (ArtifactDescriptorException) cur; - if (e.getResult() != null) artifact = e.getResult().getRequest().getArtifact(); + ArtifactDescriptorException d = (ArtifactDescriptorException) cur; + if (d.getResult() != null) { + ArtifactDescriptorRequest request = d.getResult().getRequest(); + return new FailedRequest(request.getArtifact(), request.getTrace(), true, rootMessage(e)); + } } else if (cur instanceof VersionRangeResolutionException) { - VersionRangeResolutionException e = (VersionRangeResolutionException) cur; - if (e.getResult() != null) artifact = e.getResult().getRequest().getArtifact(); + VersionRangeResolutionException r = (VersionRangeResolutionException) cur; + if (r.getResult() != null) { + VersionRangeRequest request = r.getResult().getRequest(); + return new FailedRequest(request.getArtifact(), request.getTrace(), false, rootMessage(e)); + } } - if (artifact != null) return gav(artifact); cur = cur.getCause(); } return null; } + private static boolean addFailures( + Set failures, String coord, String detail, Set scopes, Set passingScopes, + String moduleCoord) { + boolean added = false; + for (String scope : scopes) { + if (passingScopes.contains(scope)) added |= failures.add(new Failure(coord, detail, scope, moduleCoord)); + } + return added; + } + + // Null: on no classpath (from Maven 3.3 the graph keeps a failed POM's node, so an absent one lost a + // conflict). Empty: position unknown (no collect step before 3.9; Maven 4's scope rules not modelled). + private String absentScope( + MavenSession session, MavenProject module, FailedRequest failed, Set onClasspath) { + if (failed.descriptor && !collectorDropsFailedNodes) return null; + DependencyNode parent = observer.parentOf(failed.trace); + Dependency dependency = observer.dependencyOf(failed.trace); + if (parent != null && dependency != null) { + if (parent.getDependency() == null) return scopeOf(dependency); + DependencyNode placed = classpathInstance(onClasspath, parent); + if (placed == null) return null; + if (!maven3ScopeRules) return ""; + // ConflictResolver keeps a managed scope as it is, without deriving it. + String managed = managedScope(session, module, dependency.getArtifact()); + return managed != null ? managed : derivedScope(placed.getDependency().getScope(), scopeOf(dependency)); + } + String declared = declaredScope(session, module, failed.artifact); + return declared != null ? declared : ""; + } + + // ClassicDependencyManager: below the direct dependencies, only the root's dependencyManagement applies. + private static String managedScope(MavenSession session, MavenProject module, Artifact artifact) { + if (module.getDependencyManagement() == null) return null; + org.apache.maven.model.Dependency d = + modelDependency(session, module.getDependencyManagement().getDependencies(), artifact, false); + return d == null || d.getScope() == null || d.getScope().isEmpty() ? null : d.getScope(); + } + + // maven-resolver's JavaScopeDeriver. + private static String derivedScope(String parentScope, String childScope) { + if ("system".equals(childScope) || "test".equals(childScope)) return childScope; + if (parentScope == null || parentScope.isEmpty() || "compile".equals(parentScope)) return childScope; + if ("test".equals(parentScope) || "runtime".equals(parentScope)) return parentScope; + if ("system".equals(parentScope) || "provided".equals(parentScope)) return "provided"; + return "runtime"; + } + + // The module's own declaration of the artifact, with management applied. + private static String declaredScope(MavenSession session, MavenProject module, Artifact artifact) { + org.apache.maven.model.Dependency d = modelDependency(session, module.getDependencies(), artifact, true); + if (d == null) return null; + return d.getScope() == null || d.getScope().isEmpty() ? "compile" : d.getScope(); + } + + // The first of `deps` naming the artifact, compared as the resolver sees it: a `test-jar` type + // becomes classifier `tests`. With `matchVersion`, a ranged version matches whichever it resolved to. + private static org.apache.maven.model.Dependency modelDependency( + MavenSession session, List deps, Artifact artifact, boolean matchVersion) { + for (org.apache.maven.model.Dependency d : deps) { + Artifact candidate = + RepositoryUtils.toDependency(d, session.getRepositorySession().getArtifactTypeRegistry()).getArtifact(); + String version = candidate.getVersion(); + if (candidate.getGroupId().equals(artifact.getGroupId()) + && candidate.getArtifactId().equals(artifact.getArtifactId()) + && candidate.getClassifier().equals(artifact.getClassifier()) + && candidate.getExtension().equals(artifact.getExtension()) + && (!matchVersion + || version.equals(artifact.getVersion()) || version.startsWith("[") || version.startsWith("("))) { + return d; + } + } + return null; + } + + // A direct dependency of `root` naming the artifact, at any version. + private static boolean isDirectChild(DependencyNode root, Artifact artifact) { + if (root == null) return false; + for (DependencyNode child : root.getChildren()) { + Artifact a = child.getArtifact(); + if (a != null + && a.getGroupId().equals(artifact.getGroupId()) + && a.getArtifactId().equals(artifact.getArtifactId()) + && a.getClassifier().equals(artifact.getClassifier()) + && a.getExtension().equals(artifact.getExtension())) { + return true; + } + } + return false; + } + + // Instances with the same artifact and collection context share one cached child list, whose failing + // collect step ran only under the first of them; that one may have lost a conflict. + private static DependencyNode classpathInstance(Set onClasspath, DependencyNode parent) { + if (onClasspath.contains(parent)) return parent; + for (DependencyNode dn : onClasspath) { + if (dn.getChildren() == parent.getChildren()) return dn; + } + return null; + } + + // The final graph's root and every node on its classpath, by identity. + private static Set onClasspath(DependencyNode root) { + Set seen = Collections.newSetFromMap(new IdentityHashMap<>()); + if (root == null) return seen; + Deque todo = new ArrayDeque<>(); + todo.push(root); + while (!todo.isEmpty()) { + DependencyNode dn = todo.pop(); + if (dn.getData().get(NODE_DATA_CONFLICT_WINNER) != null || !seen.add(dn)) continue; + todo.addAll(dn.getChildren()); + } + return seen; + } + + private static final class FailedRequest { + final Artifact artifact; + final RequestTrace trace; + // The artifact's POM could not be read; else no version satisfied its range. + final boolean descriptor; + final String detail; + + FailedRequest(Artifact artifact, RequestTrace trace, boolean descriptor, String detail) { + this.artifact = artifact; + this.trace = trace; + this.descriptor = descriptor; + this.detail = detail; + } + } + private static final class Failure { final String coord; final String detail; - // Empty when the failure belongs to no scope. + // Empty when the failure cannot be placed in a scope. final String config; final String project; diff --git a/src/commands/manifest/scripts/records.mts b/src/commands/manifest/scripts/records.mts index 312a7613e..570d4ad02 100644 --- a/src/commands/manifest/scripts/records.mts +++ b/src/commands/manifest/scripts/records.mts @@ -19,7 +19,7 @@ import type { ResolutionFailure, Unscannable } from './resolution-report.mts' // file rootId coordId path (--with-files only) // unfetched rootId coordId (--with-files only; requested artifact not fetched) // scanned config -// failure coord detail config projectKey (config empty: no configuration) +// failure coord detail config projectKey (config empty: not placeable in one) // unscannable config detail projectKey (config empty: whole project) // // `projectKey` is the tool's unique project identity (Maven: GAV; Gradle: diff --git a/src/commands/manifest/scripts/resolution-report-maven.mts b/src/commands/manifest/scripts/resolution-report-maven.mts index 37cd7bceb..25f3800f5 100644 --- a/src/commands/manifest/scripts/resolution-report-maven.mts +++ b/src/commands/manifest/scripts/resolution-report-maven.mts @@ -19,7 +19,12 @@ export function classifyMavenFailure(detail: string): FailureCategory { t.includes('unauthorized') || t.includes('forbidden') || t.includes('peer not authenticated') || - t.includes('certpathbuilderexception') + t.includes('certpathbuilderexception') || + // Maven 3.8.1+ blocks plain-HTTP repositories behind its + // maven-default-http-blocker mirror. + t.includes('blocked mirror for repositories') || + // Offline (-o): no repository was asked, so the artifact may well exist. + t.includes('in offline mode') ) { return 'repository-or-network' } @@ -28,7 +33,10 @@ export function classifyMavenFailure(detail: string): FailureCategory { t.includes('failure to find') || t.includes('could not resolve') || t.includes('no versions available') || - t.includes('not found') + t.includes('not found') || + // maven-resolver's remote repository filters (prefixes, groupId): the + // repository is known not to serve that path or group. + t.includes('not allowed from') ) { return 'not-found' } @@ -36,7 +44,9 @@ export function classifyMavenFailure(detail: string): FailureCategory { if ( t.includes('failed to read artifact descriptor') || t.includes('invalid pom') || - t.includes('could not parse pom') + t.includes('could not parse pom') || + // Maven's warning for a POM its descriptor policy tolerated as invalid. + t.includes('is invalid, transitive dependencies') ) { return 'config-problem' } diff --git a/src/commands/manifest/scripts/resolution-report-render.test.mts b/src/commands/manifest/scripts/resolution-report-render.test.mts index 6dd05e42a..6049085ac 100644 --- a/src/commands/manifest/scripts/resolution-report-render.test.mts +++ b/src/commands/manifest/scripts/resolution-report-render.test.mts @@ -370,3 +370,42 @@ describe('resolution failure remedies', () => { expect(r.nonBlockingNotice).not.toContain('re-run with --verbose') }) }) + +const mavenReport = (detail: string) => + renderResolutionErrorReport( + [f('com.example:x:jar:1.0', detail, 'compile')], + ['compile'], + 'maven', + ) + +describe('Maven failure classification', () => { + it('classifies a repository blocked by the HTTP blocker as a repository error', () => { + const r = mavenReport( + 'Blocked mirror for repositories: [insecure (http://repo.example.invalid/maven, default, releases+snapshots)]', + ) + expect(r.hasBlockingFailures).toBe(true) + expect(r.summary).toContain('Repository or network error') + }) + + it('classifies offline mode as a repository error, not a miss', () => { + const r = mavenReport( + 'The POM for com.example:x:jar:1.0 is missing, no dependency information available: Cannot access central (https://repo.maven.apache.org/maven2) in offline mode', + ) + expect(r.summary).toContain('Repository or network error') + expect(r.summary).not.toContain('Not found in any repository') + }) + + it('classifies a POM Maven tolerated as invalid as a POM problem', () => { + const r = mavenReport( + 'The POM for com.example:x:jar:1.0 is invalid, transitive dependencies (if any) will not be available: 1 problem was encountered while building the effective model for com.example:x:1.0', + ) + expect(r.summary).toContain('POM/descriptor problem') + }) + + it('classifies a path the repository prefix filter rules out as not found', () => { + const r = mavenReport( + 'prefixes: Path com/example/x/1.0/x-1.0.pom NOT allowed from central', + ) + expect(r.summary).toContain('Not found in any repository') + }) +}) diff --git a/src/commands/manifest/scripts/resolution-report.mts b/src/commands/manifest/scripts/resolution-report.mts index 3a280195a..5bd09cdd8 100644 --- a/src/commands/manifest/scripts/resolution-report.mts +++ b/src/commands/manifest/scripts/resolution-report.mts @@ -3,8 +3,8 @@ export type ResolutionFailure = { // Build tool's own failure message (deepest cause; may be multi-line). // Classified on; first line shown by default, whole thing at --verbose. detail: string - // Empty when the failed module is absent from the tool's resolved graph, so - // no configuration holds it. + // Empty when the failure cannot be placed in a configuration; it then counts + // against every configuration of its project. config: string // projectKey of the subproject being resolved. project: string diff --git a/src/commands/manifest/scripts/test/maven-compat/.gitignore b/src/commands/manifest/scripts/test/maven-compat/.gitignore index 2923ea429..9867fa6a1 100644 --- a/src/commands/manifest/scripts/test/maven-compat/.gitignore +++ b/src/commands/manifest/scripts/test/maven-compat/.gitignore @@ -13,3 +13,6 @@ duplicate-failure/target/ duplicate-failure/*/target/ graph-failure/records.tsv graph-failure/target/ +descriptor-failure/records.tsv +descriptor-failure/target/ +descriptor-failure/*/target/ diff --git a/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/a/pom.xml b/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/a/pom.xml new file mode 100644 index 000000000..769d3d25d --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/a/pom.xml @@ -0,0 +1,107 @@ + + + 4.0.0 + + demo + descriptor-failure + 1.0 + + a + + + + + demo.descfail + managedchild + [7.0,8.0) + compile + + + + + + demo.descfail + broken + 1.0 + + + demo.descfail + rebroken + [2.0,3.0) + + + demo.descfail + nopom + 1.0 + + + demo.descfail + nothing + 1.0 + + + demo.descfail + sys + 1.0 + system + ${descfail.sysDir}/sys.jar + + + demo.descfail + sysmissing + 1.0 + system + ${descfail.sysDir}/missing.jar + + + demo.descfail + testparent + 1.0 + test + + + demo.descfail + conflict + 2.0 + + + demo.descfail + loserparent + 1.0 + + + demo.descfail + viaq + 1.0 + + + demo.descfail + nearb + 1.0 + + + demo.descfail + viax + 1.0 + + + demo.descfail + nearx + 1.0 + + + demo.descfail + managedparent + 1.0 + test + + + diff --git a/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/b/pom.xml b/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/b/pom.xml new file mode 100644 index 000000000..de0d1a467 --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/b/pom.xml @@ -0,0 +1,28 @@ + + + 4.0.0 + + demo + descriptor-failure + 1.0 + + b + + + demo.descfail + ok + 1.0 + + + demo.descfail + broken + 1.0 + test + + + demo.descfail + rebroken + 1.0 + + + diff --git a/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/pom.xml b/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/pom.xml new file mode 100644 index 000000000..b9ced32fc --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/descriptor-failure/pom.xml @@ -0,0 +1,13 @@ + + + 4.0.0 + demo + descriptor-failure + 1.0 + pom + + + b + a + + diff --git a/src/commands/manifest/scripts/test/maven-compat/smoke-test-descriptor-failure.sh b/src/commands/manifest/scripts/test/maven-compat/smoke-test-descriptor-failure.sh new file mode 100755 index 000000000..9eadf9cdb --- /dev/null +++ b/src/commands/manifest/scripts/test/maven-compat/smoke-test-descriptor-failure.sh @@ -0,0 +1,179 @@ +#!/usr/bin/env bash +# Regression guard for failures Maven's resolver hides from a plain resolution result: +# - A POM that failed once is cached build-wide, so a later module gets the dependency without an +# error. It must still fail there, in that module's scope (broken: test in b, compile in a), but +# not where a range resolved to another version (rebroken: 1.0 in b, [2.0,3.0) in a). +# - A missing POM is tolerated by Maven's default descriptor policy, losing the dependency's own +# dependencies. It must fail; the dependency stays emitted with its jar (nopom), and one whose jar +# is missing too gets no file (nothing). +# - A system-scope jar gets its file; a missing systemPath fails, under the full coordinate. +# - A failure absent from the graph fails in the scope it would have had: rangedchild under the +# test-scoped testparent is test; before Maven 3.9 and on Maven 4 the position is unknown (no scope). +# It is placed even when the collector met its parent first where that parent lost a conflict +# (sharedchild), but not where the instance on the classpath excludes it (sharedxchild; before Maven +# 3.9 its position is unknown, so it has no scope), and a scope the module manages is kept, not +# derived (managedchild: compile). +# - A version-conflict loser (conflict 1.0) is on no classpath and must not fail; Maven 3.2 drops +# a failed POM's node, so there it cannot be told apart and has no scope. +# +# Runs offline: the POMs are seeded straight into the local repository. +# +# Usage: smoke-test-descriptor-failure.sh +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +MVN="${1:?usage: smoke-test-descriptor-failure.sh }" +JAR="${2:?usage: smoke-test-descriptor-failure.sh }" +PROJECT="$HERE/descriptor-failure" +RECORDS="$PROJECT/records.tsv" +# shellcheck source=SCRIPTDIR/../compat-cache.sh +. "$HERE/../compat-cache.sh" +M2="$SOCKET_COMPAT_CACHE/m2" +REPO="$M2/demo/descfail" +SYS_DIR="$SOCKET_COMPAT_CACHE/descfail-sys" + +pom() { + # $1 artifactId, $2 version, $3 extra POM body. + mkdir -p "$REPO/$1/$2" + cat > "$REPO/$1/$2/$1-$2.pom" < + 4.0.0 + demo.descfail + $1 + $2 + $3 + +POM +} + +jar_for() { + mkdir -p "$REPO/$1/$2/empty" + ( cd "$REPO/$1/$2" && jar cf "$1-$2.jar" -C empty . && rm -r empty ) +} + +dep() { + echo "demo.descfail$1$2" +} + +rm -rf "$REPO" "$SYS_DIR" +pom ok 1.0 '' +jar_for ok 1.0 +pom broken 1.0 'demo.descfailabsent-parent1.0' +jar_for broken 1.0 +jar_for nopom 1.0 +pom testparent 1.0 "$(dep rangedchild '[5.0,6.0)')" +jar_for testparent 1.0 +pom rebroken 1.0 'demo.descfailabsent-parent1.0' +jar_for rebroken 1.0 +pom rebroken 2.0 '' +jar_for rebroken 2.0 +cat > "$REPO/rebroken/maven-metadata-local.xml" <<'XML' + + demo.descfail + rebroken + 1.02.0 + +XML +pom conflict 2.0 '' +jar_for conflict 2.0 +pom conflict 1.0 'demo.descfailabsent-parent1.0' +pom loserparent 1.0 "$(dep conflict 1.0)" +jar_for loserparent 1.0 +for parent in viaq:q q:shared nearb:shared viax:qx qx:sharedx; do + pom "${parent%%:*}" 1.0 "$(dep "${parent#*:}" 1.0)" + jar_for "${parent%%:*}" 1.0 +done +pom shared 1.0 "$(dep sharedchild '[5.0,6.0)')" +jar_for shared 1.0 +pom sharedx 1.0 "$(dep sharedxchild '[5.0,6.0)')" +jar_for sharedx 1.0 +pom nearx 1.0 'demo.descfailsharedx1.0demo.descfailsharedxchild' +jar_for nearx 1.0 +pom managedparent 1.0 "$(dep managedchild 1.0)" +jar_for managedparent 1.0 +mkdir -p "$SYS_DIR/empty" +( cd "$SYS_DIR" && jar cf sys.jar -C empty . && rm -r empty ) + +run() { + rm -f "$RECORDS" + ( cd "$PROJECT" && "$MVN" --batch-mode -q -o \ + "-Dmaven.ext.class.path=$JAR" \ + -Dcoana.task=socket-facts \ + "-Dmaven.repo.local=$M2" \ + "-Ddescfail.sysDir=$SYS_DIR" \ + "-Dsocket.recordsFile=$RECORDS" \ + -Dsocket.withFiles=true \ + "$@" \ + validate ) +} + +# $1: whether test scope is scanned. +check() { + python3 - "$RECORDS" "$1" "$STEPS" "$KEEPS_FAILED" "$TRACED" <<'PY' +import sys +with_test, steps, keeps_failed, traced = (a == 'true' for a in sys.argv[2:6]) +a, b = 'demo:a:1.0', 'demo:b:1.0' +expected = { + ('demo.descfail:broken:1.0', 'compile', a), + ('demo.descfail:nopom:1.0', 'compile', a), + ('demo.descfail:nothing:1.0', 'compile', a), + ('demo.descfail:nothing:jar:1.0', 'compile', a), + ('demo.descfail:sysmissing:jar:1.0', 'system', a), +} +if not keeps_failed: + expected.add(('demo.descfail:conflict:1.0', '', a)) +expected.add(('demo.descfail:rebroken:1.0', 'compile', b)) +if with_test: + expected.add(('demo.descfail:broken:1.0', 'test', b)) +if with_test or not steps: + expected.add(('demo.descfail:rangedchild:[5.0,6.0)', 'test' if steps else '', a)) +expected.add(('demo.descfail:sharedchild:[5.0,6.0)', 'compile' if steps else '', a)) +expected.add(('demo.descfail:managedchild:[7.0,8.0)', 'compile' if steps else '', a)) +if not traced: + expected.add(('demo.descfail:sharedxchild:[5.0,6.0)', '', a)) +rows = [l.rstrip('\n').split('\t') for l in open(sys.argv[1]) if l.strip()] +failures = [r[1:] for r in rows if r[0] == 'failure'] +actual = {(f[0], f[2], f[3]) for f in failures} +nodes = {r[2] for r in rows if r[0] == 'node'} +files = {r[2] for r in rows if r[0] == 'file'} +unfetched = {r[2] for r in rows if r[0] == 'unfetched'} +unscannable = [r[1:] for r in rows if r[0] == 'unscannable'] +errors = [] +if actual != expected: + errors.append(f"failures differ: missing={sorted(expected - actual)} unexpected={sorted(actual - expected)}") +for f in failures: + if 'One or more requests failed' in f[1]: + errors.append(f"{f[0]} reports the batch wrapper, not its cause: {f[1]!r}") +if any(n.startswith('demo.descfail:broken:') for n in nodes): + errors.append(f"broken's POM failed, so it must not be emitted; nodes={sorted(nodes)}") +for coord in ('demo.descfail:ok:jar:1.0', 'demo.descfail:nopom:jar:1.0', 'demo.descfail:sys:jar:1.0', + 'demo.descfail:rebroken:jar:2.0'): + if coord not in files: + errors.append(f"{coord} must be emitted with its jar; files={sorted(files)}") +if unfetched != {'demo.descfail:nothing:jar:1.0', 'demo.descfail:sysmissing:jar:1.0'}: + errors.append(f"only nothing's and sysmissing's jars are unfetched; unfetched={sorted(unfetched)}") +if any(f.startswith(('demo.descfail:nothing:', 'demo.descfail:sysmissing:')) for f in files): + errors.append(f"a jar that is missing must get no file; files={sorted(files)}") +if unscannable: + errors.append(f"every failure names its artifact, so nothing is unscannable: {unscannable}") +if errors: + print("FAIL:") + for e in errors: + print(" -", e) + sys.exit(1) +print(f"PASS (descriptor failure): {len(actual)} failure(s)") +PY +} + +VERSION="$("$MVN" -v 2>/dev/null | head -1)" +echo "+ $VERSION (descriptor failure)" +case "$VERSION" in + *"Maven 3.2."*) STEPS=false; KEEPS_FAILED=false; TRACED=false ;; + *"Maven 3.9."*) STEPS=true; KEEPS_FAILED=true; TRACED=true ;; + *"Maven 4."*) STEPS=false; KEEPS_FAILED=true; TRACED=true ;; + *) STEPS=false; KEEPS_FAILED=true; TRACED=false ;; +esac + +run +check true +run -Dsocket.excludeConfigs=test +check false diff --git a/src/commands/manifest/scripts/test/maven-compat/smoke-test-graph-failure.sh b/src/commands/manifest/scripts/test/maven-compat/smoke-test-graph-failure.sh index 5eb4c59cc..f9c827402 100755 --- a/src/commands/manifest/scripts/test/maven-compat/smoke-test-graph-failure.sh +++ b/src/commands/manifest/scripts/test/maven-compat/smoke-test-graph-failure.sh @@ -1,11 +1,10 @@ #!/usr/bin/env bash # Regression guard: a dependency that fails while Maven collects the graph must be reported as that -# dependency, in the module it was collected for, and never emitted as a node. Maven keeps a -# dependency whose POM failed in the graph (Maven 3.2's resolver leaves it out instead), so it fails -# in its own scope, and an excluded scope drops it; a dependency the graph lacks (an unsatisfiable -# range) has no scope. The module itself must never be reported as the failure. Under --with-files, -# every healthy dependency must still get its file, and a jar that cannot be fetched is reported under -# its full coordinate, with the cause rather than a wrapper. +# dependency, in the module it was collected for, and never emitted as a node. It fails in its own +# scope, and an excluded scope drops it; one the graph lacks (an unsatisfiable range, or a failed POM +# on Maven 3.2) fails in the scope it is declared with. The module itself must never be reported as +# the failure. Under --with-files, every healthy dependency must still get its file, and a jar that +# cannot be fetched is reported under its full coordinate, with the cause rather than a wrapper. # # Runs offline: the POMs are seeded straight into the local repository, so their missing parent and # the absent versions of ranged can only fail to resolve. @@ -60,15 +59,14 @@ run() { validate ) } -# $1: expected scope per coord (null: excluded) when the failed dependency stays in the graph. +# $1: expected scope per coord (null: excluded). check() { - python3 - "$RECORDS" "$1" "$KEEPS_FAILED" <<'PY' + python3 - "$RECORDS" "$1" <<'PY' import json, sys def read(path): return [l.rstrip('\n').split('\t') for l in open(path) if l.strip()] rows = read(sys.argv[1]) expected = json.loads(sys.argv[2]) -keeps_failed = sys.argv[3] == 'true' emitted = {f"{r[3]}:{r[4]}:{r[5]}" for r in rows if r[0] == 'node'} errors = [] if 'demo.graphfail:ok:1.0' not in emitted: @@ -81,8 +79,6 @@ for coord, scope in expected.items(): found = [f for f in failures if f[0] == coord] if coord in emitted: errors.append(f"{coord} failed to resolve, so it must not be emitted as a node") - if not keeps_failed: - scope = '' if scope is None: if found: errors.append(f"{coord} is in an excluded scope and must not be reported, got {found}") @@ -133,14 +129,10 @@ PY VERSION="$("$MVN" -v 2>/dev/null | head -1)" echo "+ $VERSION (graph failure)" -case "$VERSION" in - *"Maven 3.2."*) KEEPS_FAILED=false ;; - *) KEEPS_FAILED=true ;; -esac run -check '{"demo.graphfail:lib:1.0": "compile", "demo.graphfail:testlib:1.0": "test", "demo.graphfail:ranged:[1.0,2.0)": ""}' +check '{"demo.graphfail:lib:1.0": "compile", "demo.graphfail:testlib:1.0": "test", "demo.graphfail:ranged:[1.0,2.0)": "compile"}' run -Dsocket.excludeConfigs=test -check '{"demo.graphfail:lib:1.0": "compile", "demo.graphfail:testlib:1.0": null, "demo.graphfail:ranged:[1.0,2.0)": ""}' +check '{"demo.graphfail:lib:1.0": "compile", "demo.graphfail:testlib:1.0": null, "demo.graphfail:ranged:[1.0,2.0)": "compile"}' run -Dsocket.withFiles=true -check '{"demo.graphfail:lib:1.0": "compile", "demo.graphfail:testlib:1.0": "test", "demo.graphfail:ranged:[1.0,2.0)": ""}' +check '{"demo.graphfail:lib:1.0": "compile", "demo.graphfail:testlib:1.0": "test", "demo.graphfail:ranged:[1.0,2.0)": "compile"}' check_files diff --git a/src/commands/manifest/scripts/test/run-compat.sh b/src/commands/manifest/scripts/test/run-compat.sh index 475e19988..989be5e9e 100755 --- a/src/commands/manifest/scripts/test/run-compat.sh +++ b/src/commands/manifest/scripts/test/run-compat.sh @@ -89,6 +89,7 @@ run_maven() { bash "$HERE/maven-compat/smoke-test-repo-inheritance.sh" "$dir/bin/mvn" "$jar" bash "$HERE/maven-compat/smoke-test-duplicate-failure.sh" "$dir/bin/mvn" "$jar" bash "$HERE/maven-compat/smoke-test-graph-failure.sh" "$dir/bin/mvn" "$jar" + bash "$HERE/maven-compat/smoke-test-descriptor-failure.sh" "$dir/bin/mvn" "$jar" done }