From 0cb2f824c36aa0671b467ce960629451158243a7 Mon Sep 17 00:00:00 2001 From: Damian Momot Date: Thu, 19 Nov 2026 13:13:50 -0800 Subject: [PATCH] fix(workflow): rerun the model when a failed LlmAgent node retries PiperOrigin-RevId: 1025841981 --- .../google/adk/flows/llmflows/StepResume.java | 57 +++- .../adk/flows/llmflows/StepResumeTest.java | 80 +++++ .../adk/workflow/LlmAgentAsNodeTest.java | 290 ++++++++++++++++++ 3 files changed, 426 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/google/adk/flows/llmflows/StepResume.java b/core/src/main/java/com/google/adk/flows/llmflows/StepResume.java index 190c6a69a..a9dec30cd 100644 --- a/core/src/main/java/com/google/adk/flows/llmflows/StepResume.java +++ b/core/src/main/java/com/google/adk/flows/llmflows/StepResume.java @@ -24,12 +24,15 @@ import com.google.adk.agents.Role; import com.google.adk.events.Event; import com.google.adk.tools.BaseTool; +import com.google.adk.workflow.NodeInfo; +import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.HashMultimap; import com.google.common.collect.HashMultiset; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.common.collect.Multiset; import com.google.common.collect.SetMultimap; +import com.google.common.collect.Sets; import com.google.genai.types.Content; import com.google.genai.types.FunctionCall; import com.google.genai.types.FunctionResponse; @@ -46,7 +49,8 @@ * Decides how a resumable flow's next step continues: carry on to the model, stop because a call is * still unanswered, or run calls a previous run never executed. * - *

Ports Python ADK's resume decision. Of the sub-branch cases Python recognizes, only a user's + *

Ports Python ADK's resume decision, except that a step does not wait for calls that failed + * workflow node attempts left unanswered. Of the sub-branch cases Python recognizes, only a user's * answer on the branch a single-turn agent tool opened is handled, as in ADK Kotlin. */ final class StepResume { @@ -100,6 +104,25 @@ static Decision decide(InvocationContext context, Map tools) { } ImmutableList events = context.events(/* currentInvocation= */ true, /* currentBranch= */ true); + Decision decision = decideFromEvents(context, tools, hasAgentTools, events); + if (decision.action != Action.PAUSE || context.nodePath().isEmpty()) { + return decision; + } + // Exclude failed attempts only when they would make the step pause. + ImmutableList withoutFailed = + withoutFailedAttempts( + context.events(/* currentInvocation= */ true, /* currentBranch= */ false), events); + return withoutFailed.size() == events.size() + ? decision + : decideFromEvents(context, tools, hasAgentTools, withoutFailed); + } + + /** Decides how the next step resumes from the branch's {@code events}. */ + private static Decision decideFromEvents( + InvocationContext context, + Map tools, + boolean hasAgentTools, + List events) { if (hasAgentTools) { Optional subBranchCall = callThatOpenedBranchOfLatestAnswer(context, events); if (subBranchCall.isPresent()) { @@ -126,7 +149,39 @@ static Decision decide(InvocationContext context, Map tools) { return CONTINUE; } + /** + * Returns {@code branchEvents} without the events of failed workflow node attempts. Each node + * failure in {@code invocationEvents} removes the earlier events of that node and the nodes below + * it, because retrying the node runs them again. + */ + @VisibleForTesting + static ImmutableList withoutFailedAttempts( + List invocationEvents, List branchEvents) { + Set failedPaths = new HashSet<>(); + Set failedAttemptEvents = Sets.newIdentityHashSet(); + for (int i = invocationEvents.size() - 1; i >= 0; i--) { + Event event = invocationEvents.get(i); + String path = event.nodeInfo().map(NodeInfo::path).orElse(""); + boolean inFailedAttempt = + failedPaths.stream().anyMatch(failed -> isSameOrBelow(path, failed)); + // An error without calls is a node failure, so error responses calling tools are kept. + boolean nodeFailure = + !path.isEmpty() && event.errorCode().isPresent() && event.functionCalls().isEmpty(); + if (inFailedAttempt) { + failedAttemptEvents.add(event); + } else if (nodeFailure) { + failedPaths.add(path); + } + } + return branchEvents.stream() + .filter(event -> !failedAttemptEvents.contains(event)) + .collect(toImmutableList()); + } + /** Returns whether {@code path} is the node path {@code ancestor} or a path below it. */ + private static boolean isSameOrBelow(String path, String ancestor) { + return path.equals(ancestor) || path.startsWith(ancestor + "/"); + } /** * Returns the agent's call event to replay when a user's answer came back on the {@code diff --git a/core/src/test/java/com/google/adk/flows/llmflows/StepResumeTest.java b/core/src/test/java/com/google/adk/flows/llmflows/StepResumeTest.java index ad009ffc9..0e9c19432 100644 --- a/core/src/test/java/com/google/adk/flows/llmflows/StepResumeTest.java +++ b/core/src/test/java/com/google/adk/flows/llmflows/StepResumeTest.java @@ -26,10 +26,12 @@ import com.google.adk.sessions.InMemorySessionService; import com.google.adk.sessions.Session; import com.google.adk.tools.BaseTool; +import com.google.adk.workflow.NodeInfo; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.genai.types.Content; +import com.google.genai.types.FinishReason; import com.google.genai.types.FunctionCall; import com.google.genai.types.FunctionResponse; import com.google.genai.types.Part; @@ -228,6 +230,80 @@ public void decide_answerOnTheBranchOfACallAlreadyAnswered_continues() { assertThat(decision.action).isEqualTo(StepResume.Action.CONTINUE); } + @Test + public void withoutFailedAttempts_failureOfTheNode_dropsItsEarlierEvents() { + Event failedAttempt = nodeEvent("wf/worker", callPart("x", "c1")).build(); + Event failure = + nodeEvent("wf/worker").errorCode(new FinishReason("IllegalStateException")).build(); + Event retry = nodeEvent("wf/worker", Part.fromText("retrying")).build(); + ImmutableList history = ImmutableList.of(failedAttempt, failure, retry); + + ImmutableList events = StepResume.withoutFailedAttempts(history, history); + + assertThat(events).containsExactly(failure, retry).inOrder(); + } + + @Test + public void withoutFailedAttempts_modelErrorResponseWithCalls_keepsEveryEvent() { + ImmutableList history = + ImmutableList.of( + nodeEvent("wf/worker", Part.fromText("calling")).build(), + nodeEvent("wf/worker", callPart("y", "c2")) + .errorCode(new FinishReason(FinishReason.Known.MAX_TOKENS)) + .build()); + + ImmutableList events = StepResume.withoutFailedAttempts(history, history); + + assertThat(events).isEqualTo(history); + } + + @Test + public void withoutFailedAttempts_failureOfAnotherNode_dropsEarlierEventsOfThatNodeAndBelow() { + Event ownCall = nodeEvent("wf/worker", callPart("x", "c1")).build(); + Event otherCall = nodeEvent("wf/other", callPart("y", "c2")).build(); + Event innerCall = nodeEvent("wf/other/inner", callPart("z", "c3")).build(); + Event otherFailure = + nodeEvent("wf/other").errorCode(new FinishReason("IllegalStateException")).build(); + Event ownText = nodeEvent("wf/worker", Part.fromText("still waiting")).build(); + ImmutableList history = + ImmutableList.of(ownCall, otherCall, innerCall, otherFailure, ownText); + + ImmutableList events = StepResume.withoutFailedAttempts(history, history); + + assertThat(events).containsExactly(ownCall, otherFailure, ownText).inOrder(); + } + + @Test + public void withoutFailedAttempts_failureOfAnEnclosingNodeOnAnotherBranch_dropsEarlierEvents() { + Event failedAttempt = + nodeEvent("wf/inner/worker", callPart("x", "c1")).branch("worker@1").build(); + Event enclosingFailure = + nodeEvent("wf/inner").errorCode(new FinishReason("NodeTimeoutException")).build(); + Event retry = + nodeEvent("wf/inner/worker", Part.fromText("retrying")).branch("worker@1").build(); + + ImmutableList events = + StepResume.withoutFailedAttempts( + ImmutableList.of(failedAttempt, enclosingFailure, retry), + ImmutableList.of(failedAttempt, retry)); + + assertThat(events).containsExactly(retry); + } + + @Test + public void withoutFailedAttempts_failureOfANodeWhosePathIsAStringPrefix_keepsEveryEvent() { + ImmutableList history = + ImmutableList.of( + nodeEvent("wf@1/worker@10", callPart("x", "c1")).build(), + nodeEvent("wf@1/worker@1") + .errorCode(new FinishReason("IllegalStateException")) + .build()); + + ImmutableList events = StepResume.withoutFailedAttempts(history, history); + + assertThat(events).isEqualTo(history); + } + private static LlmAgent coordinatorWithSingleTurnWorker() { return LlmAgent.builder() .name("coordinator") @@ -307,6 +383,10 @@ private static Event event(String author, Part... parts) { .build(); } + private static Event.Builder nodeEvent(String path, Part... parts) { + return event(AGENT, parts).toBuilder().nodeInfo(NodeInfo.builder().path(path).build()); + } + private static FunctionCall call(String name, @Nullable String id) { FunctionCall.Builder call = FunctionCall.builder().name(name); if (id != null) { diff --git a/core/src/test/java/com/google/adk/workflow/LlmAgentAsNodeTest.java b/core/src/test/java/com/google/adk/workflow/LlmAgentAsNodeTest.java index 716d391f9..1cec2d3d6 100644 --- a/core/src/test/java/com/google/adk/workflow/LlmAgentAsNodeTest.java +++ b/core/src/test/java/com/google/adk/workflow/LlmAgentAsNodeTest.java @@ -23,9 +23,11 @@ import static com.google.adk.testing.TestUtils.createTextLlmResponse; import static com.google.adk.workflow.WorkflowResumeTesting.answer; import static com.google.adk.workflow.WorkflowResumeTesting.callContent; +import static com.google.adk.workflow.WorkflowResumeTesting.captureInput; import static com.google.adk.workflow.WorkflowResumeTesting.go; import static com.google.adk.workflow.WorkflowResumeTesting.interrupt; import static com.google.adk.workflow.WorkflowResumeTesting.modelText; +import static com.google.adk.workflow.WorkflowResumeTesting.replyInput; import static com.google.common.collect.ImmutableList.toImmutableList; import static com.google.common.collect.Iterables.getLast; import static com.google.common.collect.Iterables.getOnlyElement; @@ -52,12 +54,15 @@ import com.google.adk.testing.TestLlm; import com.google.adk.testing.TestNode; import com.google.adk.testing.TestUtils.EchoTool; +import com.google.adk.tools.BaseTool; +import com.google.adk.tools.ToolContext; import com.google.adk.workflow.WorkflowResumeTesting.WorkflowSession; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.genai.types.Content; import com.google.genai.types.FunctionCall; +import com.google.genai.types.FunctionDeclaration; import com.google.genai.types.FunctionResponse; import com.google.genai.types.Part; import com.google.genai.types.Schema; @@ -67,10 +72,13 @@ import io.reactivex.rxjava3.core.Single; import io.reactivex.rxjava3.schedulers.Schedulers; import io.reactivex.rxjava3.subscribers.TestSubscriber; +import java.time.Duration; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import org.jspecify.annotations.Nullable; import org.junit.Test; @@ -662,6 +670,235 @@ public Flowable runNode(Context context, @Nullable Object input) { assertThat(recorderInputs).containsExactly("summary", "summary").inOrder(); } + @Test + public void runAsync_resumableAppRetriesAWorkflowWhoseAgentsToolFailed_callsTheModelAgain() { + TestLlm model = modelCallingLookupUntilAnswered(); + LlmAgent worker = LlmAgent.builder().name("worker").model(model).tools(failsOnce()).build(); + Workflow retried = + Workflow.builder() + .name("retried") + .edges(new Edge(Start.INSTANCE, worker)) + .config(oneRetry()) + .build(); + WorkflowSession session = + new WorkflowSession( + Workflow.builder().name("wf").edges(new Edge(Start.INSTANCE, retried)).build()); + + ImmutableList events = session.turn(go()); + + assertThat(model.getRequests()).hasSize(3); + assertThat(events.stream().map(Event::stringifyContent)).contains("found it"); + } + + @Test + public void runAsync_resumableAppRetriesATimedOutWorkflow_callsItsAgentsModelAgain() { + TestLlm model = modelCallingLookupUntilAnswered(); + LlmAgent worker = + LlmAgent.builder() + .name("worker") + .model(model) + .tools(new FlakyLookupTool(Single.never())) + .build(); + Workflow retried = + Workflow.builder() + .name("retried") + .edges(new Edge(Start.INSTANCE, worker)) + .config(oneRetry().toBuilder().timeout(Duration.ofMillis(300)).build()) + .build(); + WorkflowSession session = + new WorkflowSession( + Workflow.builder().name("wf").edges(new Edge(Start.INSTANCE, retried)).build()); + + ImmutableList events = session.turn(go()); + + assertThat(model.getRequests()).hasSize(3); + assertThat(events.stream().map(Event::stringifyContent)).contains("found it"); + } + + @Test + public void runAsync_resumableAppRetriesATimedOutFanOut_callsItsAgentsModelAgain() { + TestLlm model = modelCallingLookupUntilAnswered(); + LlmAgent worker = + LlmAgent.builder() + .name("worker") + .model(model) + .tools(new FlakyLookupTool(Single.never())) + .build(); + Workflow sub = Workflow.builder().name("sub").edges(new Edge(Start.INSTANCE, worker)).build(); + Workflow retried = + Workflow.builder() + .name("retried") + .edges(new Edge(Start.INSTANCE, sub), new Edge(Start.INSTANCE, TestNode.of("other"))) + .config(oneRetry().toBuilder().timeout(Duration.ofMillis(300)).build()) + .build(); + WorkflowSession session = + new WorkflowSession( + Workflow.builder().name("wf").edges(new Edge(Start.INSTANCE, retried)).build()); + + ImmutableList events = session.turn(go()); + + assertThat(model.getRequests()).hasSize(3); + assertThat(events.stream().map(Event::stringifyContent)).contains("found it"); + } + + @Test + public void runAsync_retriedFanOutWithTheFailedCallLastOnItsBranch_runsTheCallAgain() { + TestLlm model = modelCallingLookupUntilAnswered(); + LlmAgent worker = + LlmAgent.builder() + .name("worker") + .model(model) + .tools(new FlakyLookupTool(Single.never())) + .build(); + Workflow retried = + Workflow.builder() + .name("retried") + .edges(new Edge(Start.INSTANCE, worker), new Edge(Start.INSTANCE, TestNode.of("other"))) + .config(oneRetry().toBuilder().timeout(Duration.ofMillis(300)).build()) + .build(); + WorkflowSession session = + new WorkflowSession( + Workflow.builder().name("wf").edges(new Edge(Start.INSTANCE, retried)).build()); + + ImmutableList events = session.turn(go()); + + // Nothing follows the call on the agent's own branch, so the retry runs it, as before. + assertThat( + events.stream() + .flatMap(event -> event.functionResponses().stream()) + .map(response -> response.id().orElse(""))) + .contains("call_1"); + assertThat(events.stream().map(Event::stringifyContent)).contains("found it"); + } + + @Test + public void runAsync_resumableAppRetriesAnAgentWhoseToolFailed_callsTheModelAgain() { + TestLlm model = modelCallingLookupUntilAnswered(); + LlmAgent worker = + withNodeConfig( + LlmAgent.builder().name("worker").model(model).tools(failsOnce()), oneRetry()); + WorkflowSession session = + new WorkflowSession( + Workflow.builder().name("wf").edges(new Edge(Start.INSTANCE, worker)).build()); + + ImmutableList events = session.turn(go()); + + assertThat(model.getRequests()).hasSize(3); + assertThat(events.stream().map(Event::stringifyContent)).contains("found it"); + } + + @Test + public void runAsync_resumableAppRetriesAnAgentWhoseToolTimedOut_callsTheModelAgain() { + TestLlm model = modelCallingLookupUntilAnswered(); + LlmAgent worker = + withNodeConfig( + LlmAgent.builder() + .name("worker") + .model(model) + .tools(new FlakyLookupTool(Single.never())), + oneRetry().toBuilder().timeout(Duration.ofMillis(300)).build()); + WorkflowSession session = + new WorkflowSession( + Workflow.builder().name("wf").edges(new Edge(Start.INSTANCE, worker)).build()); + + ImmutableList events = session.turn(go()); + + assertThat(model.getRequests()).hasSize(3); + assertThat(events.stream().map(Event::stringifyContent)).contains("found it"); + } + + @Test + public void runAsync_resumableAppRetriesASingleTurnAgentWhoseToolFailed_outputsItsAnswer() { + LlmAgent worker = + withNodeConfig( + LlmAgent.builder() + .name("worker") + .model(modelCallingLookupUntilAnswered()) + .mode(AgentMode.SINGLE_TURN) + .tools(failsOnce()), + oneRetry()); + WorkflowSession session = + new WorkflowSession( + Workflow.builder() + .name("wf") + .edges(new Edge(Start.INSTANCE, worker), new Edge(worker, captureInput("reply"))) + .build()); + + ImmutableList events = session.turn(go()); + + assertThat(replyInput(events)).isEqualTo("found it"); + } + + @Test + public void runAsync_loopBackToAnAgentWhoseEarlierRunRetried_callsTheModelAgain() { + TestLlm model = + TestLlm.create( + createFunctionCallLlmResponse("call_1", "lookup", ImmutableMap.of()), + createTextLlmResponse("draft"), + createTextLlmResponse("final")); + LlmAgent worker = + withNodeConfig( + LlmAgent.builder().name("worker").model(model).tools(failsOnce()), oneRetry()); + AtomicInteger reviews = new AtomicInteger(); + Node review = + new TestNode( + "review", + (ctx, input) -> { + String route = reviews.incrementAndGet() == 1 ? "again" : "done"; + ctx.routes(ImmutableList.of(new Route.Tag(route))); + return Flowable.empty(); + }); + WorkflowSession session = + new WorkflowSession( + Workflow.builder() + .name("wf") + .edges( + new Edge(Start.INSTANCE, worker), + new Edge(worker, review), + new Edge(review, worker, new Route.Tag("again"))) + .build()); + + session.advance(go()); + + assertThat(model.getRequests()).hasSize(3); + } + + @Test + public void runAsync_agentAfterAnAgentWhoseFailedAttemptAskedForApproval_callsItsModel() { + Content approveAndLookUp = + Content.builder() + .role("model") + .parts( + Part.builder() + .functionCall(FunctionCall.builder().name("pendingTool").id("p1").build()) + .build(), + Part.builder() + .functionCall(FunctionCall.builder().name("lookup").id("c1").build()) + .build()) + .build(); + LlmAgent first = + withNodeConfig( + LlmAgent.builder() + .name("first") + .model( + TestLlm.create( + createLlmResponse(approveAndLookUp), createTextLlmResponse("first done"))) + .tools(pendingFunctionTool(), failsOnce()), + oneRetry()); + TestLlm secondModel = TestLlm.create(createTextLlmResponse("second done")); + LlmAgent second = LlmAgent.builder().name("second").model(secondModel).build(); + WorkflowSession session = + new WorkflowSession( + Workflow.builder() + .name("wf") + .edges(new Edge(Start.INSTANCE, first), new Edge(first, second)) + .build()); + + session.advance(go()); + + assertThat(secondModel.getRequests()).hasSize(1); + } + @Test public void runAsync_singleTurnNodesRunningAtOnce_doNotSeeEachOthersInput() { CountDownLatch otherAsked = new CountDownLatch(1); @@ -1056,6 +1293,36 @@ private static Event nodeEvent(String author, String path, Content content) { .build(); } + /** Returns a model that calls {@code lookup}, calls it again, then answers "found it". */ + private static TestLlm modelCallingLookupUntilAnswered() { + return TestLlm.create( + createFunctionCallLlmResponse("call_1", "lookup", ImmutableMap.of()), + createFunctionCallLlmResponse("call_2", "lookup", ImmutableMap.of()), + createTextLlmResponse("found it")); + } + + private static NodeConfig oneRetry() { + return NodeConfig.builder() + .retryConfig( + RetryConfig.builder().maxAttempts(2).initialDelay(Duration.ZERO).jitter(0).build()) + .build(); + } + + /** Returns the agent that {@code builder} builds, as a node with {@code config}. */ + private static LlmAgent withNodeConfig(LlmAgent.Builder builder, NodeConfig config) { + return new LlmAgent(builder) { + @Override + public NodeConfig config() { + return config; + } + }; + } + + /** Returns a {@code lookup} tool whose first call fails, as a flaky backend's does. */ + private static BaseTool failsOnce() { + return new FlakyLookupTool(Single.error(new IllegalStateException("backend down"))); + } + /** Returns every text and function name in {@code request}'s contents, for substring checks. */ private static String textsOf(LlmRequest request) { return request.contents().stream() @@ -1193,6 +1460,29 @@ private static Context nodeContext( .build()); } + /** A {@code lookup} tool whose first call gives {@code firstCall}; later calls succeed. */ + private static final class FlakyLookupTool extends BaseTool { + private final Single> firstCall; + private final AtomicInteger calls = new AtomicInteger(); + + FlakyLookupTool(Single> firstCall) { + super("lookup", "Looks something up."); + this.firstCall = firstCall; + } + + @Override + public Optional declaration() { + return Optional.of(FunctionDeclaration.builder().name("lookup").build()); + } + + @Override + public Single> runAsync(Map args, ToolContext toolContext) { + return calls.incrementAndGet() == 1 + ? firstCall + : Single.just(ImmutableMap.of("result", "found")); + } + } + private record Trip(String city, int days) {} /** A value without properties, which JSON can't represent. */