diff --git a/CHANGELOG.md b/CHANGELOG.md
index aa91cb63..bfa44074 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,8 +10,13 @@
* Add a `spiffeIdVerifier` option to `SpiffeSslContextFactory.SslContextOptions`, so an `SSLContext` can verify peer SPIFFE IDs with a custom `SpiffeIdVerifier` instead of a fixed set of accepted SPIFFE IDs
* Add `SpiffeIdVerifier.acceptAny()`, `acceptId()`, `acceptOneOf()` and `acceptMemberOf()` factory methods for verifiers that accept any SPIFFE ID, a specific SPIFFE ID, one of a set of SPIFFE IDs, or any SPIFFE ID in a trust domain
+### Changed
+
+* Retry Workload API `PERMISSION_DENIED` responses with backoff. With the default client and no initialization timeout, `DefaultX509Source.newSource()`, `DefaultJwtSource.newSource()` and `CachedJwtSource.newSource()` now wait indefinitely for an unregistered workload instead of failing immediately. Set a nonzero `initTimeout` or the `spiffe.newX509Source.timeout` / `spiffe.newJwtSource.timeout` system property to bound the wait.
+
### Fixed
+* Always report terminal Workload API watch failures after a permission denial, including repeated denials, preserving the exception type and cause.
* Reject X.509-SVID leaf certificates with a root SPIFFE ID path in `X509SvidValidator.verifyChain`, which `SpiffeTrustManager` uses for peer certificates (#510)
* Ignore JWKs with an unsupported key type when parsing a JWT bundle instead of rejecting the whole bundle, per the SPIFFE bundle spec (#512)
* Serialize `java-spiffe-helper` KeyStore and TrustStore files to memory before writing them, so a store or encoding failure no longer leaves an empty or partial file (#490)
diff --git a/java-spiffe-core/README.md b/java-spiffe-core/README.md
index fc6ec677..b8f89ea0 100644
--- a/java-spiffe-core/README.md
+++ b/java-spiffe-core/README.md
@@ -47,7 +47,8 @@ It allows to configure another SVID picker. By default, the first SVID is used.
The `X509Source newSource()` method blocks waiting until an X.509 context is fetched. The X.509 context fetch is retried
using an exponential backoff policy with this progression of delays between retries: 1 second, 2 seconds, 4, 8, 16, 32, 60, 60, 60...
-It retries indefinitely unless a timeout is configured.
+It retries indefinitely unless a timeout is configured. This includes `PERMISSION_DENIED`, so an unregistered
+workload waits for registration instead of failing immediately. Set a nonzero `initTimeout` to bound this wait.
This timeout can be configured either with the `initTimeout` option of the `X509SourceOptions` passed to the
`newSource(X509SourceOptions options)` method:
@@ -125,7 +126,8 @@ the cached JWT-SVIDs have passed half of their lifetime.
The `JwtSource newSource()` method blocks until the JWT materials are fetched. The fetching process is retried
using an exponential backoff policy with this progression of delays between retries: 1 second, 2 seconds, 4, 8, 16, 32, 60, 60, 60...
-It retries indefinitely unless a timeout is configured.
+It retries indefinitely unless a timeout is configured. This includes `PERMISSION_DENIED`, so an unregistered
+workload waits for registration instead of failing immediately. Set a nonzero `initTimeout` to bound this wait.
This timeout can be configured either with the `initTimeout` option of the `JwtSourceOptions` passed to the
`newSource(JwtSourceOptions options)` method:
diff --git a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/CachedJwtSource.java b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/CachedJwtSource.java
index 118ff52f..f95a2e21 100644
--- a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/CachedJwtSource.java
+++ b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/CachedJwtSource.java
@@ -63,6 +63,10 @@ private CachedJwtSource(final WorkloadApiClient workloadApiClient) {
* through the system property `spiffe.newJwtSource.timeout` expires.
* If no timeout is configured, it blocks until it gets a JWT update from the Workload API.
*
+ * With the default client, {@code PERMISSION_DENIED} is retried with backoff, so an unregistered
+ * workload can wait indefinitely. To bound this wait, use {@link #newSource(JwtSourceOptions)}
+ * with a nonzero {@code initTimeout}, or set the timeout system property.
+ *
* It uses the default address socket endpoint from the environment variable to get the Workload API address.
*
* @return an instance of {@link DefaultJwtSource}, with the JWT bundles initialized
@@ -83,6 +87,9 @@ public static JwtSource newSource() throws JwtSourceException, SocketEndpointAdd
* system property `spiffe.newJwtSource.timeout`. If none is configured, this method will
* block until the JWT bundles can be retrieved from the Workload API.
*
+ * With the default client, {@code PERMISSION_DENIED} is retried with backoff. Set a nonzero
+ * {@code initTimeout} to bound the wait for an unregistered workload.
+ *
* The {@link WorkloadApiClient} can be provided in the options, if it is not,
* a new client is created.
*
diff --git a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/DefaultJwtSource.java b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/DefaultJwtSource.java
index b5542c6a..65d38492 100644
--- a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/DefaultJwtSource.java
+++ b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/DefaultJwtSource.java
@@ -50,6 +50,10 @@ private DefaultJwtSource(final WorkloadApiClient workloadApiClient) {
* through the system property `spiffe.newJwtSource.timeout` expires.
* If no timeout is configured, it blocks until it gets a JWT update from the Workload API.
*
+ * With the default client, {@code PERMISSION_DENIED} is retried with backoff, so an unregistered
+ * workload can wait indefinitely. To bound this wait, use {@link #newSource(JwtSourceOptions)}
+ * with a nonzero {@code initTimeout}, or set the timeout system property.
+ *
* It uses the default address socket endpoint from the environment variable to get the Workload API address.
*
* @return an instance of {@link DefaultJwtSource}, with the JWT bundles initialized
@@ -70,6 +74,9 @@ public static JwtSource newSource() throws JwtSourceException, SocketEndpointAdd
* system property `spiffe.newJwtSource.timeout`. If none is configured, this method will
* block until the JWT bundles can be retrieved from the Workload API.
*
+ * With the default client, {@code PERMISSION_DENIED} is retried with backoff. Set a nonzero
+ * {@code initTimeout} to bound the wait for an unregistered workload.
+ *
* The {@link WorkloadApiClient} can be provided in the options, if it is not,
* a new client is created.
*
diff --git a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/DefaultWorkloadApiClient.java b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/DefaultWorkloadApiClient.java
index 5d02d81d..6d514a3a 100644
--- a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/DefaultWorkloadApiClient.java
+++ b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/DefaultWorkloadApiClient.java
@@ -40,8 +40,18 @@
*
* Supports one-shot calls and watch updates for X.509 and JWT SVIDs and bundles.
*
- * The watch for updates methods support retries using an exponential backoff policy to reestablish
- * the stream connection to the Workload API.
+ * The watch methods retry eligible errors with exponential backoff. Before the first successfully
+ * delivered update, {@code PERMISSION_DENIED} is not reported while retries can be scheduled.
+ * After an update, the first denial is reported immediately to {@link Watcher#onError(Throwable)};
+ * retries continue if the callback returns normally and the retry policy permits them.
+ * Repeated denials are not reported again until another update is successfully delivered.
+ *
+ * Retry exhaustion or scheduling failure always reports a terminal error with a {@code Cancelling}
+ * message, including when a permission denial has already been reported. A denial can therefore
+ * produce both an immediate withdrawal notification and a terminal error if no retry can be scheduled.
+ *
+ * Rejected updates are reported to {@code onError}; if that callback returns normally, the watch
+ * remains active. An exception thrown by {@code onError} escapes the callback and prevents further retries.
*/
public final class DefaultWorkloadApiClient implements WorkloadApiClient {
diff --git a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/DefaultX509Source.java b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/DefaultX509Source.java
index c0659105..a452bbcc 100644
--- a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/DefaultX509Source.java
+++ b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/DefaultX509Source.java
@@ -76,6 +76,10 @@ private DefaultX509Source(final Function, X509Svid> svidPicker, f
* through the system property `spiffe.newX509Source.timeout` expires.
* If no timeout is configured, it blocks until it gets an X.509 update from the Workload API.
*
+ * With the default client, {@code PERMISSION_DENIED} is retried with backoff, so an unregistered
+ * workload can wait indefinitely. To bound this wait, use {@link #newSource(X509SourceOptions)}
+ * with a nonzero {@code initTimeout}, or set the timeout system property.
+ *
* It uses the default address socket endpoint from the environment variable to get the Workload API address.
*
* It uses the default X.509 SVID (picks the first SVID that comes in the Workload API response).
@@ -98,6 +102,9 @@ public static DefaultX509Source newSource() throws SocketEndpointAddressExceptio
* system property `spiffe.newX509Source.timeout`. If none is configured, this method will
* block until the X.509 materials can be retrieved from the Workload API.
*
+ * With the default client, {@code PERMISSION_DENIED} is retried with backoff. Set a nonzero
+ * {@code initTimeout} to bound the wait for an unregistered workload.
+ *
* The {@link WorkloadApiClient} can be provided in the options, if it is not,
* a new client is created.
*
diff --git a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/GrpcConversionUtils.java b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/GrpcConversionUtils.java
index 1478f3e1..d1f0e9d4 100644
--- a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/GrpcConversionUtils.java
+++ b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/GrpcConversionUtils.java
@@ -6,6 +6,7 @@
import io.spiffe.bundle.x509bundle.X509Bundle;
import io.spiffe.bundle.x509bundle.X509BundleSet;
import io.spiffe.exception.JwtBundleException;
+import io.spiffe.exception.InvalidSpiffeIdException;
import io.spiffe.exception.X509BundleException;
import io.spiffe.exception.X509ContextException;
import io.spiffe.exception.X509SvidException;
@@ -113,7 +114,12 @@ private static List getListOfX509Bundles(final Workload.X509SVIDResp
// Process federated bundles
Set> federatedBundles = x509SvidResponse.getFederatedBundlesMap().entrySet();
for (Map.Entry bundleEntry : federatedBundles) {
- TrustDomain trustDomain = TrustDomain.parse(bundleEntry.getKey());
+ final TrustDomain trustDomain;
+ try {
+ trustDomain = TrustDomain.parse(bundleEntry.getKey());
+ } catch (InvalidSpiffeIdException | IllegalArgumentException e) {
+ throw new X509ContextException("X.509 federated bundle trust domain could not be processed", e);
+ }
byte[] bundleBytes = bundleEntry.getValue().toByteArray();
final X509Bundle bundle = parseX509Bundle(trustDomain, bundleBytes);
x509BundleList.add(bundle);
@@ -176,13 +182,23 @@ private static void validateSpiffeId(String spiffeIdSvid, String spiffeIdRespons
}
private static JwtBundle createJwtBundle(Map.Entry entry) throws JwtBundleException {
- TrustDomain trustDomain = TrustDomain.parse(entry.getKey());
+ final TrustDomain trustDomain;
+ try {
+ trustDomain = TrustDomain.parse(entry.getKey());
+ } catch (InvalidSpiffeIdException | IllegalArgumentException e) {
+ throw new JwtBundleException("JWT bundle trust domain could not be processed", e);
+ }
byte[] bundleBytes = entry.getValue().toByteArray();
return JwtBundle.parse(trustDomain, bundleBytes);
}
private static X509Bundle createX509Bundle(Map.Entry bundleEntry) throws X509BundleException {
- TrustDomain trustDomain = TrustDomain.parse(bundleEntry.getKey());
+ final TrustDomain trustDomain;
+ try {
+ trustDomain = TrustDomain.parse(bundleEntry.getKey());
+ } catch (InvalidSpiffeIdException | IllegalArgumentException e) {
+ throw new X509BundleException("X.509 bundle trust domain could not be processed", e);
+ }
byte[] bundleBytes = bundleEntry.getValue().toByteArray();
return X509Bundle.parse(trustDomain, bundleBytes);
}
diff --git a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/StreamObservers.java b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/StreamObservers.java
index 70dd4403..3d282ff6 100644
--- a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/StreamObservers.java
+++ b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/StreamObservers.java
@@ -23,11 +23,11 @@ final class StreamObservers {
Logger.getLogger(StreamObservers.class.getName());
private static final String STREAM_IS_COMPLETED = "Workload API stream is completed";
- // Retry only transient stream failures. Caller/client cancellation and terminal API errors fail the watch closed.
+ // PermissionDenied is retryable, but must be reported after an update has been delivered.
+ // Caller/client cancellation and terminal API errors fail the watch closed.
private static final Set NON_RETRYABLE_CODES = EnumSet.of(
Status.Code.INVALID_ARGUMENT,
Status.Code.CANCELLED,
- Status.Code.PERMISSION_DENIED,
Status.Code.UNAUTHENTICATED,
Status.Code.UNIMPLEMENTED,
Status.Code.FAILED_PRECONDITION);
@@ -42,11 +42,16 @@ static StreamObserver getX509ContextStreamObserver(
final SpiffeWorkloadAPIGrpc.SpiffeWorkloadAPIStub workloadApiAsyncStub) {
return new StreamObserver() {
+ private boolean updateDelivered;
+ private boolean permissionDeniedReported;
+
@Override
public void onNext(final Workload.X509SVIDResponse value) {
try {
final X509Context x509Context = GrpcConversionUtils.toX509Context(value);
watcher.onUpdate(x509Context);
+ updateDelivered = true;
+ permissionDeniedReported = false;
retryHandler.reset();
} catch (X509ContextException e) {
watcher.onError(new X509ContextException("Error processing X.509 Context update", e));
@@ -65,6 +70,11 @@ private void handleWatchX509ContextError(final Throwable t) {
if (isErrorNotRetryable(t)) {
watcher.onError(new X509ContextException("Cancelling X.509 Context watch", t));
} else {
+ if (updateDelivered && !permissionDeniedReported
+ && Status.fromThrowable(t).getCode() == Status.Code.PERMISSION_DENIED) {
+ permissionDeniedReported = true;
+ watcher.onError(new X509ContextException("Permission denied for X.509 Context watch", t));
+ }
handleX509ContextRetry(t);
}
}
@@ -98,14 +108,19 @@ static StreamObserver getX509BundlesStreamObserver
final SpiffeWorkloadAPIGrpc.SpiffeWorkloadAPIStub workloadApiAsyncStub) {
return new StreamObserver() {
+ private boolean updateDelivered;
+ private boolean permissionDeniedReported;
+
@Override
public void onNext(final Workload.X509BundlesResponse value) {
try {
final X509BundleSet x509Context = GrpcConversionUtils.toX509BundleSet(value);
watcher.onUpdate(x509Context);
+ updateDelivered = true;
+ permissionDeniedReported = false;
retryHandler.reset();
} catch (X509BundleException e) {
- watcher.onError(new X509ContextException("Error processing X.509 bundles update", e));
+ watcher.onError(new X509BundleException("Error processing X.509 bundles update", e));
}
}
@@ -119,8 +134,13 @@ public void onError(final Throwable t) {
private void handleWatchX509BundlesError(final Throwable t) {
if (isErrorNotRetryable(t)) {
- watcher.onError(new X509ContextException("Cancelling X.509 bundles watch", t));
+ watcher.onError(new X509BundleException("Cancelling X.509 bundles watch", t));
} else {
+ if (updateDelivered && !permissionDeniedReported
+ && Status.fromThrowable(t).getCode() == Status.Code.PERMISSION_DENIED) {
+ permissionDeniedReported = true;
+ watcher.onError(new X509BundleException("Permission denied for X.509 bundles watch", t));
+ }
handleX509BundlesRetry(t);
}
}
@@ -153,12 +173,16 @@ static StreamObserver getJwtBundleStreamObserver(
final Context.CancellableContext cancellableContext,
final SpiffeWorkloadAPIGrpc.SpiffeWorkloadAPIStub workloadApiAsyncStub) {
return new StreamObserver() {
+ private boolean updateDelivered;
+ private boolean permissionDeniedReported;
@Override
public void onNext(final Workload.JWTBundlesResponse value) {
try {
final JwtBundleSet jwtBundleSet = GrpcConversionUtils.toJwtBundleSet(value);
watcher.onUpdate(jwtBundleSet);
+ updateDelivered = true;
+ permissionDeniedReported = false;
retryHandler.reset();
} catch (JwtBundleException e) {
watcher.onError(new JwtBundleException("Error processing JWT bundles update", e));
@@ -177,6 +201,11 @@ private void handleWatchJwtBundleError(final Throwable t) {
if (isErrorNotRetryable(t)) {
watcher.onError(new JwtBundleException("Cancelling JWT Bundles watch", t));
} else {
+ if (updateDelivered && !permissionDeniedReported
+ && Status.fromThrowable(t).getCode() == Status.Code.PERMISSION_DENIED) {
+ permissionDeniedReported = true;
+ watcher.onError(new JwtBundleException("Permission denied for JWT Bundles watch", t));
+ }
handleJwtBundleRetry(t);
}
}
diff --git a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/WorkloadApiClient.java b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/WorkloadApiClient.java
index bf55b79a..30c34026 100644
--- a/java-spiffe-core/src/main/java/io/spiffe/workloadapi/WorkloadApiClient.java
+++ b/java-spiffe-core/src/main/java/io/spiffe/workloadapi/WorkloadApiClient.java
@@ -33,6 +33,7 @@ public interface WorkloadApiClient extends Closeable {
* A new Stream to the Workload API is opened for each call to this method, so that the client starts getting
* updates immediately after the Stream is ready and doesn't have to wait until the Workload API dispatches
* the next update based on the SVIDs TTL.
+ * See {@link DefaultWorkloadApiClient} for retry and error callback behavior.
*
* @param watcher an instance that implements a {@link Watcher}.
*/
@@ -52,6 +53,7 @@ public interface WorkloadApiClient extends Closeable {
* A new Stream to the Workload API is opened for each call to this method, so that the client starts getting
* updates immediately after the Stream is ready and doesn't have to wait until the Workload API dispatches
* the next update.
+ * See {@link DefaultWorkloadApiClient} for retry and error callback behavior.
*
* @param watcher an instance that implements a {@link Watcher} for {@link X509BundleSet}.
*/
@@ -123,6 +125,7 @@ public interface WorkloadApiClient extends Closeable {
* A new Stream to the Workload API is opened for each call to this method, so that the client starts getting
* updates immediately after the Stream is ready and doesn't have to wait until the Workload API dispatches
* the next update based on the SVIDs TTL.
+ * See {@link DefaultWorkloadApiClient} for retry and error callback behavior.
*
* @param watcher receives the update for JwtBundles.
*/
diff --git a/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientCorruptedResponsesTest.java b/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientCorruptedResponsesTest.java
index 0668f381..86259a7d 100644
--- a/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientCorruptedResponsesTest.java
+++ b/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientCorruptedResponsesTest.java
@@ -17,6 +17,7 @@
import java.util.concurrent.CountDownLatch;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.fail;
class DefaultWorkloadApiClientCorruptedResponsesTest {
@@ -85,6 +86,7 @@ public void onUpdate(X509BundleSet update) {
@Override
public void onError(Throwable e) {
+ assertInstanceOf(X509BundleException.class, e);
assertEquals("Error processing X.509 bundles update", e.getMessage());
done.countDown();
}
diff --git a/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientEmptyResponseTest.java b/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientEmptyResponseTest.java
index d5857247..0efb8cc0 100644
--- a/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientEmptyResponseTest.java
+++ b/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientEmptyResponseTest.java
@@ -17,6 +17,7 @@
import java.util.concurrent.CountDownLatch;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.fail;
class DefaultWorkloadApiClientEmptyResponseTest {
@@ -86,6 +87,7 @@ public void onUpdate(X509BundleSet update) {
@Override
public void onError(Throwable e) {
+ assertInstanceOf(X509BundleException.class, e);
assertEquals("Error processing X.509 bundles update", e.getMessage());
done.countDown();
}
diff --git a/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientInvalidTrustDomainTest.java b/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientInvalidTrustDomainTest.java
new file mode 100644
index 00000000..12a7fb06
--- /dev/null
+++ b/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientInvalidTrustDomainTest.java
@@ -0,0 +1,152 @@
+package io.spiffe.workloadapi;
+
+import com.google.protobuf.ByteString;
+import io.grpc.stub.StreamObserver;
+import io.grpc.testing.GrpcCleanupRule;
+import io.spiffe.bundle.jwtbundle.JwtBundleSet;
+import io.spiffe.bundle.x509bundle.X509BundleSet;
+import io.spiffe.exception.JwtBundleException;
+import io.spiffe.exception.X509BundleException;
+import io.spiffe.exception.X509ContextException;
+import io.spiffe.workloadapi.grpc.Workload;
+import org.junit.Rule;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.UnaryOperator;
+
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class DefaultWorkloadApiClientInvalidTrustDomainTest {
+
+ @Rule
+ public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+
+ private DefaultWorkloadApiClient workloadApiClient;
+
+ @BeforeEach
+ void setUp() throws IOException {
+ workloadApiClient = WorkloadApiClientTestUtil.create(
+ new InvalidTrustDomainThenValidWorkloadApi(),
+ grpcCleanup);
+ }
+
+ @AfterEach
+ void tearDown() {
+ workloadApiClient.close();
+ }
+
+ @Test
+ void invalidTrustDomainReportsTypedErrorAndLaterUpdatesAreDelivered() throws InterruptedException {
+ CountDownLatch errors = new CountDownLatch(3);
+ CountDownLatch updates = new CountDownLatch(3);
+ AtomicReference unexpectedError = new AtomicReference<>();
+
+ workloadApiClient.watchX509Context(
+ watcher(X509ContextException.class, errors, updates, unexpectedError));
+ workloadApiClient.watchX509Bundles(
+ watcher(X509BundleException.class, errors, updates, unexpectedError));
+ workloadApiClient.watchJwtBundles(
+ watcher(JwtBundleException.class, errors, updates, unexpectedError));
+
+ assertTrue(errors.await(5, TimeUnit.SECONDS), "timed out waiting for rejected update errors");
+ assertTrue(updates.await(5, TimeUnit.SECONDS), "timed out waiting for valid updates");
+ assertNull(unexpectedError.get());
+ }
+
+ private static Watcher watcher(
+ Class extends Throwable> expectedErrorType,
+ CountDownLatch errors,
+ CountDownLatch updates,
+ AtomicReference unexpectedError) {
+ return new Watcher() {
+ @Override
+ public void onUpdate(T update) {
+ updates.countDown();
+ }
+
+ @Override
+ public void onError(Throwable error) {
+ if (!expectedErrorType.isInstance(error)) {
+ unexpectedError.compareAndSet(null, error);
+ }
+ errors.countDown();
+ }
+ };
+ }
+
+ private static final class InvalidTrustDomainThenValidWorkloadApi extends FakeWorkloadApi {
+
+ private static final String INVALID_TRUST_DOMAIN = "invalid/trust-domain";
+
+ @Override
+ public void fetchX509SVID(
+ Workload.X509SVIDRequest request,
+ StreamObserver responseObserver) {
+ super.fetchX509SVID(
+ request,
+ invalidThenValid(responseObserver, response -> {
+ ByteString bundle = response.getFederatedBundlesMap().values().iterator().next();
+ return response.toBuilder()
+ .putFederatedBundles(INVALID_TRUST_DOMAIN, bundle)
+ .build();
+ }));
+ }
+
+ @Override
+ public void fetchX509Bundles(
+ Workload.X509BundlesRequest request,
+ StreamObserver responseObserver) {
+ super.fetchX509Bundles(
+ request,
+ invalidThenValid(responseObserver, response -> {
+ ByteString bundle = response.getBundlesMap().values().iterator().next();
+ return response.toBuilder()
+ .putBundles(INVALID_TRUST_DOMAIN, bundle)
+ .build();
+ }));
+ }
+
+ @Override
+ public void fetchJWTBundles(
+ Workload.JWTBundlesRequest request,
+ StreamObserver responseObserver) {
+ super.fetchJWTBundles(
+ request,
+ invalidThenValid(responseObserver, response -> {
+ ByteString bundle = response.getBundlesMap().values().iterator().next();
+ return response.toBuilder()
+ .putBundles(INVALID_TRUST_DOMAIN, bundle)
+ .build();
+ }));
+ }
+
+ private static StreamObserver invalidThenValid(
+ StreamObserver responseObserver,
+ UnaryOperator invalidUpdate) {
+ return new StreamObserver() {
+ @Override
+ public void onNext(T validUpdate) {
+ responseObserver.onNext(invalidUpdate.apply(validUpdate));
+ responseObserver.onNext(validUpdate);
+ }
+
+ @Override
+ public void onError(Throwable error) {
+ responseObserver.onError(error);
+ }
+
+ @Override
+ public void onCompleted() {
+ responseObserver.onCompleted();
+ }
+ };
+ }
+ }
+}
diff --git a/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientPermissionDeniedRetryTest.java b/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientPermissionDeniedRetryTest.java
new file mode 100644
index 00000000..5b5de797
--- /dev/null
+++ b/java-spiffe-core/src/test/java/io/spiffe/workloadapi/DefaultWorkloadApiClientPermissionDeniedRetryTest.java
@@ -0,0 +1,188 @@
+package io.spiffe.workloadapi;
+
+import io.grpc.Status;
+import io.grpc.stub.StreamObserver;
+import io.grpc.testing.GrpcCleanupRule;
+import io.spiffe.bundle.jwtbundle.JwtBundleSet;
+import io.spiffe.bundle.x509bundle.X509BundleSet;
+import io.spiffe.exception.JwtBundleException;
+import io.spiffe.exception.X509BundleException;
+import io.spiffe.exception.X509ContextException;
+import io.spiffe.workloadapi.grpc.Workload;
+import org.junit.Rule;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class DefaultWorkloadApiClientPermissionDeniedRetryTest {
+
+ @Rule
+ public final GrpcCleanupRule grpcCleanup = new GrpcCleanupRule();
+
+ private PermissionDeniedThenSuccessWorkloadApi fakeWorkloadApi;
+ private DefaultWorkloadApiClient workloadApiClient;
+
+ @BeforeEach
+ void setUp() throws IOException {
+ fakeWorkloadApi = new PermissionDeniedThenSuccessWorkloadApi();
+ workloadApiClient = WorkloadApiClientTestUtil.create(fakeWorkloadApi, grpcCleanup);
+ }
+
+ @AfterEach
+ void tearDown() {
+ workloadApiClient.close();
+ }
+
+ @Test
+ void permissionDeniedRetriesAllWatchesUntilAnUpdateIsReceived() throws InterruptedException {
+ CountDownLatch updates = new CountDownLatch(3);
+ AtomicReference watchError = new AtomicReference<>();
+
+ workloadApiClient.watchX509Context(watcher(updates, watchError));
+ workloadApiClient.watchX509Bundles(watcher(updates, watchError));
+ workloadApiClient.watchJwtBundles(watcher(updates, watchError));
+
+ assertTrue(updates.await(5, TimeUnit.SECONDS), "timed out waiting for retried watch updates");
+ assertNull(watchError.get());
+ assertEquals(2, fakeWorkloadApi.x509ContextRequests.get());
+ assertEquals(2, fakeWorkloadApi.x509BundleRequests.get());
+ assertEquals(2, fakeWorkloadApi.jwtBundleRequests.get());
+ }
+
+ @Test
+ void permissionDeniedAfterUpdateReportsTypedErrorAndRetriesAllWatches() throws InterruptedException {
+ fakeWorkloadApi.sendUpdateBeforeDenial = true;
+ CountDownLatch updates = new CountDownLatch(6);
+ List x509ContextEvents = new CopyOnWriteArrayList<>();
+ List x509BundleEvents = new CopyOnWriteArrayList<>();
+ List jwtBundleEvents = new CopyOnWriteArrayList<>();
+
+ workloadApiClient.watchX509Context(recordingWatcher(updates, x509ContextEvents));
+ workloadApiClient.watchX509Bundles(recordingWatcher(updates, x509BundleEvents));
+ workloadApiClient.watchJwtBundles(recordingWatcher(updates, jwtBundleEvents));
+
+ assertTrue(updates.await(5, TimeUnit.SECONDS), "timed out waiting for updates after permission denial");
+ assertEquals(Arrays.asList("update", X509ContextException.class.getName(), "update"), x509ContextEvents);
+ assertEquals(Arrays.asList("update", X509BundleException.class.getName(), "update"), x509BundleEvents);
+ assertEquals(Arrays.asList("update", JwtBundleException.class.getName(), "update"), jwtBundleEvents);
+ assertEquals(2, fakeWorkloadApi.x509ContextRequests.get());
+ assertEquals(2, fakeWorkloadApi.x509BundleRequests.get());
+ assertEquals(2, fakeWorkloadApi.jwtBundleRequests.get());
+ }
+
+ private static Watcher recordingWatcher(CountDownLatch updates, List events) {
+ return new Watcher() {
+ @Override
+ public void onUpdate(T update) {
+ events.add("update");
+ updates.countDown();
+ }
+
+ @Override
+ public void onError(Throwable error) {
+ events.add(error.getClass().getName());
+ }
+ };
+ }
+
+ private static Watcher watcher(
+ CountDownLatch updates,
+ AtomicReference watchError) {
+ return new Watcher() {
+ @Override
+ public void onUpdate(T update) {
+ updates.countDown();
+ }
+
+ @Override
+ public void onError(Throwable error) {
+ watchError.compareAndSet(null, error);
+ }
+ };
+ }
+
+ private static final class PermissionDeniedThenSuccessWorkloadApi extends FakeWorkloadApi {
+
+ private boolean sendUpdateBeforeDenial;
+ private final AtomicInteger x509ContextRequests = new AtomicInteger();
+ private final AtomicInteger x509BundleRequests = new AtomicInteger();
+ private final AtomicInteger jwtBundleRequests = new AtomicInteger();
+
+ @Override
+ public void fetchX509SVID(
+ Workload.X509SVIDRequest request,
+ StreamObserver responseObserver) {
+ if (x509ContextRequests.incrementAndGet() == 1) {
+ if (sendUpdateBeforeDenial) {
+ super.fetchX509SVID(request, permissionDeniedOnCompletion(responseObserver));
+ } else {
+ responseObserver.onError(Status.PERMISSION_DENIED.asRuntimeException());
+ }
+ return;
+ }
+ super.fetchX509SVID(request, responseObserver);
+ }
+
+ @Override
+ public void fetchX509Bundles(
+ Workload.X509BundlesRequest request,
+ StreamObserver responseObserver) {
+ if (x509BundleRequests.incrementAndGet() == 1) {
+ if (sendUpdateBeforeDenial) {
+ super.fetchX509Bundles(request, permissionDeniedOnCompletion(responseObserver));
+ } else {
+ responseObserver.onError(Status.PERMISSION_DENIED.asRuntimeException());
+ }
+ return;
+ }
+ super.fetchX509Bundles(request, responseObserver);
+ }
+
+ @Override
+ public void fetchJWTBundles(
+ Workload.JWTBundlesRequest request,
+ StreamObserver responseObserver) {
+ if (jwtBundleRequests.incrementAndGet() == 1) {
+ if (sendUpdateBeforeDenial) {
+ super.fetchJWTBundles(request, permissionDeniedOnCompletion(responseObserver));
+ } else {
+ responseObserver.onError(Status.PERMISSION_DENIED.asRuntimeException());
+ }
+ return;
+ }
+ super.fetchJWTBundles(request, responseObserver);
+ }
+
+ private static StreamObserver permissionDeniedOnCompletion(StreamObserver observer) {
+ return new StreamObserver() {
+ @Override
+ public void onNext(T value) {
+ observer.onNext(value);
+ }
+
+ @Override
+ public void onError(Throwable error) {
+ observer.onError(error);
+ }
+
+ @Override
+ public void onCompleted() {
+ observer.onError(Status.PERMISSION_DENIED.asRuntimeException());
+ }
+ };
+ }
+ }
+}
diff --git a/java-spiffe-core/src/test/java/io/spiffe/workloadapi/StreamObserversTest.java b/java-spiffe-core/src/test/java/io/spiffe/workloadapi/StreamObserversTest.java
new file mode 100644
index 00000000..9b49d168
--- /dev/null
+++ b/java-spiffe-core/src/test/java/io/spiffe/workloadapi/StreamObserversTest.java
@@ -0,0 +1,481 @@
+package io.spiffe.workloadapi;
+
+import io.grpc.Context;
+import io.grpc.Status;
+import io.grpc.stub.StreamObserver;
+import io.spiffe.bundle.jwtbundle.JwtBundleSet;
+import io.spiffe.bundle.x509bundle.X509BundleSet;
+import io.spiffe.exception.JwtBundleException;
+import io.spiffe.exception.X509BundleException;
+import io.spiffe.exception.X509ContextException;
+import io.spiffe.workloadapi.grpc.Workload;
+import io.spiffe.workloadapi.retry.RetryHandler;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.EnumSource;
+import org.junit.jupiter.params.provider.MethodSource;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
+
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertSame;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class StreamObserversTest {
+
+ @ParameterizedTest
+ @EnumSource(WatchType.class)
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ void permissionDeniedAfterUpdateReportsOnceBeforeRetrying(WatchType watchType) {
+ Watcher watcher = mock(Watcher.class);
+ RetryHandler retryHandler = mock(RetryHandler.class);
+ when(retryHandler.shouldRetry()).thenReturn(true);
+ when(retryHandler.tryScheduleRetry(any(Runnable.class))).thenReturn(true);
+ Throwable denial = Status.PERMISSION_DENIED.asRuntimeException();
+
+ Context.CancellableContext context = Context.current().withCancellation();
+ try {
+ StreamObserver> observer = newStreamObserver(watchType, watcher, retryHandler, context);
+ deliverUpdate(watchType, observer);
+ observer.onError(denial);
+ observer.onError(Status.PERMISSION_DENIED.asRuntimeException());
+
+ InOrder order = inOrder(watcher, retryHandler);
+ order.verify(watcher).onUpdate(any());
+ order.verify(retryHandler).reset();
+ order.verify(watcher).onError(any(Throwable.class));
+ order.verify(retryHandler).shouldRetry();
+ order.verify(retryHandler).tryScheduleRetry(any(Runnable.class));
+ order.verify(retryHandler).shouldRetry();
+ order.verify(retryHandler).tryScheduleRetry(any(Runnable.class));
+ order.verifyNoMoreInteractions();
+ assertWatcherError(watcher, watchType, denial);
+ } finally {
+ context.close();
+ }
+ }
+
+ @ParameterizedTest
+ @EnumSource(WatchType.class)
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ void successfulUpdateAllowsAnotherPermissionDeniedNotification(WatchType watchType) {
+ Watcher watcher = mock(Watcher.class);
+ RetryHandler retryHandler = mock(RetryHandler.class);
+ when(retryHandler.shouldRetry()).thenReturn(true);
+ when(retryHandler.tryScheduleRetry(any(Runnable.class))).thenReturn(true);
+ Throwable firstDenial = Status.PERMISSION_DENIED.asRuntimeException();
+ Throwable secondDenial = Status.PERMISSION_DENIED.asRuntimeException();
+
+ Context.CancellableContext context = Context.current().withCancellation();
+ try {
+ StreamObserver> observer = newStreamObserver(watchType, watcher, retryHandler, context);
+ deliverUpdate(watchType, observer);
+ observer.onError(firstDenial);
+ deliverUpdate(watchType, observer);
+ observer.onError(secondDenial);
+
+ ArgumentCaptor errors = ArgumentCaptor.forClass(Throwable.class);
+ InOrder order = inOrder(watcher);
+ order.verify(watcher).onUpdate(any());
+ order.verify(watcher).onError(errors.capture());
+ order.verify(watcher).onUpdate(any());
+ order.verify(watcher).onError(errors.capture());
+ order.verifyNoMoreInteractions();
+ assertInstanceOf(watchType.errorType, errors.getAllValues().get(0));
+ assertSame(firstDenial, errors.getAllValues().get(0).getCause());
+ assertInstanceOf(watchType.errorType, errors.getAllValues().get(1));
+ assertSame(secondDenial, errors.getAllValues().get(1).getCause());
+ verify(retryHandler, times(2)).tryScheduleRetry(any(Runnable.class));
+ } finally {
+ context.close();
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("permissionDeniedRetryFailureCases")
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ void permissionDeniedRetryFailureReportsDenialAndTerminalError(WatchType watchType, boolean shouldRetry) {
+ Watcher watcher = mock(Watcher.class);
+ RetryHandler retryHandler = mock(RetryHandler.class);
+ when(retryHandler.shouldRetry()).thenReturn(shouldRetry);
+ if (shouldRetry) {
+ when(retryHandler.tryScheduleRetry(any(Runnable.class))).thenReturn(false);
+ }
+ Throwable denial = Status.PERMISSION_DENIED.asRuntimeException();
+
+ Context.CancellableContext context = Context.current().withCancellation();
+ try {
+ StreamObserver> observer = newStreamObserver(watchType, watcher, retryHandler, context);
+ deliverUpdate(watchType, observer);
+ observer.onError(denial);
+
+ ArgumentCaptor errors = ArgumentCaptor.forClass(Throwable.class);
+ verify(watcher, times(2)).onError(errors.capture());
+ assertInstanceOf(watchType.errorType, errors.getAllValues().get(0));
+ assertSame(denial, errors.getAllValues().get(0).getCause());
+ assertTrue(errors.getAllValues().get(0).getMessage().startsWith("Permission denied for "));
+ assertInstanceOf(watchType.errorType, errors.getAllValues().get(1));
+ assertSame(denial, errors.getAllValues().get(1).getCause());
+ assertTrue(errors.getAllValues().get(1).getMessage().startsWith("Cancelling "));
+ InOrder order = inOrder(watcher, retryHandler);
+ order.verify(watcher).onUpdate(any());
+ order.verify(retryHandler).reset();
+ order.verify(watcher).onError(errors.getAllValues().get(0));
+ order.verify(retryHandler).shouldRetry();
+ if (shouldRetry) {
+ order.verify(retryHandler).tryScheduleRetry(any(Runnable.class));
+ } else {
+ verify(retryHandler, never()).tryScheduleRetry(any(Runnable.class));
+ }
+ order.verify(watcher).onError(errors.getAllValues().get(1));
+ order.verifyNoMoreInteractions();
+ } finally {
+ context.close();
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("permissionDeniedRetryFailureCases")
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ void repeatedDenialRetryFailureReportsTerminalError(WatchType watchType, boolean shouldRetry) {
+ Watcher watcher = mock(Watcher.class);
+ RetryHandler retryHandler = mock(RetryHandler.class);
+ when(retryHandler.shouldRetry()).thenReturn(true, shouldRetry);
+ when(retryHandler.tryScheduleRetry(any(Runnable.class))).thenReturn(true, false);
+ Throwable denial = Status.PERMISSION_DENIED.asRuntimeException();
+ Throwable terminalDenial = Status.PERMISSION_DENIED.asRuntimeException();
+
+ Context.CancellableContext context = Context.current().withCancellation();
+ try {
+ StreamObserver> observer = newStreamObserver(watchType, watcher, retryHandler, context);
+ deliverUpdate(watchType, observer);
+ observer.onError(denial);
+ observer.onError(terminalDenial);
+
+ ArgumentCaptor errors = ArgumentCaptor.forClass(Throwable.class);
+ verify(watcher, times(2)).onError(errors.capture());
+ assertInstanceOf(watchType.errorType, errors.getAllValues().get(0));
+ assertSame(denial, errors.getAllValues().get(0).getCause());
+ assertTrue(errors.getAllValues().get(0).getMessage().startsWith("Permission denied for "));
+ assertInstanceOf(watchType.errorType, errors.getAllValues().get(1));
+ assertSame(terminalDenial, errors.getAllValues().get(1).getCause());
+ assertTrue(errors.getAllValues().get(1).getMessage().startsWith("Cancelling "));
+ InOrder order = inOrder(watcher, retryHandler);
+ order.verify(watcher).onUpdate(any());
+ order.verify(retryHandler).reset();
+ order.verify(watcher).onError(errors.getAllValues().get(0));
+ order.verify(retryHandler).shouldRetry();
+ order.verify(retryHandler).tryScheduleRetry(any(Runnable.class));
+ order.verify(retryHandler).shouldRetry();
+ if (shouldRetry) {
+ order.verify(retryHandler).tryScheduleRetry(any(Runnable.class));
+ }
+ order.verify(watcher).onError(errors.getAllValues().get(1));
+ order.verifyNoMoreInteractions();
+ } finally {
+ context.close();
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("permissionDeniedRetryFailureCases")
+ @SuppressWarnings("rawtypes")
+ void permissionDeniedBeforeUpdateReportsOnlyTerminalError(WatchType watchType, boolean shouldRetry) {
+ Watcher watcher = mock(Watcher.class);
+ RetryHandler retryHandler = mock(RetryHandler.class);
+ when(retryHandler.shouldRetry()).thenReturn(shouldRetry);
+ if (shouldRetry) {
+ when(retryHandler.tryScheduleRetry(any(Runnable.class))).thenReturn(false);
+ }
+ Throwable denial = Status.PERMISSION_DENIED.asRuntimeException();
+
+ Context.CancellableContext context = Context.current().withCancellation();
+ try {
+ StreamObserver> observer = newStreamObserver(watchType, watcher, retryHandler, context);
+ observer.onError(denial);
+
+ ArgumentCaptor errors = ArgumentCaptor.forClass(Throwable.class);
+ verify(watcher).onError(errors.capture());
+ assertInstanceOf(watchType.errorType, errors.getValue());
+ assertSame(denial, errors.getValue().getCause());
+ assertTrue(errors.getValue().getMessage().startsWith("Cancelling "));
+ InOrder order = inOrder(watcher, retryHandler);
+ order.verify(retryHandler).shouldRetry();
+ if (shouldRetry) {
+ order.verify(retryHandler).tryScheduleRetry(any(Runnable.class));
+ } else {
+ verify(retryHandler, never()).tryScheduleRetry(any(Runnable.class));
+ }
+ order.verify(watcher).onError(errors.getValue());
+ order.verifyNoMoreInteractions();
+ } finally {
+ context.close();
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("permissionDeniedRetryFailureCases")
+ @SuppressWarnings("rawtypes")
+ void distinctRetryFailureAfterReportedDenialNotifiesWatcher(WatchType watchType, boolean shouldRetry) {
+ Watcher watcher = mock(Watcher.class);
+ RetryHandler retryHandler = mock(RetryHandler.class);
+ when(retryHandler.shouldRetry()).thenReturn(true, shouldRetry);
+ when(retryHandler.tryScheduleRetry(any(Runnable.class))).thenReturn(true, false);
+ Throwable denial = Status.PERMISSION_DENIED.asRuntimeException();
+ Throwable retryError = Status.UNAVAILABLE.asRuntimeException();
+
+ Context.CancellableContext context = Context.current().withCancellation();
+ try {
+ StreamObserver> observer = newStreamObserver(watchType, watcher, retryHandler, context);
+ deliverUpdate(watchType, observer);
+ observer.onError(denial);
+ observer.onError(retryError);
+
+ ArgumentCaptor errors = ArgumentCaptor.forClass(Throwable.class);
+ verify(watcher, times(2)).onError(errors.capture());
+ assertInstanceOf(watchType.errorType, errors.getAllValues().get(0));
+ assertSame(denial, errors.getAllValues().get(0).getCause());
+ assertInstanceOf(watchType.errorType, errors.getAllValues().get(1));
+ assertSame(retryError, errors.getAllValues().get(1).getCause());
+ verify(retryHandler, times(2)).shouldRetry();
+ verify(retryHandler, times(shouldRetry ? 2 : 1)).tryScheduleRetry(any(Runnable.class));
+ } finally {
+ context.close();
+ }
+ }
+
+ @ParameterizedTest
+ @EnumSource(WatchType.class)
+ @SuppressWarnings("rawtypes")
+ void throwingPermissionDeniedCallbackPreventsRetry(WatchType watchType) {
+ Watcher watcher = mock(Watcher.class);
+ RetryHandler retryHandler = mock(RetryHandler.class);
+ RuntimeException callbackError = new RuntimeException("watcher stopped");
+ doThrow(callbackError).when(watcher).onError(any(Throwable.class));
+
+ Context.CancellableContext context = Context.current().withCancellation();
+ try {
+ StreamObserver> observer = newStreamObserver(watchType, watcher, retryHandler, context);
+ deliverUpdate(watchType, observer);
+
+ assertSame(callbackError, assertThrows(RuntimeException.class,
+ () -> observer.onError(Status.PERMISSION_DENIED.asRuntimeException())));
+ verify(retryHandler, never()).shouldRetry();
+ verify(retryHandler, never()).tryScheduleRetry(any(Runnable.class));
+ } finally {
+ context.close();
+ }
+ }
+
+ @ParameterizedTest
+ @EnumSource(WatchType.class)
+ @SuppressWarnings("rawtypes")
+ void permissionDeniedSchedulesWatchRetry(WatchType watchType) {
+ Watcher watcher = mock(Watcher.class);
+ RetryHandler retryHandler = mock(RetryHandler.class);
+ when(retryHandler.shouldRetry()).thenReturn(true);
+ when(retryHandler.tryScheduleRetry(any(Runnable.class))).thenReturn(true);
+
+ Context.CancellableContext cancellableContext = Context.current().withCancellation();
+ try {
+ StreamObserver> streamObserver =
+ newStreamObserver(watchType, watcher, retryHandler, cancellableContext);
+ streamObserver.onError(Status.PERMISSION_DENIED.asRuntimeException());
+
+ verify(retryHandler).shouldRetry();
+ verify(retryHandler).tryScheduleRetry(any(Runnable.class));
+ verify(watcher, never()).onError(any(Throwable.class));
+ } finally {
+ cancellableContext.close();
+ }
+ }
+
+ @ParameterizedTest
+ @MethodSource("nonRetryableStatusCases")
+ @SuppressWarnings("rawtypes")
+ void nonRetryableStatusStopsWatch(WatchType watchType, Status.Code statusCode) {
+ Watcher watcher = mock(Watcher.class);
+ RetryHandler retryHandler = mock(RetryHandler.class);
+
+ Context.CancellableContext cancellableContext = Context.current().withCancellation();
+ try {
+ StreamObserver> streamObserver =
+ newStreamObserver(watchType, watcher, retryHandler, cancellableContext);
+ streamObserver.onError(Status.fromCode(statusCode).asRuntimeException());
+
+ ArgumentCaptor errorCaptor = ArgumentCaptor.forClass(Throwable.class);
+ verify(watcher).onError(errorCaptor.capture());
+ assertInstanceOf(watchType.errorType, errorCaptor.getValue());
+ verify(retryHandler, never()).shouldRetry();
+ verify(retryHandler, never()).tryScheduleRetry(any(Runnable.class));
+ } finally {
+ cancellableContext.close();
+ }
+ }
+
+ @ParameterizedTest
+ @EnumSource(WatchType.class)
+ @SuppressWarnings("rawtypes")
+ void exhaustedRetriesReportTypedError(WatchType watchType) {
+ Watcher watcher = mock(Watcher.class);
+ RetryHandler retryHandler = mock(RetryHandler.class);
+ when(retryHandler.shouldRetry()).thenReturn(false);
+ Throwable retryError = Status.UNAVAILABLE.asRuntimeException();
+
+ Context.CancellableContext cancellableContext = Context.current().withCancellation();
+ try {
+ StreamObserver> streamObserver =
+ newStreamObserver(watchType, watcher, retryHandler, cancellableContext);
+ streamObserver.onError(retryError);
+
+ assertWatcherError(watcher, watchType, retryError);
+ verify(retryHandler).shouldRetry();
+ verify(retryHandler, never()).tryScheduleRetry(any(Runnable.class));
+ } finally {
+ cancellableContext.close();
+ }
+ }
+
+ @ParameterizedTest
+ @EnumSource(WatchType.class)
+ @SuppressWarnings("rawtypes")
+ void rejectedRetrySchedulingReportsTypedError(WatchType watchType) {
+ Watcher watcher = mock(Watcher.class);
+ RetryHandler retryHandler = mock(RetryHandler.class);
+ when(retryHandler.shouldRetry()).thenReturn(true);
+ when(retryHandler.tryScheduleRetry(any(Runnable.class))).thenReturn(false);
+ Throwable retryError = Status.UNAVAILABLE.asRuntimeException();
+
+ Context.CancellableContext cancellableContext = Context.current().withCancellation();
+ try {
+ StreamObserver> streamObserver =
+ newStreamObserver(watchType, watcher, retryHandler, cancellableContext);
+ streamObserver.onError(retryError);
+
+ assertWatcherError(watcher, watchType, retryError);
+ verify(retryHandler).shouldRetry();
+ verify(retryHandler).tryScheduleRetry(any(Runnable.class));
+ } finally {
+ cancellableContext.close();
+ }
+ }
+
+ private static Stream permissionDeniedRetryFailureCases() {
+ return Stream.of(WatchType.values())
+ .flatMap(watchType -> Stream.of(false, true)
+ .map(shouldRetry -> Arguments.of(watchType, shouldRetry)));
+ }
+
+ @SuppressWarnings("unchecked")
+ private void deliverUpdate(WatchType watchType, StreamObserver> observer) {
+ FakeWorkloadApi service = new FakeWorkloadApi();
+ switch (watchType) {
+ case X509_CONTEXT:
+ service.fetchX509SVID(Workload.X509SVIDRequest.getDefaultInstance(),
+ withoutCompletion((StreamObserver) observer));
+ break;
+ case X509_BUNDLES:
+ service.fetchX509Bundles(Workload.X509BundlesRequest.getDefaultInstance(),
+ withoutCompletion((StreamObserver) observer));
+ break;
+ case JWT_BUNDLES:
+ service.fetchJWTBundles(Workload.JWTBundlesRequest.getDefaultInstance(),
+ withoutCompletion((StreamObserver) observer));
+ break;
+ default:
+ throw new IllegalArgumentException("Unsupported watch type: " + watchType);
+ }
+ }
+
+ private static StreamObserver withoutCompletion(StreamObserver observer) {
+ return new StreamObserver() {
+ @Override
+ public void onNext(T value) {
+ observer.onNext(value);
+ }
+
+ @Override
+ public void onError(Throwable error) {
+ observer.onError(error);
+ }
+
+ @Override
+ public void onCompleted() {
+ // Leave the watch open for the next update or error.
+ }
+ };
+ }
+
+ private static Stream nonRetryableStatusCases() {
+ return Stream.of(WatchType.values())
+ .flatMap(watchType -> Stream.of(
+ Status.Code.INVALID_ARGUMENT,
+ Status.Code.CANCELLED,
+ Status.Code.UNAUTHENTICATED,
+ Status.Code.UNIMPLEMENTED,
+ Status.Code.FAILED_PRECONDITION)
+ .map(statusCode -> Arguments.of(watchType, statusCode)));
+ }
+
+ @SuppressWarnings({"rawtypes", "unchecked"})
+ private StreamObserver> newStreamObserver(
+ WatchType watchType,
+ Watcher watcher,
+ RetryHandler retryHandler,
+ Context.CancellableContext cancellableContext) {
+ switch (watchType) {
+ case X509_CONTEXT:
+ return StreamObservers.getX509ContextStreamObserver(
+ (Watcher) watcher,
+ retryHandler,
+ cancellableContext,
+ null);
+ case X509_BUNDLES:
+ return StreamObservers.getX509BundlesStreamObserver(
+ (Watcher) watcher,
+ retryHandler,
+ cancellableContext,
+ null);
+ case JWT_BUNDLES:
+ return StreamObservers.getJwtBundleStreamObserver(
+ (Watcher) watcher,
+ retryHandler,
+ cancellableContext,
+ null);
+ default:
+ throw new IllegalArgumentException("Unsupported watch type: " + watchType);
+ }
+ }
+
+ @SuppressWarnings("rawtypes")
+ private void assertWatcherError(Watcher watcher, WatchType watchType, Throwable cause) {
+ ArgumentCaptor errorCaptor = ArgumentCaptor.forClass(Throwable.class);
+ verify(watcher).onError(errorCaptor.capture());
+ Throwable watcherError = errorCaptor.getValue();
+ assertInstanceOf(watchType.errorType, watcherError);
+ assertSame(cause, watcherError.getCause());
+ }
+
+ private enum WatchType {
+ X509_CONTEXT(X509ContextException.class),
+ X509_BUNDLES(X509BundleException.class),
+ JWT_BUNDLES(JwtBundleException.class);
+
+ private final Class extends Throwable> errorType;
+
+ WatchType(Class extends Throwable> errorType) {
+ this.errorType = errorType;
+ }
+ }
+}