feat(sdk,cmdline): move HTTP range TDF loading in cmdline DSPX-5113

cmdline/pom.xml

@@ -77,7 +77,40 @@
             <artifactId>sdk</artifactId>
             <version>${project.version}</version>
         </dependency>
+        <!-- Used directly by HttpRangeChannel; exclusions mirror sdk/pom.xml so the okio and
+             kotlin-stdlib versions declared there stay the ones that win. -->
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>okhttp</artifactId>
+            <version>${okhttp.version}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.squareup.okio</groupId>
+                    <artifactId>okio-jvm</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.jetbrains.kotlin</groupId>
+                    <artifactId>kotlin-stdlib-jdk8</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <!-- Test Dependencies -->
+        <dependency>
+            <groupId>com.squareup.okhttp3</groupId>
+            <artifactId>mockwebserver</artifactId>
+            <version>${okhttp.version}</version>
+            <scope>test</scope>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.squareup.okio</groupId>
+                    <artifactId>okio-jvm</artifactId>
+                </exclusion>
+                <exclusion>
+                    <groupId>org.jetbrains.kotlin</groupId>
+                    <artifactId>kotlin-stdlib-jdk8</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter</artifactId>

cmdline/src/main/java/io/opentdf/platform/Command.java

@@ -19,6 +19,7 @@
 import java.io.PrintWriter;
 import java.net.URI;
 import java.nio.channels.FileChannel;
+import java.nio.channels.SeekableByteChannel;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.AtomicMoveNotSupportedException;
 import java.nio.file.Files;
@@ -450,14 +451,11 @@ void decrypt(
 
             var readerConfig = Config.newTDFReaderConfig(opts.toArray(new Consumer[0]));
 
-            if (tdfFile.startsWith("http://") || tdfFile.startsWith("https://")) {
-                var reader = sdk.loadTDF(URI.create(tdfFile), readerConfig);
+            // Opened last: a bad --with-assertion-verification-keys should fail before the
+            // TDF is touched, which over a URL means before any request goes out.
+            try (var in = openTdf(sdk, tdfFile)) {
+                var reader = sdk.loadTDF(in, readerConfig);
                 writePlaintext(out -> reader.readPayload(out, offset, length), output);
-            } else {
-                try (var in = FileChannel.open(Path.of(tdfFile), StandardOpenOption.READ)) {
-                    var reader = sdk.loadTDF(in, readerConfig);
-                    writePlaintext(out -> reader.readPayload(out, offset, length), output);
-                }
             }
         }
     }
@@ -495,6 +493,17 @@ static void moveIntoPlace(Path source, Path target) throws IOException {
         }
     }
 
+    /**
+     * Opens the TDF named by {@code -f}: an {@code http(s)://} URL, read in place with range
+     * requests, or anything else as a local file path.
+     */
+    private static SeekableByteChannel openTdf(SDK sdk, String tdfFile) throws IOException {
+        if (tdfFile.startsWith("http://") || tdfFile.startsWith("https://")) {
+            return HttpRangeChannel.open(sdk, URI.create(tdfFile));
+        }
+        return FileChannel.open(Path.of(tdfFile), StandardOpenOption.READ);
+    }
+
     @CommandLine.Command(name = "metadata")
     void readMetadata(
             @Option(names = { "-f", "--file" }, required = true) Path tdfPath,

sdk/src/main/java/io/opentdf/platform/sdk/HttpRangeChannel.java → cmdline/src/main/java/io/opentdf/platform/HttpRangeChannel.java

@@ -1,39 +1,91 @@
-package io.opentdf.platform.sdk;
+package io.opentdf.platform;
 
+import io.opentdf.platform.sdk.SDK;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
 
+import javax.net.ssl.SSLContext;
+import javax.net.ssl.TrustManager;
+import javax.net.ssl.X509TrustManager;
 import java.io.IOException;
 import java.net.URI;
 import java.nio.ByteBuffer;
 import java.nio.channels.ClosedChannelException;
 import java.nio.channels.NonWritableChannelException;
 import java.nio.channels.SeekableByteChannel;
+import java.security.GeneralSecurityException;
 import java.util.regex.Pattern;
 
 /**
  * A read-only {@link SeekableByteChannel} over a remote resource, where every read is one
- * HTTP range request. The size is learned by probing {@code bytes=0-0}, and every later
- * response must report the same total size, so a resource that changes underneath the
- * reader is rejected rather than spliced together.
+ * HTTP range request. Handed to {@link SDK#loadTDF(SeekableByteChannel, io.opentdf.platform.sdk.Config.TDFReaderConfig)}
+ * it lets the SDK read a TDF in place, downloading only the bytes it actually touches.
  * <p>
- * Redirects are not followed: the client handed in must have redirects disabled, and a 3xx
- * is reported as an error so the caller can use the final URL.
+ * The size is learned by probing {@code bytes=0-0}, and every later response must report the
+ * same total size, so a resource that changes underneath the reader is rejected rather than
+ * spliced together.
+ * <p>
+ * Redirects are not followed, and a 3xx is reported as an error so the caller can use the
+ * final URL.
  */
 // TODO: Add a read-ahead window if small ZIP header reads become a network bottleneck.
 class HttpRangeChannel implements SeekableByteChannel {
     private static final Pattern CONTENT_RANGE = Pattern.compile("bytes (\\d+)-(\\d+)/(\\d+)");
 
     private final OkHttpClient client;
+    // Set only when this channel built the client, and so must shut it down on close.
+    private final boolean ownsClient;
     private final URI url;
     private final long size;
     private long position;
     private boolean open = true;
 
+    /**
+     * Opens {@code url} with a client that is unauthenticated -- platform credentials never go
+     * to the TDF host -- but that keeps the TLS settings the SDK was built with, so {@code
+     * --insecure} applies here too.
+     */
+    static HttpRangeChannel open(SDK sdk, URI url) throws IOException {
+        String scheme = url.getScheme();
+        if (!"http".equalsIgnoreCase(scheme) && !"https".equalsIgnoreCase(scheme)) {
+            throw new IllegalArgumentException("unsupported URL scheme: " + scheme);
+        }
+        return new HttpRangeChannel(rangeClient(sdk), true, url);
+    }
+
     HttpRangeChannel(OkHttpClient client, URI url) throws IOException {
+        this(client, false, url);
+    }
+
+    private HttpRangeChannel(OkHttpClient client, boolean ownsClient, URI url) throws IOException {
         this.client = client;
+        this.ownsClient = ownsClient;
         this.url = url;
-        this.size = fetch(ByteBuffer.allocate(1), 0);
+        try {
+            this.size = fetch(ByteBuffer.allocate(1), 0);
+        } catch (IOException | RuntimeException e) {
+            shutdown();
+            throw e;
+        }
+    }
+
+    private static OkHttpClient rangeClient(SDK sdk) {
+        // A plain client rather than one derived from the SDK's: the SDK's carries the auth
+        // interceptor, and under --plaintext it is pinned to h2c prior knowledge, while a TDF
+        // host is an ordinary HTTP server. Default protocols (HTTP/2, HTTP/1.1) are right here.
+        var builder = new OkHttpClient.Builder().followRedirects(false).followSslRedirects(false);
+        var trustManager = sdk.getTrustManager().orElse(null);
+        if (trustManager instanceof X509TrustManager) {
+            var x509 = (X509TrustManager) trustManager;
+            try {
+                var context = SSLContext.getInstance("TLS");
+                context.init(null, new TrustManager[] { x509 }, null);
+                builder.sslSocketFactory(context.getSocketFactory(), x509);
+            } catch (GeneralSecurityException e) {
+                throw new IllegalStateException("could not build a TLS context for range requests", e);
+            }
+        }
+        return builder.build();
     }
 
     @Override
@@ -70,7 +122,7 @@ private long fetch(ByteBuffer dst, long offset) throws IOException {
         try (var response = client.newCall(request).execute()) {
             int status = response.code();
             if (status == 200) {
-                throw new SDK.RangeNotSupportedException(
+                throw new RangeNotSupportedException(
                         "status 200 for bytes " + offset + "-" + end);
             }
             if (status >= 300 && status < 400) {
@@ -83,12 +135,12 @@ private long fetch(ByteBuffer dst, long offset) throws IOException {
             String contentRange = response.header("Content-Range");
             var m = contentRange == null ? null : CONTENT_RANGE.matcher(contentRange);
             if (m == null || !m.matches()) {
-                throw new SDK.RangeNotSupportedException("invalid Content-Range " + contentRange);
+                throw new RangeNotSupportedException("invalid Content-Range " + contentRange);
             }
             long gotStart = Long.parseLong(m.group(1));
             long gotEnd = Long.parseLong(m.group(2));
             if (gotStart != offset || gotEnd != end) {
-                throw new SDK.RangeNotSupportedException("requested bytes " + offset + "-" + end
+                throw new RangeNotSupportedException("requested bytes " + offset + "-" + end
                         + ", got " + gotStart + "-" + gotEnd);
             }
             var source = response.body().source();
@@ -141,11 +193,29 @@ public boolean isOpen() {
     @Override
     public void close() {
         open = false;
+        shutdown();
+    }
+
+    private void shutdown() {
+        if (ownsClient) {
+            client.dispatcher().executorService().shutdown();
+            client.connectionPool().evictAll();
+        }
     }
 
     private void ensureOpen() throws ClosedChannelException {
         if (!open) {
             throw new ClosedChannelException();
         }
     }
+
+    /**
+     * The server holding a TDF did not honor an HTTP range request, so the TDF cannot be
+     * read in place; download it and decrypt it from a file instead.
+     */
+    static class RangeNotSupportedException extends IOException {
+        RangeNotSupportedException(String errorMessage) {
+            super("server did not honor the HTTP range request: " + errorMessage);
+        }
+    }
 }

sdk/src/test/java/io/opentdf/platform/sdk/HttpRangeChannelTest.java → cmdline/src/test/java/io/opentdf/platform/HttpRangeChannelTest.java

@@ -1,20 +1,26 @@
-package io.opentdf.platform.sdk;
+package io.opentdf.platform;
 
 import okhttp3.OkHttpClient;
 import okhttp3.mockwebserver.Dispatcher;
 import okhttp3.mockwebserver.MockResponse;
 import okhttp3.mockwebserver.MockWebServer;
 import okhttp3.mockwebserver.RecordedRequest;
+import io.opentdf.platform.sdk.ZipReader;
 import okio.Buffer;
 import org.junit.jupiter.api.Test;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.net.URI;
 import java.nio.ByteBuffer;
+import java.nio.channels.ClosedChannelException;
 import java.nio.charset.StandardCharsets;
 import java.util.concurrent.atomic.AtomicLong;
 import java.util.function.Supplier;
 import java.util.regex.Pattern;
+import java.util.zip.CRC32;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -71,12 +77,41 @@ void readsAndSeeks() throws Exception {
         }
     }
 
+    /** Only the bytes asked for cross the wire, which is the point of reading a TDF in place. */
+    @Test
+    void readsOnlyTheRequestedBytes() throws Exception {
+        byte[] data = new byte[100_000];
+        var served = new AtomicLong();
+        try (var server = new MockWebServer()) {
+            server.setDispatcher(rangeDispatcher(() -> data, served));
+            try (var ch = new HttpRangeChannel(CLIENT, url(server))) {
+                served.set(0); // discount the one-byte size probe
+                assertThat(ch.position(50_000).read(ByteBuffer.allocate(1_000))).isEqualTo(1_000);
+            }
+            assertThat(served).hasValue(1_000);
+        }
+    }
+
+    @Test
+    void usesAfterCloseAreRejected() throws Exception {
+        byte[] data = "0123456789".getBytes(StandardCharsets.UTF_8);
+        try (var server = new MockWebServer()) {
+            server.setDispatcher(rangeDispatcher(() -> data, new AtomicLong()));
+            var ch = new HttpRangeChannel(CLIENT, url(server));
+            ch.close();
+
+            assertThat(ch.isOpen()).isFalse();
+            assertThatThrownBy(() -> ch.read(ByteBuffer.allocate(1)))
+                    .isInstanceOf(ClosedChannelException.class);
+        }
+    }
+
     @Test
     void rangeNotSupported() throws Exception {
         try (var server = new MockWebServer()) {
             server.enqueue(new MockResponse().setResponseCode(200).setBody("not a partial response"));
             assertThatThrownBy(() -> new HttpRangeChannel(CLIENT, url(server)))
-                    .isInstanceOf(SDK.RangeNotSupportedException.class);
+                    .isInstanceOf(HttpRangeChannel.RangeNotSupportedException.class);
         }
     }
 
@@ -86,7 +121,7 @@ void mismatchedContentRangeIsRangeNotSupported() throws Exception {
             server.enqueue(new MockResponse().setResponseCode(206)
                     .setHeader("Content-Range", "bytes 0-9/10").setBody("0123456789"));
             assertThatThrownBy(() -> new HttpRangeChannel(CLIENT, url(server)))
-                    .isInstanceOf(SDK.RangeNotSupportedException.class);
+                    .isInstanceOf(HttpRangeChannel.RangeNotSupportedException.class);
         }
     }
 
@@ -97,7 +132,7 @@ void errorStatusesReportTheirStatus() throws Exception {
                 server.enqueue(new MockResponse().setResponseCode(status).setHeader("Location", "/elsewhere"));
                 assertThatThrownBy(() -> new HttpRangeChannel(CLIENT, url(server)))
                         .isInstanceOf(IOException.class)
-                        .isNotInstanceOf(SDK.RangeNotSupportedException.class)
+                        .isNotInstanceOf(HttpRangeChannel.RangeNotSupportedException.class)
                         .hasMessageContaining(String.valueOf(status))
                         .satisfies(e -> {
                             if (status == 302) {
@@ -121,13 +156,57 @@ void sizeChanged() throws Exception {
         }
     }
 
+    /**
+     * The SDK reads a TDF as a zip over whatever channel it is handed, so the channel has to
+     * satisfy ZipReader's seek-heavy access pattern -- not just the reads a test drives by hand.
+     */
     @Test
-    void rejectsNonHttpSchemes() {
-        var sdk = new SDK(new FakeServicesBuilder().build(), null, null, null, null, null);
-        var uri = URI.create("file:///etc/passwd");
-        var config = Config.newTDFReaderConfig();
+    void readsAZipArchiveThroughZipReader() throws Exception {
+        var zip = new ByteArrayOutputStream();
+        byte[] payload = new byte[64_000];
+        new java.util.Random(1).nextBytes(payload);
+        // STORED, like TDFWriter writes: ZipReader hands back entry bytes as they sit in the
+        // archive and does not inflate.
+        try (var out = new ZipOutputStream(zip)) {
+            out.setMethod(ZipOutputStream.STORED);
+            out.putNextEntry(stored("0.manifest.json", "{\"hello\":\"world\"}".getBytes(StandardCharsets.UTF_8)));
+            out.write("{\"hello\":\"world\"}".getBytes(StandardCharsets.UTF_8));
+            out.putNextEntry(stored("0.payload", payload));
+            out.write(payload);
+        }
+        byte[] archive = zip.toByteArray();
+
+        try (var server = new MockWebServer()) {
+            server.setDispatcher(rangeDispatcher(() -> archive, new AtomicLong()));
+            try (var ch = new HttpRangeChannel(CLIENT, url(server))) {
+                var entries = new ZipReader(ch).getEntries();
+
+                assertThat(entries).extracting(ZipReader.Entry::getName)
+                        .containsExactlyInAnyOrder("0.manifest.json", "0.payload");
+                var found = entries.stream().filter(e -> e.getName().equals("0.payload")).findFirst().orElseThrow();
+                try (var data = found.getData()) {
+                    assertThat(data.readAllBytes()).isEqualTo(payload);
+                }
+            }
+        }
+    }
 
-        assertThatThrownBy(() -> sdk.loadTDF(uri, config))
-                .isInstanceOf(IllegalArgumentException.class);
+    private static ZipEntry stored(String name, byte[] data) {
+        var crc = new CRC32();
+        crc.update(data);
+        var entry = new ZipEntry(name);
+        entry.setMethod(ZipEntry.STORED);
+        entry.setSize(data.length);
+        entry.setCompressedSize(data.length);
+        entry.setCrc(crc.getValue());
+        return entry;
+    }
+
+    /** The scheme is checked before the SDK is touched, so a null SDK is enough here. */
+    @Test
+    void rejectsNonHttpSchemes() {
+        assertThatThrownBy(() -> HttpRangeChannel.open(null, URI.create("file:///etc/passwd")))
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessageContaining("file");
     }
 }