From 0a33ae5009b56ff87002be295fcb46e4655e26cb Mon Sep 17 00:00:00 2001 From: Konstantin Date: Thu, 13 Aug 2026 16:59:32 +0200 Subject: [PATCH 1/3] implemented zfp codec --- USERGUIDE.md | 15 +- pom.xml | 6 + .../zarr/zarrjava/v3/codec/CodecBuilder.java | 60 +++ .../zarr/zarrjava/v3/codec/CodecRegistry.java | 1 + .../zarr/zarrjava/v3/codec/core/ZfpCodec.java | 479 ++++++++++++++++++ .../java/dev/zarr/zarrjava/ZarrV3Test.java | 239 +++++++++ 6 files changed, 799 insertions(+), 1 deletion(-) create mode 100644 src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java diff --git a/USERGUIDE.md b/USERGUIDE.md index 65bf9944..8916feee 100644 --- a/USERGUIDE.md +++ b/USERGUIDE.md @@ -20,7 +20,7 @@ zarr-java is a Java implementation of the [Zarr specification](https://zarr.dev/ ### Key Features - **Full Zarr v2 and v3 support**: Read and write arrays in both formats - **Multiple storage backends**: Filesystem, HTTP, S3, ZIP, and in-memory storage -- **Compression codecs**: Blosc, Gzip, Zstd, and more +- **Compression codecs**: Blosc, Gzip, Zstd, Zfp, and more - **Sharding support**: Efficient storage for many small chunks (v3) - **Parallel I/O**: Optional parallel reading and writing for performance - **Type-safe API**: Strong typing with covariant return types @@ -448,6 +448,19 @@ Array array = Array.create( ```java .withCodecs(c -> c.withZstd(3)) // Level 1-22 ``` +#### Zfp Compression +Compresses numerical chunks with [zfp](https://zfp.io), lossless or with a chosen error bound: +```java +.withCodecs(c -> c.withZfpReversible()) // Lossless +.withCodecs(c -> c.withZfpFixedAccuracy(0.05)) // Absolute error bound +.withCodecs(c -> c.withZfpFixedRate(8)) // Compressed bits per value +.withCodecs(c -> c.withZfpFixedPrecision(19)) // Bit planes retained +.withCodecs(c -> c.withZfpExpert(1, 13, 19, -2)) // zfp's expert mode parameters +``` +Zfp replaces the `bytes` codec, so it cannot be combined with it. Chunks may have at most four +dimensions, and `bool` is not supported. Data types narrower than 32 bits are promoted to `int32`; +`uint32` and `uint64` values beyond the signed range are clamped, so they do not survive a round trip +even in reversible mode. #### Transpose Codec ```java .withCodecs(c -> c diff --git a/pom.xml b/pom.xml index 36eb31ca..20a49c71 100644 --- a/pom.xml +++ b/pom.xml @@ -48,6 +48,7 @@ 2.34.6 5.9.1 1.5.5-7 + 0.1-1.0.1 5.14.0 3.0.2 @@ -104,6 +105,11 @@ blosc-java 0.3-1.21.6 + + com.scalableminds + zfp-java + ${zfpVersion} + com.github.luben zstd-jni diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index 5c3487ce..d995a703 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -110,6 +110,66 @@ public CodecBuilder withZstd(int level) { return withZstd(level, true); } + public CodecBuilder withZfp(ZfpCodec.Configuration configuration) { + codecs.add(new ZfpCodec(configuration)); + return this; + } + + /** + * Compresses chunks losslessly with zfp. + */ + public CodecBuilder withZfpReversible() { + try { + return withZfp(ZfpCodec.Configuration.reversible()); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + } + + /** + * Compresses chunks with zfp, with a guaranteed absolute error bound. + */ + public CodecBuilder withZfpFixedAccuracy(double tolerance) { + try { + return withZfp(ZfpCodec.Configuration.fixedAccuracy(tolerance)); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + } + + /** + * Compresses chunks with zfp, at a fixed number of compressed bits per value. + */ + public CodecBuilder withZfpFixedRate(double rate) { + try { + return withZfp(ZfpCodec.Configuration.fixedRate(rate)); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + } + + /** + * Compresses chunks with zfp, retaining a fixed number of bit planes. + */ + public CodecBuilder withZfpFixedPrecision(int precision) { + try { + return withZfp(ZfpCodec.Configuration.fixedPrecision(precision)); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + } + + /** + * Compresses chunks with zfp, setting all four of zfp's expert mode parameters directly. + */ + public CodecBuilder withZfpExpert(int minbits, int maxbits, int maxprec, int minexp) { + try { + return withZfp(ZfpCodec.Configuration.expert(minbits, maxbits, maxprec, minexp)); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + } + public CodecBuilder withSharding(int[] chunkShape) { try { codecs.add( diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java index ad249e08..4187bbcb 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java @@ -16,6 +16,7 @@ public class CodecRegistry { addType("blosc", BloscCodec.class); addType("gzip", GzipCodec.class); addType("zstd", ZstdCodec.class); + addType("zfp", ZfpCodec.class); addType("crc32c", Crc32cCodec.class); addType("sharding_indexed", ShardingIndexedCodec.class); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java new file mode 100644 index 00000000..80ebb155 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java @@ -0,0 +1,479 @@ +package dev.zarr.zarrjava.v3.codec.core; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; +import com.scalableminds.zfpjava.Zfp; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; +import dev.zarr.zarrjava.utils.Utils; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.codec.Codec; +import ucar.ma2.Array; +import ucar.ma2.DataType; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; + +/** + * Compresses chunks with the zfp algorithm, as specified by the + * zfp codec + * extension. + * + *

zfp itself only handles 1- to 4-dimensional arrays of {@code int32}, {@code int64}, + * {@code float32} and {@code float64}. Narrower integer data types are promoted to {@code int32} and + * demoted back with the shifts given in the specification; unsigned 32- and 64-bit values are + * clamped into the signed range, matching the reference implementation in zarrs. Clamping is lossy + * for values above {@code 2^31 - 1} respectively {@code 2^63 - 1}, even in reversible mode. + */ +public class ZfpCodec extends ArrayBytesCodec implements Codec { + + @JsonIgnore + @Nonnull + public final String name = "zfp"; + @Nonnull + public final Configuration configuration; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public ZfpCodec( + @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration + ) { + this.configuration = configuration; + } + + private static int clamp(int value, int min, int max) { + return Math.min(Math.max(value, min), max); + } + + @Override + public ByteBuffer encode(Array chunkArray) throws ZarrException { + final DataType dataType = arrayMetadata.dataType.getMA2DataType(); + final Zfp.Type zfpType = zfpType(dataType); + final int[] zfpShape = zfpShape(); + + final ByteBuffer promoted = + ByteBuffer.allocate((int) Zfp.countValues(zfpShape) * zfpType.getByteCount()) + .order(ByteOrder.nativeOrder()); + promote(chunkArray, dataType, promoted); + promoted.rewind(); + + try { + return ByteBuffer.wrap( + Zfp.compress(promoted.array(), zfpType, zfpShape, params(zfpType, zfpShape.length))); + } catch (RuntimeException ex) { + throw new ZarrException("Error in encoding zfp.", ex); + } + } + + @Override + public Array decode(ByteBuffer chunkBytes) throws ZarrException { + final DataType dataType = arrayMetadata.dataType.getMA2DataType(); + final Zfp.Type zfpType = zfpType(dataType); + final int[] zfpShape = zfpShape(); + + final byte[] promoted; + try { + promoted = Zfp.decompress(Utils.toArray(chunkBytes), zfpType, zfpShape, + params(zfpType, zfpShape.length)); + } catch (RuntimeException ex) { + throw new ZarrException("Error in decoding zfp.", ex); + } + return demote(ByteBuffer.wrap(promoted).order(ByteOrder.nativeOrder()), dataType, + (int) Zfp.countValues(zfpShape)); + } + + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + throw new ZarrException("Not implemented for Zfp codec."); + } + + /** + * The zfp scalar type the chunk's values are compressed as. Data types narrower than 32 bits are + * compressed as {@code int32}, unsigned types as their signed counterpart. + */ + private Zfp.Type zfpType(DataType dataType) throws ZarrException { + switch (dataType) { + case BYTE: + case UBYTE: + case SHORT: + case USHORT: + case INT: + case UINT: + return Zfp.Type.INT32; + case LONG: + case ULONG: + return Zfp.Type.INT64; + case FLOAT: + return Zfp.Type.FLOAT; + case DOUBLE: + return Zfp.Type.DOUBLE; + default: + throw new ZarrException( + "The zfp codec does not support the data type '" + arrayMetadata.dataType + "'."); + } + } + + /** + * The chunk shape as a zfp field shape, in row-major order. The chunk of a zero-dimensional array + * is a 1D field holding a single value. + */ + private int[] zfpShape() throws ZarrException { + final int[] chunkShape = arrayMetadata.chunkShape; + if (chunkShape.length == 0) { + return new int[]{1}; + } + if (chunkShape.length > Zfp.MAX_DIMS) { + throw new ZarrException( + "The zfp codec supports at most " + Zfp.MAX_DIMS + " dimensions, but the chunk shape " + + Arrays.toString(chunkShape) + " has " + chunkShape.length + "."); + } + return chunkShape; + } + + /** + * Resolves the configured compression mode to zfp's expert mode parameters. Fixed-rate mode + * depends on the zfp type and the dimensionality, which is why this cannot happen while parsing + * the configuration. + */ + private Zfp.Params params(Zfp.Type zfpType, int dims) throws ZarrException { + try { + switch (configuration.mode) { + case FIXED_RATE: + return Zfp.Params.fixedRate(configuration.rate, zfpType, dims); + default: + return configuration.resolveParams(); + } + } catch (IllegalArgumentException ex) { + throw new ZarrException("Invalid zfp codec configuration.", ex); + } + } + + /** + * Writes the chunk's values into {@code out} as zfp scalars, promoting narrower integers into + * {@code int32} as specified. + */ + private void promote(Array chunkArray, DataType dataType, ByteBuffer out) { + switch (dataType) { + case BYTE: { + final byte[] values = (byte[]) chunkArray.copyTo1DJavaArray(); + for (byte value : values) { + out.putInt(value << 23); + } + break; + } + case UBYTE: { + final byte[] values = (byte[]) chunkArray.copyTo1DJavaArray(); + for (byte value : values) { + out.putInt(((value & 0xFF) - 0x80) << 23); + } + break; + } + case SHORT: { + final short[] values = (short[]) chunkArray.copyTo1DJavaArray(); + for (short value : values) { + out.putInt(value << 15); + } + break; + } + case USHORT: { + final short[] values = (short[]) chunkArray.copyTo1DJavaArray(); + for (short value : values) { + out.putInt(((value & 0xFFFF) - 0x8000) << 15); + } + break; + } + case INT: { + final int[] values = (int[]) chunkArray.copyTo1DJavaArray(); + for (int value : values) { + out.putInt(value); + } + break; + } + case UINT: { + final int[] values = (int[]) chunkArray.copyTo1DJavaArray(); + for (int value : values) { + // Values above Integer.MAX_VALUE have their sign bit set and are clamped + out.putInt(value < 0 ? Integer.MAX_VALUE : value); + } + break; + } + case LONG: { + final long[] values = (long[]) chunkArray.copyTo1DJavaArray(); + for (long value : values) { + out.putLong(value); + } + break; + } + case ULONG: { + final long[] values = (long[]) chunkArray.copyTo1DJavaArray(); + for (long value : values) { + // Values above Long.MAX_VALUE have their sign bit set and are clamped + out.putLong(value < 0 ? Long.MAX_VALUE : value); + } + break; + } + case FLOAT: { + final float[] values = (float[]) chunkArray.copyTo1DJavaArray(); + for (float value : values) { + out.putFloat(value); + } + break; + } + case DOUBLE: { + final double[] values = (double[]) chunkArray.copyTo1DJavaArray(); + for (double value : values) { + out.putDouble(value); + } + break; + } + default: + throw new IllegalStateException("Unsupported data type: " + dataType); + } + } + + /** + * Reads {@code valueCount} zfp scalars from {@code in} and demotes them back into the chunk's data + * type. + */ + private Array demote(ByteBuffer in, DataType dataType, int valueCount) { + final int[] shape = arrayMetadata.chunkShape; + switch (dataType) { + case BYTE: + case UBYTE: { + final byte[] values = new byte[valueCount]; + for (int i = 0; i < valueCount; i++) { + final int value = in.getInt() >> 23; + values[i] = dataType == DataType.BYTE + ? (byte) clamp(value, -0x80, 0x7F) + : (byte) clamp(value + 0x80, 0x00, 0xFF); + } + return Array.factory(dataType, shape, values); + } + case SHORT: + case USHORT: { + final short[] values = new short[valueCount]; + for (int i = 0; i < valueCount; i++) { + final int value = in.getInt() >> 15; + values[i] = dataType == DataType.SHORT + ? (short) clamp(value, -0x8000, 0x7FFF) + : (short) clamp(value + 0x8000, 0x0000, 0xFFFF); + } + return Array.factory(dataType, shape, values); + } + case INT: + case UINT: { + final int[] values = new int[valueCount]; + for (int i = 0; i < valueCount; i++) { + final int value = in.getInt(); + values[i] = dataType == DataType.INT ? value : Math.max(value, 0); + } + return Array.factory(dataType, shape, values); + } + case LONG: + case ULONG: { + final long[] values = new long[valueCount]; + for (int i = 0; i < valueCount; i++) { + final long value = in.getLong(); + values[i] = dataType == DataType.LONG ? value : Math.max(value, 0); + } + return Array.factory(dataType, shape, values); + } + case FLOAT: { + final float[] values = new float[valueCount]; + for (int i = 0; i < valueCount; i++) { + values[i] = in.getFloat(); + } + return Array.factory(dataType, shape, values); + } + case DOUBLE: { + final double[] values = new double[valueCount]; + for (int i = 0; i < valueCount; i++) { + values[i] = in.getDouble(); + } + return Array.factory(dataType, shape, values); + } + default: + throw new IllegalStateException("Unsupported data type: " + dataType); + } + } + + /** + * The zfp compression modes. + */ + public enum Mode { + REVERSIBLE("reversible"), + EXPERT("expert"), + FIXED_ACCURACY("fixed_accuracy"), + FIXED_RATE("fixed_rate"), + FIXED_PRECISION("fixed_precision"); + + private final String mode; + + Mode(String mode) { + this.mode = mode; + } + + @JsonCreator + public static Mode fromValue(String value) { + for (Mode mode : values()) { + if (mode.mode.equals(value)) { + return mode; + } + } + throw new IllegalArgumentException("Unknown zfp mode '" + value + "'."); + } + + @JsonValue + public String getValue() { + return mode; + } + } + + public static final class Configuration { + + @Nonnull + public final Mode mode; + @Nullable + public final Integer minbits; + @Nullable + public final Integer maxbits; + @Nullable + public final Integer maxprec; + @Nullable + public final Integer minexp; + @Nullable + public final Double tolerance; + @Nullable + public final Double rate; + @Nullable + public final Integer precision; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public Configuration( + @Nonnull @JsonProperty(value = "mode", required = true) Mode mode, + @Nullable @JsonProperty("minbits") Integer minbits, + @Nullable @JsonProperty("maxbits") Integer maxbits, + @Nullable @JsonProperty("maxprec") Integer maxprec, + @Nullable @JsonProperty("minexp") Integer minexp, + @Nullable @JsonProperty("tolerance") Double tolerance, + @Nullable @JsonProperty("rate") Double rate, + @Nullable @JsonProperty("precision") Integer precision + ) throws ZarrException { + if (mode == null) { + throw new ZarrException("'mode' is required for the zfp codec."); + } + this.mode = mode; + this.minbits = minbits; + this.maxbits = maxbits; + this.maxprec = maxprec; + this.minexp = minexp; + this.tolerance = tolerance; + this.rate = rate; + this.precision = precision; + + switch (mode) { + case REVERSIBLE: + requireAbsent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", + minexp, "tolerance", tolerance, "rate", rate, "precision", precision); + break; + case EXPERT: + requirePresent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", + minexp); + requireAbsent("tolerance", tolerance, "rate", rate, "precision", precision); + break; + case FIXED_ACCURACY: + requirePresent("tolerance", tolerance); + requireAbsent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", + minexp, "rate", rate, "precision", precision); + break; + case FIXED_RATE: + requirePresent("rate", rate); + requireAbsent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", + minexp, "tolerance", tolerance, "precision", precision); + if (!(rate > 0)) { + throw new ZarrException("'rate' needs to be positive, got " + rate + "."); + } + break; + case FIXED_PRECISION: + requirePresent("precision", precision); + requireAbsent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", + minexp, "tolerance", tolerance, "rate", rate); + break; + } + if (mode != Mode.FIXED_RATE) { + // Fails fast on out-of-range parameters. Fixed-rate mode needs the data type and the + // dimensionality, so it can only be resolved once the array metadata is known. + try { + resolveParams(); + } catch (IllegalArgumentException ex) { + throw new ZarrException("Invalid zfp codec configuration.", ex); + } + } + } + + public static Configuration reversible() throws ZarrException { + return new Configuration(Mode.REVERSIBLE, null, null, null, null, null, null, null); + } + + public static Configuration expert(int minbits, int maxbits, int maxprec, int minexp) + throws ZarrException { + return new Configuration(Mode.EXPERT, minbits, maxbits, maxprec, minexp, null, null, null); + } + + public static Configuration fixedAccuracy(double tolerance) throws ZarrException { + return new Configuration(Mode.FIXED_ACCURACY, null, null, null, null, tolerance, null, null); + } + + public static Configuration fixedRate(double rate) throws ZarrException { + return new Configuration(Mode.FIXED_RATE, null, null, null, null, null, rate, null); + } + + public static Configuration fixedPrecision(int precision) throws ZarrException { + return new Configuration(Mode.FIXED_PRECISION, null, null, null, null, null, null, precision); + } + + private static void requirePresent(Object... namesAndValues) throws ZarrException { + for (int i = 0; i < namesAndValues.length; i += 2) { + if (namesAndValues[i + 1] == null) { + throw new ZarrException( + "'" + namesAndValues[i] + "' is required for the zfp codec."); + } + } + } + + private static void requireAbsent(Object... namesAndValues) throws ZarrException { + for (int i = 0; i < namesAndValues.length; i += 2) { + if (namesAndValues[i + 1] != null) { + throw new ZarrException( + "'" + namesAndValues[i] + "' is not a parameter of this zfp mode."); + } + } + } + + /** + * Resolves this configuration to zfp's expert mode parameters. Not supported for fixed-rate + * mode, which additionally depends on the data type and the dimensionality of the chunk. + */ + @JsonIgnore + Zfp.Params resolveParams() { + switch (mode) { + case REVERSIBLE: + return Zfp.Params.reversible(); + case EXPERT: + return Zfp.Params.expert(minbits, maxbits, maxprec, minexp); + case FIXED_ACCURACY: + return Zfp.Params.fixedAccuracy(tolerance); + case FIXED_PRECISION: + return Zfp.Params.fixedPrecision(precision); + default: + throw new IllegalStateException( + "The zfp parameters of mode '" + mode.getValue() + "' depend on the array metadata."); + } + } + } +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 614a2b90..036f410a 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -1,5 +1,7 @@ package dev.zarr.zarrjava; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -12,10 +14,12 @@ import dev.zarr.zarrjava.v3.*; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; +import dev.zarr.zarrjava.v3.codec.CodecRegistry; import dev.zarr.zarrjava.v3.codec.core.BloscCodec; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; +import dev.zarr.zarrjava.v3.codec.core.ZfpCodec; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -27,6 +31,7 @@ import java.io.BufferedReader; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -1052,4 +1057,238 @@ public void testEndianness(DataType dataType, BytesCodec.Endian endian) throws I ucar.ma2.Array readData = reopenedArray.read(); assertIsTestdata(readData, dataType); } + + static Stream zfpDataTypeProvider() { + return dataTypeProviderV3().filter(dataType -> dataType != DataType.BOOL); + } + + static Stream zfpConfigurationJsonProvider() { + // The examples of the zfp codec specification + return Stream.of( + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"reversible\"}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"expert\",\"minbits\":1,\"maxbits\":13,\"maxprec\":19,\"minexp\":-2}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_accuracy\",\"tolerance\":0.05}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_rate\",\"rate\":10.5}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_precision\",\"precision\":19}}" + ); + } + + static Stream invalidZfpConfigurationJsonProvider() { + return Stream.of( + // Unknown mode + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_nonsense\"}}", + // Missing mode + "{\"name\":\"zfp\",\"configuration\":{\"tolerance\":0.05}}", + // Missing required parameters + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_accuracy\"}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_rate\"}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_precision\"}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"expert\",\"minbits\":1,\"maxbits\":13,\"maxprec\":19}}", + // Parameters that do not belong to the mode + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"reversible\",\"tolerance\":0.05}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_rate\",\"rate\":10.5,\"precision\":19}}", + // Parameters out of range + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"expert\",\"minbits\":13,\"maxbits\":1,\"maxprec\":19,\"minexp\":-2}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"expert\",\"minbits\":1,\"maxbits\":13,\"maxprec\":65,\"minexp\":-2}}", + "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_rate\",\"rate\":0}}" + ); + } + + static ObjectMapper zfpObjectMapper() { + // Mirrors dev.zarr.zarrjava.v3.Node#makeObjectMapper, which is not visible here + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); + objectMapper.setDefaultPropertyInclusion(JsonInclude.Include.NON_NULL); + return objectMapper; + } + + static ZfpCodec zfpCodec(ZfpCodec.Configuration configuration, DataType dataType, + int[] chunkShape) throws ZarrException { + ZfpCodec codec = new ZfpCodec(configuration); + codec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + toLongArray(chunkShape), chunkShape, dataType, null)); + return codec; + } + + static ucar.ma2.Array sineTestData(ucar.ma2.DataType dataType, int[] shape) { + ucar.ma2.Array array = ucar.ma2.Array.factory(dataType, shape); + for (int i = 0; i < array.getSize(); i++) { + array.setDouble(i, Math.sin(i * 0.01) * 100.0); + } + return array; + } + + @ParameterizedTest + @MethodSource("zfpDataTypeProvider") + public void testZfpCodecReversibleReadWrite(DataType dataType) throws ZarrException, IOException { + ucar.ma2.Array testData = testdata(dataType); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZfpCodecReversibleReadWrite", dataType.name()); + Array writeArray = Array.create(storeHandle, Array.metadataBuilder() + .withShape(toLongArray(testData.getShape())) + .withDataType(dataType) + .withChunkShape(8, 8, 8) + .withFillValue(0) + .withCodecs(CodecBuilder::withZfpReversible) + .build()); + writeArray.write(testData); + + ucar.ma2.Array readData = Array.open(storeHandle).read(); + assertIsTestdata(readData, dataType); + } + + @Test + public void testZfpCodecShardingReadWrite() throws ZarrException, IOException { + ucar.ma2.Array testData = testdata(DataType.FLOAT32); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZfpCodecShardingReadWrite"); + Array writeArray = Array.create(storeHandle, Array.metadataBuilder() + .withShape(toLongArray(testData.getShape())) + .withDataType(DataType.FLOAT32) + .withChunkShape(16, 16, 16) + .withFillValue(0) + .withCodecs(c -> c.withSharding(new int[]{4, 8, 8}, CodecBuilder::withZfpReversible)) + .build()); + writeArray.write(testData); + + ucar.ma2.Array readData = Array.open(storeHandle).read(); + assertIsTestdata(readData, DataType.FLOAT32); + } + + @ParameterizedTest + @CsvSource({"FLOAT32, 0.05", "FLOAT32, 0.5", "FLOAT64, 0.05", "FLOAT64, 0.5"}) + public void testZfpCodecFixedAccuracy(DataType dataType, double tolerance) throws ZarrException { + int[] chunkShape = new int[]{8, 16, 32}; + ucar.ma2.Array testData = sineTestData(dataType.getMA2DataType(), chunkShape); + + ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.fixedAccuracy(tolerance), dataType, chunkShape); + ByteBuffer encoded = codec.encode(testData); + ucar.ma2.Array decoded = codec.decode(encoded.duplicate()); + + Assertions.assertTrue(encoded.remaining() < testData.getSize() * dataType.getByteCount(), + "zfp should compress smooth data"); + for (int i = 0; i < testData.getSize(); i++) { + Assertions.assertEquals(testData.getDouble(i), decoded.getDouble(i), tolerance); + } + } + + @Test + public void testZfpCodecFixedRate() throws ZarrException { + int[] chunkShape = new int[]{8, 8, 8}; + ucar.ma2.Array testData = sineTestData(ucar.ma2.DataType.DOUBLE, chunkShape); + + // Every one of the 2*2*2 blocks of 4*4*4 values takes exactly 8 * 64 bits + ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.fixedRate(8), DataType.FLOAT64, chunkShape); + ByteBuffer encoded = codec.encode(testData); + Assertions.assertEquals(8 * 8 * 64 / 8, encoded.remaining()); + + ucar.ma2.Array decoded = codec.decode(encoded.duplicate()); + for (int i = 0; i < testData.getSize(); i++) { + Assertions.assertEquals(testData.getDouble(i), decoded.getDouble(i), 1.0); + } + } + + @Test + public void testZfpCodecFixedPrecision() throws ZarrException { + int[] chunkShape = new int[]{8, 16, 32}; + ucar.ma2.Array testData = sineTestData(ucar.ma2.DataType.FLOAT, chunkShape); + + ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.fixedPrecision(24), DataType.FLOAT32, chunkShape); + ByteBuffer encoded = codec.encode(testData); + ucar.ma2.Array decoded = codec.decode(encoded.duplicate()); + + Assertions.assertTrue(encoded.remaining() < testData.getSize() * Float.BYTES, + "zfp should compress smooth data"); + for (int i = 0; i < testData.getSize(); i++) { + Assertions.assertEquals(testData.getDouble(i), decoded.getDouble(i), 0.01); + } + } + + @Test + public void testZfpCodecExpert() throws ZarrException { + int[] chunkShape = new int[]{8, 16, 32}; + ucar.ma2.Array testData = sineTestData(ucar.ma2.DataType.DOUBLE, chunkShape); + + // Reversible mode expressed through its expert mode parameters + ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.expert(1, 16658, 64, -1075), DataType.FLOAT64, + chunkShape); + ucar.ma2.Array decoded = codec.decode(codec.encode(testData)); + for (int i = 0; i < testData.getSize(); i++) { + Assertions.assertEquals(testData.getDouble(i), decoded.getDouble(i)); + } + } + + @Test + public void testZfpCodecOneDimensionalAndZeroDimensional() throws ZarrException { + ucar.ma2.Array oneDimensional = sineTestData(ucar.ma2.DataType.DOUBLE, new int[]{64}); + ZfpCodec oneDimensionalCodec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.FLOAT64, + new int[]{64}); + ucar.ma2.Array decoded = oneDimensionalCodec.decode(oneDimensionalCodec.encode(oneDimensional)); + for (int i = 0; i < oneDimensional.getSize(); i++) { + Assertions.assertEquals(oneDimensional.getDouble(i), decoded.getDouble(i)); + } + + // The chunk of a zero-dimensional array is a 1D zfp field holding a single value + ucar.ma2.Array zeroDimensional = ucar.ma2.Array.factory(ucar.ma2.DataType.DOUBLE, new int[0], + new double[]{42.5}); + ZfpCodec zeroDimensionalCodec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.FLOAT64, + new int[0]); + ucar.ma2.Array decodedScalar = + zeroDimensionalCodec.decode(zeroDimensionalCodec.encode(zeroDimensional)); + Assertions.assertEquals(42.5, decodedScalar.getDouble(0)); + } + + @Test + public void testZfpCodecClampsLargeUnsignedValues() throws ZarrException { + // uint32 and uint64 values beyond the signed range are clamped, matching zarrs + int[] chunkShape = new int[]{4}; + ucar.ma2.Array uint32Data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, chunkShape, + new int[]{0, 1, Integer.MAX_VALUE, (int) 4_000_000_000L}); + ZfpCodec uint32Codec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.UINT32, chunkShape); + ucar.ma2.Array decodedUint32 = uint32Codec.decode(uint32Codec.encode(uint32Data)); + Assertions.assertArrayEquals(new long[]{0, 1, Integer.MAX_VALUE, Integer.MAX_VALUE}, + new long[]{decodedUint32.getLong(0), decodedUint32.getLong(1), decodedUint32.getLong(2), + decodedUint32.getLong(3)}); + + ucar.ma2.Array uint64Data = ucar.ma2.Array.factory(ucar.ma2.DataType.ULONG, chunkShape, + new long[]{0, 1, Long.MAX_VALUE, Long.MIN_VALUE}); + ZfpCodec uint64Codec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.UINT64, chunkShape); + ucar.ma2.Array decodedUint64 = uint64Codec.decode(uint64Codec.encode(uint64Data)); + Assertions.assertArrayEquals(new long[]{0, 1, Long.MAX_VALUE, Long.MAX_VALUE}, + new long[]{decodedUint64.getLong(0), decodedUint64.getLong(1), decodedUint64.getLong(2), + decodedUint64.getLong(3)}); + } + + @Test + public void testZfpCodecRejectsBool() throws ZarrException { + int[] chunkShape = new int[]{4, 4}; + ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.BOOL, chunkShape); + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.BOOLEAN, chunkShape); + assertThrows(ZarrException.class, () -> codec.encode(testData)); + } + + @Test + public void testZfpCodecRejectsMoreThanFourDimensions() throws ZarrException { + int[] chunkShape = new int[]{2, 2, 2, 2, 2}; + ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.FLOAT64, chunkShape); + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.DOUBLE, chunkShape); + assertThrows(ZarrException.class, () -> codec.encode(testData)); + } + + @ParameterizedTest + @MethodSource("zfpConfigurationJsonProvider") + public void testZfpCodecJsonRoundTrip(String json) throws IOException { + ObjectMapper objectMapper = zfpObjectMapper(); + Codec codec = objectMapper.readValue(json, Codec.class); + Assertions.assertInstanceOf(ZfpCodec.class, codec); + Assertions.assertEquals(objectMapper.readTree(json), + objectMapper.readTree(objectMapper.writeValueAsString(codec))); + } + + @ParameterizedTest + @MethodSource("invalidZfpConfigurationJsonProvider") + public void testZfpCodecInvalidConfiguration(String json) { + ObjectMapper objectMapper = zfpObjectMapper(); + assertThrows(JsonProcessingException.class, () -> objectMapper.readValue(json, Codec.class)); + } } From e75dc6ccb19966efc412a9642a189354788d58a2 Mon Sep 17 00:00:00 2001 From: Norman Rzepka Date: Fri, 4 Sep 2026 13:56:57 +0200 Subject: [PATCH 2/3] Fix int overflow in zfp promoted-buffer allocation The promoted-buffer size was computed as `(int) Zfp.countValues(zfpShape) * zfpType.getByteCount()`, so the multiplication happened in int and silently wrapped for large chunks. A 1024^3 int8 chunk wrapped to 0 and `promote` then threw BufferOverflowException outside the try/catch; a 512x1024x1024 chunk wrapped to Integer.MIN_VALUE and `allocate` threw IllegalArgumentException. Compute the size as a long and reject oversized chunks with a ZarrException before allocating. Co-Authored-By: Claude Opus 5 --- .../java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java index 80ebb155..f9b5efee 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java @@ -55,9 +55,15 @@ public ByteBuffer encode(Array chunkArray) throws ZarrException { final Zfp.Type zfpType = zfpType(dataType); final int[] zfpShape = zfpShape(); + final long promotedByteCount = Zfp.countValues(zfpShape) * zfpType.getByteCount(); + if (promotedByteCount > Integer.MAX_VALUE) { + throw new ZarrException( + "The chunk shape " + Arrays.toString(zfpShape) + " needs " + promotedByteCount + + " bytes as " + zfpType + " values, which exceeds the maximum Java array size of " + + Integer.MAX_VALUE + "."); + } final ByteBuffer promoted = - ByteBuffer.allocate((int) Zfp.countValues(zfpShape) * zfpType.getByteCount()) - .order(ByteOrder.nativeOrder()); + ByteBuffer.allocate((int) promotedByteCount).order(ByteOrder.nativeOrder()); promote(chunkArray, dataType, promoted); promoted.rewind(); From 454f41f87fd8388283a41f8f818cf8e85c7ca880 Mon Sep 17 00:00:00 2001 From: Konstantin Date: Wed, 28 Oct 2026 13:22:33 +0100 Subject: [PATCH 3/3] Move the zfp codec to zfp-java zfp needs a native library, which zarr-java should not force on all its users. The codec now lives in scalableminds/zfp-java and registers itself through CodecRegistry; the user guide points there. Co-Authored-By: Claude Opus 5.5 --- USERGUIDE.md | 23 +- pom.xml | 6 - .../zarr/zarrjava/v3/codec/CodecBuilder.java | 60 --- .../zarr/zarrjava/v3/codec/CodecRegistry.java | 1 - .../zarr/zarrjava/v3/codec/core/ZfpCodec.java | 485 ------------------ .../java/dev/zarr/zarrjava/ZarrV3Test.java | 239 --------- 6 files changed, 10 insertions(+), 804 deletions(-) delete mode 100644 src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java diff --git a/USERGUIDE.md b/USERGUIDE.md index 8916feee..1e04bb29 100644 --- a/USERGUIDE.md +++ b/USERGUIDE.md @@ -20,7 +20,7 @@ zarr-java is a Java implementation of the [Zarr specification](https://zarr.dev/ ### Key Features - **Full Zarr v2 and v3 support**: Read and write arrays in both formats - **Multiple storage backends**: Filesystem, HTTP, S3, ZIP, and in-memory storage -- **Compression codecs**: Blosc, Gzip, Zstd, Zfp, and more +- **Compression codecs**: Blosc, Gzip, Zstd, and more - **Sharding support**: Efficient storage for many small chunks (v3) - **Parallel I/O**: Optional parallel reading and writing for performance - **Type-safe API**: Strong typing with covariant return types @@ -449,18 +449,15 @@ Array array = Array.create( .withCodecs(c -> c.withZstd(3)) // Level 1-22 ``` #### Zfp Compression -Compresses numerical chunks with [zfp](https://zfp.io), lossless or with a chosen error bound: -```java -.withCodecs(c -> c.withZfpReversible()) // Lossless -.withCodecs(c -> c.withZfpFixedAccuracy(0.05)) // Absolute error bound -.withCodecs(c -> c.withZfpFixedRate(8)) // Compressed bits per value -.withCodecs(c -> c.withZfpFixedPrecision(19)) // Bit planes retained -.withCodecs(c -> c.withZfpExpert(1, 13, 19, -2)) // zfp's expert mode parameters -``` -Zfp replaces the `bytes` codec, so it cannot be combined with it. Chunks may have at most four -dimensions, and `bool` is not supported. Data types narrower than 32 bits are promoted to `int32`; -`uint32` and `uint64` values beyond the signed range are clamped, so they do not survive a round trip -even in reversible mode. +The [zfp](https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/zfp) codec is not part +of zarr-java, because it needs the native zfp library. It is provided by +[zfp-java](https://github.com/scalableminds/zfp-java); add that dependency and register the codec +once at startup: +```java +ZfpCodec.register(); +// ... +.withCodecs(new ZfpCodec(ZfpCodec.Configuration.reversible())) +``` #### Transpose Codec ```java .withCodecs(c -> c diff --git a/pom.xml b/pom.xml index 2de98b61..e3d04d97 100644 --- a/pom.xml +++ b/pom.xml @@ -48,7 +48,6 @@ 2.34.6 5.9.1 1.5.5-7 - 0.1-1.0.1 5.14.0 3.0.2 @@ -105,11 +104,6 @@ blosc-java 0.3-1.21.6 - - com.scalableminds - zfp-java - ${zfpVersion} - com.github.luben zstd-jni diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index c725aa9d..715d5f48 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -133,66 +133,6 @@ public CodecBuilder withZstd(int level) { return withZstd(level, true); } - public CodecBuilder withZfp(ZfpCodec.Configuration configuration) { - codecs.add(new ZfpCodec(configuration)); - return this; - } - - /** - * Compresses chunks losslessly with zfp. - */ - public CodecBuilder withZfpReversible() { - try { - return withZfp(ZfpCodec.Configuration.reversible()); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - } - - /** - * Compresses chunks with zfp, with a guaranteed absolute error bound. - */ - public CodecBuilder withZfpFixedAccuracy(double tolerance) { - try { - return withZfp(ZfpCodec.Configuration.fixedAccuracy(tolerance)); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - } - - /** - * Compresses chunks with zfp, at a fixed number of compressed bits per value. - */ - public CodecBuilder withZfpFixedRate(double rate) { - try { - return withZfp(ZfpCodec.Configuration.fixedRate(rate)); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - } - - /** - * Compresses chunks with zfp, retaining a fixed number of bit planes. - */ - public CodecBuilder withZfpFixedPrecision(int precision) { - try { - return withZfp(ZfpCodec.Configuration.fixedPrecision(precision)); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - } - - /** - * Compresses chunks with zfp, setting all four of zfp's expert mode parameters directly. - */ - public CodecBuilder withZfpExpert(int minbits, int maxbits, int maxprec, int minexp) { - try { - return withZfp(ZfpCodec.Configuration.expert(minbits, maxbits, maxprec, minexp)); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - } - public CodecBuilder withSharding(int[] chunkShape) { try { codecs.add( diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java index 8bc85cf8..94d80ae5 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java @@ -18,7 +18,6 @@ public class CodecRegistry { addType("blosc", BloscCodec.class); addType("gzip", GzipCodec.class); addType("zstd", ZstdCodec.class); - addType("zfp", ZfpCodec.class); addType("crc32c", Crc32cCodec.class); addType("sharding_indexed", ShardingIndexedCodec.class); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java deleted file mode 100644 index f9b5efee..00000000 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZfpCodec.java +++ /dev/null @@ -1,485 +0,0 @@ -package dev.zarr.zarrjava.v3.codec.core; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonValue; -import com.scalableminds.zfpjava.Zfp; -import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; -import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.v3.codec.Codec; -import ucar.ma2.Array; -import ucar.ma2.DataType; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.Arrays; - -/** - * Compresses chunks with the zfp algorithm, as specified by the - * zfp codec - * extension. - * - *

zfp itself only handles 1- to 4-dimensional arrays of {@code int32}, {@code int64}, - * {@code float32} and {@code float64}. Narrower integer data types are promoted to {@code int32} and - * demoted back with the shifts given in the specification; unsigned 32- and 64-bit values are - * clamped into the signed range, matching the reference implementation in zarrs. Clamping is lossy - * for values above {@code 2^31 - 1} respectively {@code 2^63 - 1}, even in reversible mode. - */ -public class ZfpCodec extends ArrayBytesCodec implements Codec { - - @JsonIgnore - @Nonnull - public final String name = "zfp"; - @Nonnull - public final Configuration configuration; - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public ZfpCodec( - @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration - ) { - this.configuration = configuration; - } - - private static int clamp(int value, int min, int max) { - return Math.min(Math.max(value, min), max); - } - - @Override - public ByteBuffer encode(Array chunkArray) throws ZarrException { - final DataType dataType = arrayMetadata.dataType.getMA2DataType(); - final Zfp.Type zfpType = zfpType(dataType); - final int[] zfpShape = zfpShape(); - - final long promotedByteCount = Zfp.countValues(zfpShape) * zfpType.getByteCount(); - if (promotedByteCount > Integer.MAX_VALUE) { - throw new ZarrException( - "The chunk shape " + Arrays.toString(zfpShape) + " needs " + promotedByteCount - + " bytes as " + zfpType + " values, which exceeds the maximum Java array size of " - + Integer.MAX_VALUE + "."); - } - final ByteBuffer promoted = - ByteBuffer.allocate((int) promotedByteCount).order(ByteOrder.nativeOrder()); - promote(chunkArray, dataType, promoted); - promoted.rewind(); - - try { - return ByteBuffer.wrap( - Zfp.compress(promoted.array(), zfpType, zfpShape, params(zfpType, zfpShape.length))); - } catch (RuntimeException ex) { - throw new ZarrException("Error in encoding zfp.", ex); - } - } - - @Override - public Array decode(ByteBuffer chunkBytes) throws ZarrException { - final DataType dataType = arrayMetadata.dataType.getMA2DataType(); - final Zfp.Type zfpType = zfpType(dataType); - final int[] zfpShape = zfpShape(); - - final byte[] promoted; - try { - promoted = Zfp.decompress(Utils.toArray(chunkBytes), zfpType, zfpShape, - params(zfpType, zfpShape.length)); - } catch (RuntimeException ex) { - throw new ZarrException("Error in decoding zfp.", ex); - } - return demote(ByteBuffer.wrap(promoted).order(ByteOrder.nativeOrder()), dataType, - (int) Zfp.countValues(zfpShape)); - } - - @Override - public long computeEncodedSize(long inputByteLength, - ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - throw new ZarrException("Not implemented for Zfp codec."); - } - - /** - * The zfp scalar type the chunk's values are compressed as. Data types narrower than 32 bits are - * compressed as {@code int32}, unsigned types as their signed counterpart. - */ - private Zfp.Type zfpType(DataType dataType) throws ZarrException { - switch (dataType) { - case BYTE: - case UBYTE: - case SHORT: - case USHORT: - case INT: - case UINT: - return Zfp.Type.INT32; - case LONG: - case ULONG: - return Zfp.Type.INT64; - case FLOAT: - return Zfp.Type.FLOAT; - case DOUBLE: - return Zfp.Type.DOUBLE; - default: - throw new ZarrException( - "The zfp codec does not support the data type '" + arrayMetadata.dataType + "'."); - } - } - - /** - * The chunk shape as a zfp field shape, in row-major order. The chunk of a zero-dimensional array - * is a 1D field holding a single value. - */ - private int[] zfpShape() throws ZarrException { - final int[] chunkShape = arrayMetadata.chunkShape; - if (chunkShape.length == 0) { - return new int[]{1}; - } - if (chunkShape.length > Zfp.MAX_DIMS) { - throw new ZarrException( - "The zfp codec supports at most " + Zfp.MAX_DIMS + " dimensions, but the chunk shape " - + Arrays.toString(chunkShape) + " has " + chunkShape.length + "."); - } - return chunkShape; - } - - /** - * Resolves the configured compression mode to zfp's expert mode parameters. Fixed-rate mode - * depends on the zfp type and the dimensionality, which is why this cannot happen while parsing - * the configuration. - */ - private Zfp.Params params(Zfp.Type zfpType, int dims) throws ZarrException { - try { - switch (configuration.mode) { - case FIXED_RATE: - return Zfp.Params.fixedRate(configuration.rate, zfpType, dims); - default: - return configuration.resolveParams(); - } - } catch (IllegalArgumentException ex) { - throw new ZarrException("Invalid zfp codec configuration.", ex); - } - } - - /** - * Writes the chunk's values into {@code out} as zfp scalars, promoting narrower integers into - * {@code int32} as specified. - */ - private void promote(Array chunkArray, DataType dataType, ByteBuffer out) { - switch (dataType) { - case BYTE: { - final byte[] values = (byte[]) chunkArray.copyTo1DJavaArray(); - for (byte value : values) { - out.putInt(value << 23); - } - break; - } - case UBYTE: { - final byte[] values = (byte[]) chunkArray.copyTo1DJavaArray(); - for (byte value : values) { - out.putInt(((value & 0xFF) - 0x80) << 23); - } - break; - } - case SHORT: { - final short[] values = (short[]) chunkArray.copyTo1DJavaArray(); - for (short value : values) { - out.putInt(value << 15); - } - break; - } - case USHORT: { - final short[] values = (short[]) chunkArray.copyTo1DJavaArray(); - for (short value : values) { - out.putInt(((value & 0xFFFF) - 0x8000) << 15); - } - break; - } - case INT: { - final int[] values = (int[]) chunkArray.copyTo1DJavaArray(); - for (int value : values) { - out.putInt(value); - } - break; - } - case UINT: { - final int[] values = (int[]) chunkArray.copyTo1DJavaArray(); - for (int value : values) { - // Values above Integer.MAX_VALUE have their sign bit set and are clamped - out.putInt(value < 0 ? Integer.MAX_VALUE : value); - } - break; - } - case LONG: { - final long[] values = (long[]) chunkArray.copyTo1DJavaArray(); - for (long value : values) { - out.putLong(value); - } - break; - } - case ULONG: { - final long[] values = (long[]) chunkArray.copyTo1DJavaArray(); - for (long value : values) { - // Values above Long.MAX_VALUE have their sign bit set and are clamped - out.putLong(value < 0 ? Long.MAX_VALUE : value); - } - break; - } - case FLOAT: { - final float[] values = (float[]) chunkArray.copyTo1DJavaArray(); - for (float value : values) { - out.putFloat(value); - } - break; - } - case DOUBLE: { - final double[] values = (double[]) chunkArray.copyTo1DJavaArray(); - for (double value : values) { - out.putDouble(value); - } - break; - } - default: - throw new IllegalStateException("Unsupported data type: " + dataType); - } - } - - /** - * Reads {@code valueCount} zfp scalars from {@code in} and demotes them back into the chunk's data - * type. - */ - private Array demote(ByteBuffer in, DataType dataType, int valueCount) { - final int[] shape = arrayMetadata.chunkShape; - switch (dataType) { - case BYTE: - case UBYTE: { - final byte[] values = new byte[valueCount]; - for (int i = 0; i < valueCount; i++) { - final int value = in.getInt() >> 23; - values[i] = dataType == DataType.BYTE - ? (byte) clamp(value, -0x80, 0x7F) - : (byte) clamp(value + 0x80, 0x00, 0xFF); - } - return Array.factory(dataType, shape, values); - } - case SHORT: - case USHORT: { - final short[] values = new short[valueCount]; - for (int i = 0; i < valueCount; i++) { - final int value = in.getInt() >> 15; - values[i] = dataType == DataType.SHORT - ? (short) clamp(value, -0x8000, 0x7FFF) - : (short) clamp(value + 0x8000, 0x0000, 0xFFFF); - } - return Array.factory(dataType, shape, values); - } - case INT: - case UINT: { - final int[] values = new int[valueCount]; - for (int i = 0; i < valueCount; i++) { - final int value = in.getInt(); - values[i] = dataType == DataType.INT ? value : Math.max(value, 0); - } - return Array.factory(dataType, shape, values); - } - case LONG: - case ULONG: { - final long[] values = new long[valueCount]; - for (int i = 0; i < valueCount; i++) { - final long value = in.getLong(); - values[i] = dataType == DataType.LONG ? value : Math.max(value, 0); - } - return Array.factory(dataType, shape, values); - } - case FLOAT: { - final float[] values = new float[valueCount]; - for (int i = 0; i < valueCount; i++) { - values[i] = in.getFloat(); - } - return Array.factory(dataType, shape, values); - } - case DOUBLE: { - final double[] values = new double[valueCount]; - for (int i = 0; i < valueCount; i++) { - values[i] = in.getDouble(); - } - return Array.factory(dataType, shape, values); - } - default: - throw new IllegalStateException("Unsupported data type: " + dataType); - } - } - - /** - * The zfp compression modes. - */ - public enum Mode { - REVERSIBLE("reversible"), - EXPERT("expert"), - FIXED_ACCURACY("fixed_accuracy"), - FIXED_RATE("fixed_rate"), - FIXED_PRECISION("fixed_precision"); - - private final String mode; - - Mode(String mode) { - this.mode = mode; - } - - @JsonCreator - public static Mode fromValue(String value) { - for (Mode mode : values()) { - if (mode.mode.equals(value)) { - return mode; - } - } - throw new IllegalArgumentException("Unknown zfp mode '" + value + "'."); - } - - @JsonValue - public String getValue() { - return mode; - } - } - - public static final class Configuration { - - @Nonnull - public final Mode mode; - @Nullable - public final Integer minbits; - @Nullable - public final Integer maxbits; - @Nullable - public final Integer maxprec; - @Nullable - public final Integer minexp; - @Nullable - public final Double tolerance; - @Nullable - public final Double rate; - @Nullable - public final Integer precision; - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public Configuration( - @Nonnull @JsonProperty(value = "mode", required = true) Mode mode, - @Nullable @JsonProperty("minbits") Integer minbits, - @Nullable @JsonProperty("maxbits") Integer maxbits, - @Nullable @JsonProperty("maxprec") Integer maxprec, - @Nullable @JsonProperty("minexp") Integer minexp, - @Nullable @JsonProperty("tolerance") Double tolerance, - @Nullable @JsonProperty("rate") Double rate, - @Nullable @JsonProperty("precision") Integer precision - ) throws ZarrException { - if (mode == null) { - throw new ZarrException("'mode' is required for the zfp codec."); - } - this.mode = mode; - this.minbits = minbits; - this.maxbits = maxbits; - this.maxprec = maxprec; - this.minexp = minexp; - this.tolerance = tolerance; - this.rate = rate; - this.precision = precision; - - switch (mode) { - case REVERSIBLE: - requireAbsent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", - minexp, "tolerance", tolerance, "rate", rate, "precision", precision); - break; - case EXPERT: - requirePresent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", - minexp); - requireAbsent("tolerance", tolerance, "rate", rate, "precision", precision); - break; - case FIXED_ACCURACY: - requirePresent("tolerance", tolerance); - requireAbsent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", - minexp, "rate", rate, "precision", precision); - break; - case FIXED_RATE: - requirePresent("rate", rate); - requireAbsent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", - minexp, "tolerance", tolerance, "precision", precision); - if (!(rate > 0)) { - throw new ZarrException("'rate' needs to be positive, got " + rate + "."); - } - break; - case FIXED_PRECISION: - requirePresent("precision", precision); - requireAbsent("minbits", minbits, "maxbits", maxbits, "maxprec", maxprec, "minexp", - minexp, "tolerance", tolerance, "rate", rate); - break; - } - if (mode != Mode.FIXED_RATE) { - // Fails fast on out-of-range parameters. Fixed-rate mode needs the data type and the - // dimensionality, so it can only be resolved once the array metadata is known. - try { - resolveParams(); - } catch (IllegalArgumentException ex) { - throw new ZarrException("Invalid zfp codec configuration.", ex); - } - } - } - - public static Configuration reversible() throws ZarrException { - return new Configuration(Mode.REVERSIBLE, null, null, null, null, null, null, null); - } - - public static Configuration expert(int minbits, int maxbits, int maxprec, int minexp) - throws ZarrException { - return new Configuration(Mode.EXPERT, minbits, maxbits, maxprec, minexp, null, null, null); - } - - public static Configuration fixedAccuracy(double tolerance) throws ZarrException { - return new Configuration(Mode.FIXED_ACCURACY, null, null, null, null, tolerance, null, null); - } - - public static Configuration fixedRate(double rate) throws ZarrException { - return new Configuration(Mode.FIXED_RATE, null, null, null, null, null, rate, null); - } - - public static Configuration fixedPrecision(int precision) throws ZarrException { - return new Configuration(Mode.FIXED_PRECISION, null, null, null, null, null, null, precision); - } - - private static void requirePresent(Object... namesAndValues) throws ZarrException { - for (int i = 0; i < namesAndValues.length; i += 2) { - if (namesAndValues[i + 1] == null) { - throw new ZarrException( - "'" + namesAndValues[i] + "' is required for the zfp codec."); - } - } - } - - private static void requireAbsent(Object... namesAndValues) throws ZarrException { - for (int i = 0; i < namesAndValues.length; i += 2) { - if (namesAndValues[i + 1] != null) { - throw new ZarrException( - "'" + namesAndValues[i] + "' is not a parameter of this zfp mode."); - } - } - } - - /** - * Resolves this configuration to zfp's expert mode parameters. Not supported for fixed-rate - * mode, which additionally depends on the data type and the dimensionality of the chunk. - */ - @JsonIgnore - Zfp.Params resolveParams() { - switch (mode) { - case REVERSIBLE: - return Zfp.Params.reversible(); - case EXPERT: - return Zfp.Params.expert(minbits, maxbits, maxprec, minexp); - case FIXED_ACCURACY: - return Zfp.Params.fixedAccuracy(tolerance); - case FIXED_PRECISION: - return Zfp.Params.fixedPrecision(precision); - default: - throw new IllegalStateException( - "The zfp parameters of mode '" + mode.getValue() + "' depend on the array metadata."); - } - } - } -} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 036f410a..614a2b90 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -1,7 +1,5 @@ package dev.zarr.zarrjava; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonMappingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; @@ -14,12 +12,10 @@ import dev.zarr.zarrjava.v3.*; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; -import dev.zarr.zarrjava.v3.codec.CodecRegistry; import dev.zarr.zarrjava.v3.codec.core.BloscCodec; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; -import dev.zarr.zarrjava.v3.codec.core.ZfpCodec; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -31,7 +27,6 @@ import java.io.BufferedReader; import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; @@ -1057,238 +1052,4 @@ public void testEndianness(DataType dataType, BytesCodec.Endian endian) throws I ucar.ma2.Array readData = reopenedArray.read(); assertIsTestdata(readData, dataType); } - - static Stream zfpDataTypeProvider() { - return dataTypeProviderV3().filter(dataType -> dataType != DataType.BOOL); - } - - static Stream zfpConfigurationJsonProvider() { - // The examples of the zfp codec specification - return Stream.of( - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"reversible\"}}", - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"expert\",\"minbits\":1,\"maxbits\":13,\"maxprec\":19,\"minexp\":-2}}", - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_accuracy\",\"tolerance\":0.05}}", - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_rate\",\"rate\":10.5}}", - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_precision\",\"precision\":19}}" - ); - } - - static Stream invalidZfpConfigurationJsonProvider() { - return Stream.of( - // Unknown mode - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_nonsense\"}}", - // Missing mode - "{\"name\":\"zfp\",\"configuration\":{\"tolerance\":0.05}}", - // Missing required parameters - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_accuracy\"}}", - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_rate\"}}", - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_precision\"}}", - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"expert\",\"minbits\":1,\"maxbits\":13,\"maxprec\":19}}", - // Parameters that do not belong to the mode - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"reversible\",\"tolerance\":0.05}}", - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_rate\",\"rate\":10.5,\"precision\":19}}", - // Parameters out of range - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"expert\",\"minbits\":13,\"maxbits\":1,\"maxprec\":19,\"minexp\":-2}}", - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"expert\",\"minbits\":1,\"maxbits\":13,\"maxprec\":65,\"minexp\":-2}}", - "{\"name\":\"zfp\",\"configuration\":{\"mode\":\"fixed_rate\",\"rate\":0}}" - ); - } - - static ObjectMapper zfpObjectMapper() { - // Mirrors dev.zarr.zarrjava.v3.Node#makeObjectMapper, which is not visible here - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); - objectMapper.setDefaultPropertyInclusion(JsonInclude.Include.NON_NULL); - return objectMapper; - } - - static ZfpCodec zfpCodec(ZfpCodec.Configuration configuration, DataType dataType, - int[] chunkShape) throws ZarrException { - ZfpCodec codec = new ZfpCodec(configuration); - codec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - toLongArray(chunkShape), chunkShape, dataType, null)); - return codec; - } - - static ucar.ma2.Array sineTestData(ucar.ma2.DataType dataType, int[] shape) { - ucar.ma2.Array array = ucar.ma2.Array.factory(dataType, shape); - for (int i = 0; i < array.getSize(); i++) { - array.setDouble(i, Math.sin(i * 0.01) * 100.0); - } - return array; - } - - @ParameterizedTest - @MethodSource("zfpDataTypeProvider") - public void testZfpCodecReversibleReadWrite(DataType dataType) throws ZarrException, IOException { - ucar.ma2.Array testData = testdata(dataType); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZfpCodecReversibleReadWrite", dataType.name()); - Array writeArray = Array.create(storeHandle, Array.metadataBuilder() - .withShape(toLongArray(testData.getShape())) - .withDataType(dataType) - .withChunkShape(8, 8, 8) - .withFillValue(0) - .withCodecs(CodecBuilder::withZfpReversible) - .build()); - writeArray.write(testData); - - ucar.ma2.Array readData = Array.open(storeHandle).read(); - assertIsTestdata(readData, dataType); - } - - @Test - public void testZfpCodecShardingReadWrite() throws ZarrException, IOException { - ucar.ma2.Array testData = testdata(DataType.FLOAT32); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZfpCodecShardingReadWrite"); - Array writeArray = Array.create(storeHandle, Array.metadataBuilder() - .withShape(toLongArray(testData.getShape())) - .withDataType(DataType.FLOAT32) - .withChunkShape(16, 16, 16) - .withFillValue(0) - .withCodecs(c -> c.withSharding(new int[]{4, 8, 8}, CodecBuilder::withZfpReversible)) - .build()); - writeArray.write(testData); - - ucar.ma2.Array readData = Array.open(storeHandle).read(); - assertIsTestdata(readData, DataType.FLOAT32); - } - - @ParameterizedTest - @CsvSource({"FLOAT32, 0.05", "FLOAT32, 0.5", "FLOAT64, 0.05", "FLOAT64, 0.5"}) - public void testZfpCodecFixedAccuracy(DataType dataType, double tolerance) throws ZarrException { - int[] chunkShape = new int[]{8, 16, 32}; - ucar.ma2.Array testData = sineTestData(dataType.getMA2DataType(), chunkShape); - - ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.fixedAccuracy(tolerance), dataType, chunkShape); - ByteBuffer encoded = codec.encode(testData); - ucar.ma2.Array decoded = codec.decode(encoded.duplicate()); - - Assertions.assertTrue(encoded.remaining() < testData.getSize() * dataType.getByteCount(), - "zfp should compress smooth data"); - for (int i = 0; i < testData.getSize(); i++) { - Assertions.assertEquals(testData.getDouble(i), decoded.getDouble(i), tolerance); - } - } - - @Test - public void testZfpCodecFixedRate() throws ZarrException { - int[] chunkShape = new int[]{8, 8, 8}; - ucar.ma2.Array testData = sineTestData(ucar.ma2.DataType.DOUBLE, chunkShape); - - // Every one of the 2*2*2 blocks of 4*4*4 values takes exactly 8 * 64 bits - ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.fixedRate(8), DataType.FLOAT64, chunkShape); - ByteBuffer encoded = codec.encode(testData); - Assertions.assertEquals(8 * 8 * 64 / 8, encoded.remaining()); - - ucar.ma2.Array decoded = codec.decode(encoded.duplicate()); - for (int i = 0; i < testData.getSize(); i++) { - Assertions.assertEquals(testData.getDouble(i), decoded.getDouble(i), 1.0); - } - } - - @Test - public void testZfpCodecFixedPrecision() throws ZarrException { - int[] chunkShape = new int[]{8, 16, 32}; - ucar.ma2.Array testData = sineTestData(ucar.ma2.DataType.FLOAT, chunkShape); - - ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.fixedPrecision(24), DataType.FLOAT32, chunkShape); - ByteBuffer encoded = codec.encode(testData); - ucar.ma2.Array decoded = codec.decode(encoded.duplicate()); - - Assertions.assertTrue(encoded.remaining() < testData.getSize() * Float.BYTES, - "zfp should compress smooth data"); - for (int i = 0; i < testData.getSize(); i++) { - Assertions.assertEquals(testData.getDouble(i), decoded.getDouble(i), 0.01); - } - } - - @Test - public void testZfpCodecExpert() throws ZarrException { - int[] chunkShape = new int[]{8, 16, 32}; - ucar.ma2.Array testData = sineTestData(ucar.ma2.DataType.DOUBLE, chunkShape); - - // Reversible mode expressed through its expert mode parameters - ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.expert(1, 16658, 64, -1075), DataType.FLOAT64, - chunkShape); - ucar.ma2.Array decoded = codec.decode(codec.encode(testData)); - for (int i = 0; i < testData.getSize(); i++) { - Assertions.assertEquals(testData.getDouble(i), decoded.getDouble(i)); - } - } - - @Test - public void testZfpCodecOneDimensionalAndZeroDimensional() throws ZarrException { - ucar.ma2.Array oneDimensional = sineTestData(ucar.ma2.DataType.DOUBLE, new int[]{64}); - ZfpCodec oneDimensionalCodec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.FLOAT64, - new int[]{64}); - ucar.ma2.Array decoded = oneDimensionalCodec.decode(oneDimensionalCodec.encode(oneDimensional)); - for (int i = 0; i < oneDimensional.getSize(); i++) { - Assertions.assertEquals(oneDimensional.getDouble(i), decoded.getDouble(i)); - } - - // The chunk of a zero-dimensional array is a 1D zfp field holding a single value - ucar.ma2.Array zeroDimensional = ucar.ma2.Array.factory(ucar.ma2.DataType.DOUBLE, new int[0], - new double[]{42.5}); - ZfpCodec zeroDimensionalCodec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.FLOAT64, - new int[0]); - ucar.ma2.Array decodedScalar = - zeroDimensionalCodec.decode(zeroDimensionalCodec.encode(zeroDimensional)); - Assertions.assertEquals(42.5, decodedScalar.getDouble(0)); - } - - @Test - public void testZfpCodecClampsLargeUnsignedValues() throws ZarrException { - // uint32 and uint64 values beyond the signed range are clamped, matching zarrs - int[] chunkShape = new int[]{4}; - ucar.ma2.Array uint32Data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, chunkShape, - new int[]{0, 1, Integer.MAX_VALUE, (int) 4_000_000_000L}); - ZfpCodec uint32Codec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.UINT32, chunkShape); - ucar.ma2.Array decodedUint32 = uint32Codec.decode(uint32Codec.encode(uint32Data)); - Assertions.assertArrayEquals(new long[]{0, 1, Integer.MAX_VALUE, Integer.MAX_VALUE}, - new long[]{decodedUint32.getLong(0), decodedUint32.getLong(1), decodedUint32.getLong(2), - decodedUint32.getLong(3)}); - - ucar.ma2.Array uint64Data = ucar.ma2.Array.factory(ucar.ma2.DataType.ULONG, chunkShape, - new long[]{0, 1, Long.MAX_VALUE, Long.MIN_VALUE}); - ZfpCodec uint64Codec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.UINT64, chunkShape); - ucar.ma2.Array decodedUint64 = uint64Codec.decode(uint64Codec.encode(uint64Data)); - Assertions.assertArrayEquals(new long[]{0, 1, Long.MAX_VALUE, Long.MAX_VALUE}, - new long[]{decodedUint64.getLong(0), decodedUint64.getLong(1), decodedUint64.getLong(2), - decodedUint64.getLong(3)}); - } - - @Test - public void testZfpCodecRejectsBool() throws ZarrException { - int[] chunkShape = new int[]{4, 4}; - ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.BOOL, chunkShape); - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.BOOLEAN, chunkShape); - assertThrows(ZarrException.class, () -> codec.encode(testData)); - } - - @Test - public void testZfpCodecRejectsMoreThanFourDimensions() throws ZarrException { - int[] chunkShape = new int[]{2, 2, 2, 2, 2}; - ZfpCodec codec = zfpCodec(ZfpCodec.Configuration.reversible(), DataType.FLOAT64, chunkShape); - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.DOUBLE, chunkShape); - assertThrows(ZarrException.class, () -> codec.encode(testData)); - } - - @ParameterizedTest - @MethodSource("zfpConfigurationJsonProvider") - public void testZfpCodecJsonRoundTrip(String json) throws IOException { - ObjectMapper objectMapper = zfpObjectMapper(); - Codec codec = objectMapper.readValue(json, Codec.class); - Assertions.assertInstanceOf(ZfpCodec.class, codec); - Assertions.assertEquals(objectMapper.readTree(json), - objectMapper.readTree(objectMapper.writeValueAsString(codec))); - } - - @ParameterizedTest - @MethodSource("invalidZfpConfigurationJsonProvider") - public void testZfpCodecInvalidConfiguration(String json) { - ObjectMapper objectMapper = zfpObjectMapper(); - assertThrows(JsonProcessingException.class, () -> objectMapper.readValue(json, Codec.class)); - } }