Merge pull request #12 from asdf-format/eslavich-lz4-compressor

LZ4 compressor + generalize reference file testing

Author: eslavich

.github/workflows/publish.yml

@@ -28,3 +28,7 @@ jobs:
         run: mvn --batch-mode deploy
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }}
+          OSSRH_TOKEN: ${{ secrets.OSSRH_TOKEN }}
+          MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
+          MAVEN_GPG_KEY: ${{ secrets.MAVEN_GPG_KEY }}

.github/workflows/test.yml

@@ -31,9 +31,9 @@ jobs:
         with:
           python-version: '3.12'
 
-      - name: Install test file generator requirements
+      - name: Install reference file generator requirements
         run: |
-          pip install asdf numpy
+          pip install asdf numpy lz4
 
       - name: Locate Python
         id: locate-python

asdf-compression-commons-compress/pom.xml

@@ -10,7 +10,7 @@
     <artifactId>asdf-compression-commons-compress</artifactId>
 
     <name>asdf-compression-commons-compress</name>
-    <description>Support for BZip2 and LZ4 Frame compressed blocks</description>
+    <description>Support for BZip2 and LZ4 compressed blocks</description>
 
     <dependencies>
         <dependency>

asdf-compression-commons-compress/src/main/java/org/asdfformat/asdf/io/compression/Lz4Compressor.java

@@ -0,0 +1,46 @@
+package org.asdfformat.asdf.io.compression;
+
+import org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorInputStream;
+import org.asdfformat.asdf.io.util.IOUtils;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+
+public class Lz4Compressor implements Compressor {
+    public static final byte[] IDENTIFIER = {108, 122, 52, 0}; // 'lz4' + padding
+
+    @Override
+    public byte[] getIdentifier() {
+        return IDENTIFIER;
+    }
+
+    @Override
+    public long decompress(final ByteBuffer inputBuffer, final ByteBuffer outputBuffer) throws IOException {
+        long bytesDecompressed = 0L;
+
+        while (inputBuffer.hasRemaining()) {
+            inputBuffer.order(ByteOrder.BIG_ENDIAN);
+            final int lz4BlockLength = inputBuffer.getInt() - 4;
+            if (lz4BlockLength < 0) {
+                throw new RuntimeException("LZ4 block length > " + Integer.MAX_VALUE + " not supported");
+            }
+
+            // Discard the uncompressed data size written by the Python LZ4 bindings:
+            inputBuffer.getInt();
+
+            final ByteBuffer lz4BlockInputBuffer = inputBuffer.duplicate();
+            lz4BlockInputBuffer.limit(inputBuffer.position() + lz4BlockLength);
+
+            try (final ByteBufferInputStream byteBufferInputStream = new ByteBufferInputStream(lz4BlockInputBuffer);
+                 final BlockLZ4CompressorInputStream blockLZ4CompressorInputStream = new BlockLZ4CompressorInputStream(byteBufferInputStream)
+            ) {
+                bytesDecompressed += IOUtils.transferTo(blockLZ4CompressorInputStream, outputBuffer);
+            }
+
+            inputBuffer.position(inputBuffer.position() + lz4BlockLength);
+        }
+
+        return bytesDecompressed;
+    }
+}

asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/Lz4CompressorReferenceTest.java

@@ -0,0 +1,35 @@
+package org.asdfformat.asdf.io.compression;
+
+import org.asdfformat.asdf.Asdf;
+import org.asdfformat.asdf.AsdfFile;
+import org.asdfformat.asdf.io.compression.testing.CommonsCompressReferenceFileType;
+import org.asdfformat.asdf.ndarray.DoubleNdArray;
+import org.asdfformat.asdf.standard.AsdfStandardType;
+import org.asdfformat.asdf.testing.ReferenceFileUtils;
+import org.junit.jupiter.api.Tag;
+import org.junitpioneer.jupiter.cartesian.CartesianTest;
+
+import java.io.IOException;
+import java.nio.file.Path;
+
+import static org.asdfformat.asdf.io.compression.testing.TestCategories.REFERENCE_TESTS;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+@Tag(REFERENCE_TESTS)
+public class Lz4CompressorReferenceTest {
+    @CartesianTest
+    public void test1d(
+            @CartesianTest.Enum(value = CommonsCompressReferenceFileType.class, names = {"NDARRAY_COMPRESSED_LZ4"}) final CommonsCompressReferenceFileType referenceFileType,
+            @CartesianTest.Enum(AsdfStandardType.class) final AsdfStandardType asdfStandardType
+    ) throws IOException {
+        final Path path = ReferenceFileUtils.getPath(referenceFileType, asdfStandardType.getVersion());
+
+        try (final AsdfFile asdfFile = Asdf.open(path)) {
+            final DoubleNdArray doubleNdArray = asdfFile.getTree().get("arr").asNdArray().asDoubleNdArray();
+
+            for (int i = 0; i < 10; i++) {
+                assertEquals(doubleNdArray.get(i), i);
+            }
+        }
+    }
+}

asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/testing/CommonsCompressReferenceFileType.java

@@ -0,0 +1,23 @@
+package org.asdfformat.asdf.io.compression.testing;
+
+import org.asdfformat.asdf.testing.ReferenceFile;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public enum CommonsCompressReferenceFileType implements ReferenceFile {
+    NDARRAY_COMPRESSED_LZ4,
+    ;
+
+    @Override
+    public String getName() {
+        return name();
+    }
+
+    @Override
+    public InputStream openScript() throws IOException {
+        return CommonsCompressReferenceFileType.class.getResourceAsStream(
+                String.format("/test-file-scripts/%s.py", name().toLowerCase())
+        );
+    }
+}

asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/testing/TestCategories.java

@@ -0,0 +1,5 @@
+package org.asdfformat.asdf.io.compression.testing;
+
+public class TestCategories {
+    public static final String REFERENCE_TESTS = "reference-tests";
+}

asdf-compression-commons-compress/src/test/resources/test-file-scripts/ndarray_compressed_lz4.py

@@ -0,0 +1,3 @@
+af["arr"] = np.arange(0, 10, dtype=np.float64)
+
+af.set_array_compression(af["arr"], "lz4")

asdf-core/src/main/java/org/asdfformat/asdf/io/compression/Compressors.java

@@ -22,6 +22,11 @@ public class Compressors {
                     .identifier("bzp2".getBytes(StandardCharsets.UTF_8))
                     .moduleName("asdf-compression-commons-compress")
                     .build(),
+            OptionalCompressor.builder()
+                    .className("org.asdfformat.asdf.io.compression.Lz4Compressor")
+                    .identifier(new byte[] {108, 122, 52, 0})
+                    .moduleName("asdf-compression-commons-compress")
+                    .build(),
             OptionalCompressor.builder()
                     .className("org.asdfformat.asdf.io.compression.Lz4FrameCompressor")
                     .identifier("lz4f".getBytes(StandardCharsets.UTF_8))

asdf-core/src/main/java/org/asdfformat/asdf/testing/ReferenceFile.java

@@ -0,0 +1,9 @@
+package org.asdfformat.asdf.testing;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+public interface ReferenceFile {
+    String getName();
+    InputStream openScript() throws IOException;
+}