Add lz4hdf5

README.md

@@ -49,6 +49,15 @@ CONFIG_SITE.local.* files in in the configure/ directory of the appropriate rele
 [top-level areaDetector](https://github.com/areaDetector/areaDetector) repository
 or in ADSupport/configure/CONFIG_SITE.*.
 
+This module also builds a library for the HDF5 version of the LZ4 codec which compresses
+in block units.  The library is called lz4hdf5.
+This codec adds a 12-byte header that contains the size of the uncompressed data and the block size.
+Each block contains a 4-byte header that contains the compressed length of that block.
+[Documentation](https://github.com/dectris/HDF5Plugin/blob/master/HDF5_LZ4.pdf)
+The code for this library is derived from the 
+[H5Zlz4.c file](https://github.com/nexusformat/HDF5-External-Filter-Plugins/blob/master/LZ4/src/H5Zlz4.c)
+in the HDF5 library.
+
 This module also builds shareable libraries for HDF5 compression filter plugins.  
 These plugins can be used with any HDF5 application built with HDF5 1.8.11 or later.
 The plugin libraries that are built are HDF5_blosc_plugin.so, HDF5_bshuf_plugin.so,

RELEASE.md

@@ -12,6 +12,14 @@ the EXAMPLE_RELEASE_PATHS.local, EXAMPLE_RELEASE_LIBS.local, and EXAMPLE_RELEASE
 files respectively, in the configure/ directory of the appropriate release of the 
 [top-level areaDetector](https://github.com/areaDetector/areaDetector) repository.
 
+## __R1-11 (April XXX, 2026)__
+  * Added support for the HDF5 version of the LZ4 codec which compress in blocks.
+    The code is modified from the HFZlz4.c file in the HDF external filter plugins package.
+    This is the codec used by Dectris for LZ4 compression on the Stream2 interface.
+  * Minor changes to support building with mingw32.
+  * Made X11 support in GraphicsMagick optional to support cross-compiling.
+  * Fixed incompatible pointer type in GraphicsMagick libjasper that caused error on RHEL 10.
+
 ## __R1-10 (May 26, 2021)__
   * Changed the support for reading MJPEG streams in GraphicsMagickSrc and xml2Src.
     In R1-5 nanohttp.c and nanohppt.h in xml2Src were changed to support MJPEG streams.

supportApp/Makefile

@@ -39,5 +39,8 @@ GraphicsMagickSrc_DEPEND_DIRS += tiffSrc zlibSrc xml2Src jpegSrc
 
 DIRS += cbfSrc
 
+DIRS += lz4hdf5Src
+lz4hdf5Src_DEPEND_DIRS += bitshuffleSrc
+
 include $(TOP)/configure/RULES_DIRS
 

supportApp/lz4hdf5Src/Makefile

@@ -0,0 +1,16 @@
+TOP=../..
+include $(TOP)/configure/CONFIG
+#----------------------------------------
+#  ADD MACRO DEFINITIONS AFTER THIS LINE
+#=============================
+
+INC += lz4hdf5.h
+
+LIBRARY_IOC  += lz4hdf5
+lz4hdf5_SRCS += lz4hdf5.c
+lz4hdf5_LIBS += bitshuffle
+lz4hdf5_SYS_LIBS_WIN32 += ws2_32
+
+include $(TOP)/configure/RULES
+#----------------------------------------
+#  ADD RULES AFTER THIS LINE

supportApp/lz4hdf5Src/lz4hdf5.c

@@ -0,0 +1,173 @@
+/*
+ * This file implements a version of the lz4 codec where the data is compressed in blocks.
+ * This codec adds a 12-byte header that contains the size of the uncompressed data and the block size.
+ * Each block contains a 4-byte header that contains the compressed length of that block.
+ * This codec is used by the HDF5 library and by Dectris for the Stream2 interface on their detectors.
+ * The documentation is here: https://github.com/dectris/HDF5Plugin/blob/master/HDF5_LZ4.pdf
+ * This code for this library is derived from the H5Zlz4.c file in the HDF5 External Filter Plugins library.
+ * https://github.com/nexusformat/HDF5-External-Filter-Plugins/blob/master/LZ4/src/H5Zlz4.c
+ */
+
+#include <sys/types.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#if defined(_WIN32)
+#include <winsock2.h>
+#else
+#include <arpa/inet.h>
+#endif
+#include <lz4.h>
+#include <epicsExport.h>
+#include <lz4hdf5.h>
+
+#define htonll(x) ( ( (uint64_t)(htonl( (uint32_t)((x << 32) >> 32)))<< 32) | htonl( ((uint32_t)(x >> 32)) ))
+#define ntohll(x) htonll(x)
+
+#define htobe16t(x) htons(x)
+#define htobe32t(x) htonl(x)
+#define htobe64t(x) htonll(x)
+#define be16toht(x) ntohs(x)
+#define be32toht(x) ntohl(x)
+#define be64toht(x) ntohll(x)
+
+
+#define DEFAULT_BLOCK_SIZE 1<<30; /* 1GB. LZ4 needs blocks < 1.9GB. */
+
+epicsShareFunc size_t decompress_lz4hdf5(const char *inbuf, char *outbuf, size_t maxOutputSize, size_t *blockSizeOut)
+{
+    size_t ret_value;
+
+    uint32_t *i32Buf;
+    uint32_t blockSize;
+    char *roBuf;   /* pointer to current write position */
+    uint64_t decompSize;
+    const char* rpos = inbuf; /* pointer to current read position */
+    const uint64_t * const i64Buf = (uint64_t *) rpos;
+    const uint64_t origSize = (uint64_t)(be64toht(*i64Buf));/* is saved in be format */
+    if (origSize > maxOutputSize) {
+        printf("maxOutputSize=%d too small, needs to be at least %d\n", (int)maxOutputSize, (int)origSize);
+        return 0;
+    }
+    rpos += 8; /* advance the pointer */
+
+    i32Buf = (uint32_t*)rpos;
+    blockSize = (uint32_t)(be32toht(*i32Buf));
+    rpos += 4;
+    if(blockSize>origSize)
+        blockSize = (uint32_t)origSize;
+    *blockSizeOut = blockSize;
+    roBuf = (char*)outbuf;   /* pointer to current write position */
+    decompSize     = 0;
+    /// start with the first block ///
+    while(decompSize < origSize)
+    {
+        uint32_t compressedBlockSize;  /// is saved in be format
+
+        if(origSize-decompSize < blockSize) /* the last block can be smaller than blockSize. */
+            blockSize = (uint32_t)(origSize-decompSize);
+        i32Buf = (uint32_t*)rpos;
+        compressedBlockSize =  be32toht(*i32Buf);  /// is saved in be format
+        rpos += 4;
+        if(compressedBlockSize == blockSize) /* there was no compression */
+        {
+            memcpy(roBuf, rpos, blockSize);
+        }
+        else /* do the decompression */
+        {
+#if LZ4_VERSION_NUMBER > 10300
+            int compressedBytes = LZ4_decompress_fast(rpos, roBuf, blockSize);
+#else
+            int compressedBytes = LZ4_uncompress(rpos, roBuf, blockSize);
+#endif
+            if(compressedBytes != compressedBlockSize)
+            {
+                printf("decompressed size not the same: %d, != %d\n", compressedBytes, compressedBlockSize);
+                return 0;
+            }
+        }
+
+        rpos += compressedBlockSize;   /* advance the read pointer to the next block */
+        roBuf += blockSize;            /* advance the write pointer */
+        decompSize += blockSize;
+    }
+    ret_value = (size_t)origSize;  // should always work, as orig_size cannot be > 2GB (sizeof(size_t) < 4GB)
+    return ret_value;
+}
+
+epicsShareFunc size_t compress_lz4hdf5(const char *inbuf, char *outbuf, size_t nbytes, size_t maxOutputSize, size_t blockSize)
+{
+    size_t nBlocks;
+    size_t outSize; /* size of the output buffer. Header size (12 bytes) is included */
+    size_t block;
+    uint64_t *i64Buf;
+    uint32_t *i32Buf;
+    size_t maxDestSize;
+    const char *rpos;      /* pointer to current read position */
+    char *roBuf;    /* pointer to current write position */
+
+    if (nbytes > INT32_MAX)
+    {
+        /* can only compress chunks up to 2GB */
+        return 0;
+    }
+
+    if (blockSize == 0) blockSize = DEFAULT_BLOCK_SIZE;
+    if (blockSize > nbytes)
+    {
+        blockSize = nbytes;
+    }
+    nBlocks = (nbytes-1)/blockSize +1;
+    maxDestSize = LZ4_compressBound((int)nbytes) + 4 + 8 + nBlocks*4;
+    if (maxDestSize > maxOutputSize) {
+        printf("maxOutputSize=%d too small, needs to be at least %d\n", (int)maxOutputSize, (int)maxDestSize);
+        return 0;
+    }
+
+    rpos  = inbuf;      /* pointer to current read position */
+    roBuf = outbuf;    /* pointer to current write position */
+    /* header */
+    i64Buf = (uint64_t *) (roBuf);
+    i64Buf[0] = htobe64t((uint64_t)nbytes); /* Store decompressed size in be format */
+    roBuf += 8;
+
+    i32Buf =  (uint32_t *) (roBuf);
+    i32Buf[0] = htobe32t((uint32_t)blockSize); /* Store the block size in be format */
+    roBuf += 4;
+
+    outSize = 12; /* size of the output buffer. Header size (12 bytes) is included */
+
+    for(block = 0; block < nBlocks; ++block)
+    {
+        uint32_t compBlockSize; /// reserve space for compBlockSize
+        size_t origWritten = block*blockSize;
+        if(nbytes - origWritten < blockSize) /* the last block may be < blockSize */
+            blockSize = nbytes - origWritten;
+
+#if LZ4_VERSION_NUMBER > 10300
+        compBlockSize = LZ4_compress_default(rpos, roBuf+4, (int)blockSize, (int)(maxDestSize-outSize)); /// reserve space for compBlockSize
+#else
+        compBlockSize = LZ4_compress(rpos, roBuf+4, blockSize); /// reserve space for compBlockSize
+#endif
+        if(!compBlockSize)
+            return 0;
+        if(compBlockSize >= blockSize) /* compression did not save any space, do a memcpy instead */
+        {
+            compBlockSize = (uint32_t)blockSize;
+            memcpy(roBuf+4, rpos, blockSize);
+        }
+
+        i32Buf =  (uint32_t *) (roBuf);
+        i32Buf[0] = htobe32t((uint32_t)compBlockSize);  /* write blocksize */
+        roBuf += 4;
+
+        rpos += blockSize;     	/* advance read pointer */
+        roBuf += compBlockSize;       /* advance write pointer */
+        outSize += compBlockSize + 4;
+    }
+
+    return outSize;
+
+}
+

supportApp/lz4hdf5Src/lz4hdf5.h

@@ -0,0 +1,17 @@
+#ifndef LZ4HDF5_H
+#define LZ4HDF5_H
+
+#include <shareLib.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif /* __cplusplus */
+
+epicsShareFunc size_t decompress_lz4hdf5(const char *inbuf, char *outbuf, size_t maxOutputSize, size_t *blockSize);
+epicsShareFunc size_t compress_lz4hdf5(const char *inbuf, char *outbuf, size_t nbytes, size_t maxOutputSize, size_t blockSize);
+
+#ifdef __cplusplus
+}
+#endif /* __cplusplus */
+
+#endif /* LZ4HDF5_H */