diff --git a/.gitignore b/.gitignore index 329a460..3b18a56 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,5 @@ O.*/ .project .settings .clangd +.iocsh_history .zed/ diff --git a/eigerApp/Makefile b/eigerApp/Makefile index 4262911..645d854 100644 --- a/eigerApp/Makefile +++ b/eigerApp/Makefile @@ -3,12 +3,10 @@ include $(TOP)/configure/CONFIG DIRS := $(DIRS) frozenSrc DIRS := $(DIRS) tinyCBORSrc -DIRS := $(DIRS) compressionSrc DIRS := $(DIRS) rfc3339Src DIRS := $(DIRS) src src_DEPEND_DIRS += frozenSrc src_DEPEND_DIRS += tinyCBORSrc -src_DEPEND_DIRS += compressionSrc src_DEPEND_DIRS += rfc3339Src DIRS := $(DIRS) $(filter-out $(DIRS), $(wildcard *db*)) DIRS := $(DIRS) $(filter-out $(DIRS), $(wildcard *Db*)) diff --git a/eigerApp/compressionSrc/LICENSE b/eigerApp/compressionSrc/LICENSE deleted file mode 100644 index 8ba71d2..0000000 --- a/eigerApp/compressionSrc/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2020 DECTRIS Ltd. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/eigerApp/compressionSrc/Makefile b/eigerApp/compressionSrc/Makefile deleted file mode 100644 index e513307..0000000 --- a/eigerApp/compressionSrc/Makefile +++ /dev/null @@ -1,17 +0,0 @@ -TOP=../.. -include $(TOP)/configure/CONFIG -#--------------------------------------- -# ADD MACRO DEFINITIONS AFTER THIS LINE -#======================================= - -LIBRARY_IOC_Linux = dectrisCompression -LIBRARY_IOC_Darwin = dectrisCompression - -INC += compression.h - -LIB_SRCS += compression.c - -include $(TOP)/configure/RULES -#--------------------------------------- -# ADD RULES AFTER THIS LINE -#======================================= diff --git a/eigerApp/compressionSrc/compression.c b/eigerApp/compressionSrc/compression.c deleted file mode 100644 index f8300f7..0000000 --- a/eigerApp/compressionSrc/compression.c +++ /dev/null @@ -1,226 +0,0 @@ -#if defined(__APPLE__) -#include -#define be32toh(x) OSSwapBigToHostInt32(x) -#define be64toh(x) OSSwapBigToHostInt64(x) -#elif defined(_WIN32) -#define INCL_EXTRA_HTON_FUNCTIONS 1 -#include -#define be32toh(x) ntohl(x) -#define be64toh(x) ntohll(x) -#pragma comment(lib, "ws2_32.lib") -#else -#define _BSD_SOURCE -/* #define _DEFAULT_SOURCE */ -#include -#endif -#include -#include -#include - -#include "bitshuffle.h" -#include "lz4.h" -#include "compression.h" - -#if defined(__clang__) -#pragma GCC diagnostic ignored "-Wtautological-constant-out-of-range-compare" -#elif defined(__GNUC__) -#pragma GCC diagnostic ignored "-Wtype-limits" -#endif - -#ifndef BSHUF_BLOCKED_MULT -#define BSHUF_BLOCKED_MULT 8 -#endif - -static uint32_t read_u32_be(const char* buf) { - uint32_t value; - memcpy(&value, buf, sizeof(value)); - return be32toh(value); -} - -static uint64_t read_u64_be(const char* buf) { - uint64_t value; - memcpy(&value, buf, sizeof(value)); - return be64toh(value); -} - -static _Bool parse_header(const char** const src, - const char* src_end, - uint64_t* const orig_size, - uint32_t* const block_size) { - if (src_end - *src < 12) - return 0; - *orig_size = read_u64_be(*src); - *block_size = read_u32_be(*src + 8); - if (*orig_size >= SIZE_MAX || *block_size >= SIZE_MAX) - return 0; - *src += 12; - return 1; -} - -static _Bool decompress_bslz4_block(char** const dst, - const char** const src, - const char* const src_end, - char* const tmp_buf, - const uint32_t block_size, - const size_t elem_size) { - if (src_end - *src < 4) - return 0; - const uint32_t compressed_size = read_u32_be(*src); - *src += 4; - if (compressed_size > INT_MAX || (int)compressed_size > src_end - *src) - return 0; - if (LZ4_decompress_safe(*src, &tmp_buf[0], compressed_size, block_size) != - (int)block_size) - { - return 0; - } -// if (bitshuf_decode_block(*dst, &tmp_buf[0], &tmp_buf[block_size], -// block_size / elem_size, elem_size) != 0) - { - return 0; - } - *dst += block_size; - *src += compressed_size; - return 1; -} - -static size_t decompress_buffer_bslz4_hdf5(char* dst, - size_t dst_size, - const char* src, - size_t src_size, - size_t elem_size) { - if (elem_size == 0 || elem_size > UINT32_MAX / BSHUF_BLOCKED_MULT) - return COMPRESSION_ERROR; - - const char* const src_end = src + src_size; - uint64_t orig_size; - uint32_t block_size; - if (!parse_header(&src, src_end, &orig_size, &block_size)) - return COMPRESSION_ERROR; - - if (dst_size == 0) - return (size_t)orig_size; - - if (orig_size > dst_size || (orig_size != 0 && block_size == 0) || - block_size % (BSHUF_BLOCKED_MULT * elem_size) != 0 || - block_size > INT_MAX) - { - return COMPRESSION_ERROR; - } - - if (orig_size == 0) - return 0; - - char* tmp_buf = malloc(block_size * 2); - if (!tmp_buf) - return COMPRESSION_ERROR; - - const char* const dst_end = dst + dst_size; - const int leftover_bytes = - orig_size % (int)(BSHUF_BLOCKED_MULT * elem_size); - const uint64_t full_block_count = orig_size / block_size; - const uint32_t last_block_size = (orig_size % block_size) - leftover_bytes; - - for (uint64_t block = 0; block < full_block_count; ++block) { - if (!decompress_bslz4_block(&dst, &src, src_end, tmp_buf, block_size, - elem_size)) - { - free(tmp_buf); - return COMPRESSION_ERROR; - } - } - if (last_block_size > 0) { - if (!decompress_bslz4_block(&dst, &src, src_end, tmp_buf, - last_block_size, elem_size)) - { - free(tmp_buf); - return COMPRESSION_ERROR; - } - } - free(tmp_buf); - - if (leftover_bytes > 0) { - if (leftover_bytes > dst_end - dst || leftover_bytes != src_end - src) - return COMPRESSION_ERROR; - memcpy(dst, src, leftover_bytes); - } - - return (size_t)orig_size; -} - -static _Bool decompress_lz4_block(char** const dst, - const char** const src, - const char* const src_end, - const uint32_t block_size) { - if (src_end - *src < 4) - return 0; - const uint32_t compressed_size = read_u32_be(*src); - *src += 4; - if (compressed_size > INT_MAX || (int)compressed_size > src_end - *src) - return 0; - if (compressed_size == block_size) { - memcpy(*dst, *src, block_size); - } else { - if (LZ4_decompress_safe(*src, *dst, compressed_size, block_size) != - (int)block_size) - { - return 0; - } - } - *dst += block_size; - *src += compressed_size; - return 1; -} - -static size_t decompress_buffer_lz4_hdf5(char* dst, - size_t dst_size, - const char* src, - size_t src_size) { - const char* const src_end = src + src_size; - uint64_t orig_size; - uint32_t block_size; - if (!parse_header(&src, src_end, &orig_size, &block_size)) - return COMPRESSION_ERROR; - - if (dst_size == 0) - return (size_t)orig_size; - - if (orig_size > dst_size || (orig_size != 0 && block_size == 0) || - block_size > INT_MAX) - { - return COMPRESSION_ERROR; - } - - if (orig_size == 0) - return 0; - - const uint64_t full_block_count = orig_size / block_size; - const uint32_t last_block_size = orig_size % block_size; - - for (uint64_t block = 0; block < full_block_count; ++block) { - if (!decompress_lz4_block(&dst, &src, src_end, block_size)) - return COMPRESSION_ERROR; - } - if (last_block_size > 0) { - if (!decompress_lz4_block(&dst, &src, src_end, last_block_size)) - return COMPRESSION_ERROR; - } - - return (size_t)orig_size; -} - -size_t compression_decompress_buffer(CompressionAlgorithm algorithm, - char* dst, - size_t dst_size, - const char* src, - size_t src_size, - size_t elem_size) { - switch (algorithm) { - case COMPRESSION_BSLZ4: - return decompress_buffer_bslz4_hdf5(dst, dst_size, src, src_size, - elem_size); - case COMPRESSION_LZ4: - return decompress_buffer_lz4_hdf5(dst, dst_size, src, src_size); - } - return COMPRESSION_ERROR; -} diff --git a/eigerApp/compressionSrc/compression.h b/eigerApp/compressionSrc/compression.h deleted file mode 100644 index c28952e..0000000 --- a/eigerApp/compressionSrc/compression.h +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef DECTRIS_COMPRESSION_H_ -#define DECTRIS_COMPRESSION_H_ - -#include -#include - -#if defined(__cplusplus) -extern "C" { -#endif - -#define COMPRESSION_ERROR SIZE_MAX - -/* DECTRIS compression algorithms */ -typedef enum { - /* Bitshuffle with LZ4 compression (HDF5 framing) - * - * Data is stored as a series of bitshuffle transposed blocks compressed - * with LZ4. The format is the same as the bitshuffle HDF5 filter. - * - * # See also - * - * https://github.com/kiyo-masui/bitshuffle - * https://github.com/kiyo-masui/bitshuffle/blob/f57e2e50cc6855d5cf7689b9bc688b3ef1dffe02/src/bshuf_h5filter.c - * https://lz4.github.io/lz4/ - * https://github.com/lz4/lz4/blob/master/doc/lz4_Block_format.md - */ - COMPRESSION_BSLZ4, - - /* LZ4 compression (HDF5 framing) - * - * Data is stored as a series of LZ4 compressed blocks. The LZ4 filter - * format for HDF5 is used for framing. - * - * # See also - * - * https://lz4.github.io/lz4/ - * https://github.com/lz4/lz4/blob/master/doc/lz4_Block_format.md - * https://support.hdfgroup.org/services/filters/HDF5_LZ4.pdf - */ - COMPRESSION_LZ4, - -} CompressionAlgorithm; - -/* Decompresses the contents of a source buffer into a destination buffer. */ -size_t compression_decompress_buffer(CompressionAlgorithm algorithm, - char* dst, - size_t dst_size, - const char* src, - size_t src_size, - size_t elem_size); - -#if defined(__cplusplus) -} /* extern "C" */ -#endif - -#endif /* DECTRIS_COMPRESSION_H_ */ diff --git a/eigerApp/src/Makefile b/eigerApp/src/Makefile index b6cbe2f..db9de56 100755 --- a/eigerApp/src/Makefile +++ b/eigerApp/src/Makefile @@ -31,11 +31,6 @@ else LIB_SYS_LIBS += hdf5_hl endif -PROD += test_lz4 -test_lz4_SRCS += test_lz4.cpp -test_lz4_LIBS += dectrisCompression -test_lz4_SYS_LIBS += lz4 - include $(ADCORE)/ADApp/commonLibraryMakefile #============================= diff --git a/eigerApp/src/Stream2_lz4_data b/eigerApp/src/Stream2_lz4_data deleted file mode 100644 index 3c0e85c..0000000 Binary files a/eigerApp/src/Stream2_lz4_data and /dev/null differ diff --git a/eigerApp/src/Stream_lz4_data b/eigerApp/src/Stream_lz4_data deleted file mode 100644 index fa6f62d..0000000 Binary files a/eigerApp/src/Stream_lz4_data and /dev/null differ diff --git a/eigerApp/src/stream2Api.cpp b/eigerApp/src/stream2Api.cpp index be10863..7f787b1 100644 --- a/eigerApp/src/stream2Api.cpp +++ b/eigerApp/src/stream2Api.cpp @@ -13,9 +13,8 @@ #include #include #include -#include +#include #include -#include #define ZMQ_PORT 31001 @@ -36,6 +35,7 @@ static int uncompress (const unsigned char *pInput, char *dest, char *encoding, orig_size = be64toh(orig_size); uint32_t block_size = *(uint32_t *)(pInput + 8); block_size = be32toh(block_size); + int result; switch (dataType) { case NDUInt32: elemSize=4; break; @@ -46,17 +46,11 @@ static int uncompress (const unsigned char *pInput, char *dest, char *encoding, return STREAM_ERROR; } if (strcmp(encoding, "lz4") == 0) { -// ERR("calling LZ4_decompress_fast"); -// size_t result = LZ4_decompress_fast((const char *)pInput, dest, (int)uncompressedSize); - size_t result = compression_decompress_buffer(COMPRESSION_LZ4, - dest, - uncompressedSize, - (const char *)pInput, - compressedSize, - elemSize); - if (result < 0) + result = decompress_lz4hdf5((const char *)pInput, dest, uncompressedSize); + + if (result <= 0) { - ERR_ARGS("LZ4_decompress failed, result=%lu\n", result); + ERR_ARGS("decompress_lz4hdf5 failed, result=%d\n", result); return STREAM_ERROR; } } @@ -289,7 +283,7 @@ int Stream2API::getFrame (NDArray **pArrayOut, NDArrayPool *pNDArrayPool, int th const unsigned char *pInput = pSB->ptr; if (strcmp(encoding, "lz4") == 0) { - pArray->codec.name = "lz4"; + pArray->codec.name = "lz4hdf5"; } else if (strcmp(encoding, "bslz4") == 0) { diff --git a/eigerApp/src/test_lz4.cpp b/eigerApp/src/test_lz4.cpp deleted file mode 100644 index 5bf1d92..0000000 --- a/eigerApp/src/test_lz4.cpp +++ /dev/null @@ -1,39 +0,0 @@ -// This program reads lz4 compressed buffers read from the stream and stream2 interfaces. -// It attempts to decompress them using the standard lz4 library. -// It also decompresses the stream2 buffer using the decompression functions from compress. -// If decompression is successful it writes the decompressed data as a TIFF file so it can be visualized. - -#include -#include -#include -#include - -int main(int argc, char *argv[]) -{ - #define MAX_COMPRESSED_SIZE 100000 - #define MAX_DECOMPRESSED_SIZE 10000000 - char *compressedBuff = (char *)calloc(MAX_COMPRESSED_SIZE, 1); - char *decompressedBuff = (char *)calloc(MAX_DECOMPRESSED_SIZE, 1); - FILE *fp; - size_t compressedSize, decompressedSize; - - // Read the buffer read from the Stream interface with lz4 compression - fp = fopen("Stream_lz4_data", "rb"); - compressedSize = fread(compressedBuff, 1, MAX_COMPRESSED_SIZE, fp); - fclose(fp); - decompressedSize = LZ4_decompress_safe(compressedBuff, (char *)decompressedBuff, compressedSize, MAX_DECOMPRESSED_SIZE); - printf("Stream_lz4_data decompressed with lz4_decompress_safe, compressedSize=%d, decompressedSize=%d\n", (int)compressedSize, (int)decompressedSize); - - // Read the buffer read from the Stream2 interface with lz4 compression - fp = fopen("Stream2_lz4_data", "rb"); - - compressedSize = fread(compressedBuff, 1, MAX_COMPRESSED_SIZE, fp); - fclose(fp); - decompressedSize = LZ4_decompress_safe(compressedBuff, (char *)decompressedBuff, compressedSize, MAX_DECOMPRESSED_SIZE); - printf("Stream2_lz4_data decompressed with lz4_decompress_safe, compressedSize=%d, decompressedSize=%d\n", (int)compressedSize, (int)decompressedSize); - - // Decompress with the Dectris routine - decompressedSize = compression_decompress_buffer(COMPRESSION_LZ4, decompressedBuff, MAX_DECOMPRESSED_SIZE, compressedBuff, compressedSize, 4); - printf("Stream2_lz4_data decompressed with Dectris compression_decompress_buffer, compressedSize=%d, decompressedSize=%d\n", - (int)compressedSize, (int)decompressedSize); -} diff --git a/iocs/eigerIOC/eigerApp/src/Makefile b/iocs/eigerIOC/eigerApp/src/Makefile index 5e0641f..b01b4e4 100755 --- a/iocs/eigerIOC/eigerApp/src/Makefile +++ b/iocs/eigerIOC/eigerApp/src/Makefile @@ -16,7 +16,7 @@ PROD_SRCS += $(PROD_NAME)_registerRecordDeviceDriver.cpp $(PROD_NAME)Main.cpp # Note that the lz4 library will be found in ADSupport via the blosc package # So this application must be built with WITH_BLOSC=YES and BLOSC_EXTERNAL=NO. -PROD_LIBS += eigerDetector frozen hdf5_hl tinyCBOR dectrisCompression +PROD_LIBS += eigerDetector frozen hdf5_hl tinyCBOR ifdef ZMQ_LIB zmq_DIR += $(ZMQ_LIB)