split up into library and windows / unix test tools

- pre compiled windows binaries for testing will be uploaded - no real code change, just movements of files gelöscht: Makefile geändert: README.md neue Datei: lib/README.md umbenannt: list.h -> lib/list.h umbenannt: lz4mt.h -> lib/lz4mt.h umbenannt: lz4mt_common.c -> lib/lz4mt_common.c umbenannt: lz4mt_compress.c -> lib/lz4mt_compress.c umbenannt: lz4mt_decompress.c -> lib/lz4mt_decompress.c umbenannt: lz5mt.h -> lib/lz5mt.h umbenannt: lz5mt_common.c -> lib/lz5mt_common.c umbenannt: lz5mt_compress.c -> lib/lz5mt_compress.c umbenannt: lz5mt_decompress.c -> lib/lz5mt_decompress.c umbenannt: mem.h -> lib/mem.h umbenannt: threading.c -> lib/threading.c umbenannt: threading.h -> lib/threading.h umbenannt: zstdmt.h -> lib/zstdmt.h umbenannt: zstdmt_common.c -> lib/zstdmt_common.c umbenannt: zstdmt_compress.c -> lib/zstdmt_compress.c umbenannt: zstdmt_decompress.c -> lib/zstdmt_decompress.c geändert: multistream/README.md neue Datei: unix/Makefile neue Datei: unix/README.md umbenannt: lz4mt.c -> unix/lz4mt.c umbenannt: lz5mt.c -> unix/lz5mt.c umbenannt: util.c -> unix/util.c umbenannt: util.h -> unix/util.h umbenannt: zstdmt.c -> unix/zstdmt.c neue Datei: windows/Makefile neue Datei: windows/README.md neue Datei: windows/README.mdc neue Datei: windows/lz4mt.c neue Datei: windows/lz5mt.c neue Datei: windows/zstdmt.c
mcmilk · Sep 28, 2016 · 74afa68 · 74afa68
1 parent 8a665d5
commit 74afa68
Show file tree

Hide file tree

Showing 33 changed files with 1,387 additions and 177 deletions.
diff --git a/Makefile b/Makefile
diff --git a/README.md b/README.md
@@ -9,114 +9,12 @@
 
 ## Usage of the Testutils
 
-### [LZ4]
-```
-Usage: lz4mt [options] infile outfile
-
-Otions:
- -l N    set level (1..16) of compression (default: 1)
- -t N    set number of (de)compression threads (default: 2)
- -i N    set number of iterations for testing (default: 1)
- -b N    set input chunksize to N MiB (default: auto)
- -c      compress (default mode)
- -d      use decompress mode
- -H      print headline for the testing values
- -h      show usage
- -v      show version
-```
-
-### [LZ5]
-```
-Usage: lz5mt [options] infile outfile
-
-Otions:
- -l N    set level (1..16) of compression (default: 1)
- -t N    set number of (de)compression threads (default: 2)
- -i N    set number of iterations for testing (default: 1)
- -b N    set input chunksize to N MiB (default: auto)
- -c      compress (default mode)
- -d      use decompress mode
- -H      print headline for the testing values
- -h      show usage
- -v      show version
-```
-
-### [ZStandard]
-```
-Usage: zstdmt [options] infile outfile
-
-Otions:
- -l N    set level of compression (default: 3)
- -t N    set number of (de)compression threads (default: 2)
- -i N    set number of iterations for testing (default: 1)
- -b N    set input chunksize to N MiB (default: auto)
- -c      compress (default mode)
- -d      use decompress mode (XXX, not done)
- -H      print headline for the testing values
- -h      show usage
- -v      show version
-```
+- see unix
+- see windows
 
 ## Usage of the Library
 
-### Compression
-```
-typedef struct {
-	void *buf;		/* ptr to data */
-	size_t size;		/* current filled in buf */
-	size_t allocated;	/* length of buf */
-} LZ4MT_Buffer;
-
-/**
- * reading and writing functions
- * - you can use stdio functions or plain read/write ...
- * - a sample is given in 7-Zip ZS or lz4mt.c
- */
-typedef int (fn_read) (void *args, LZ4MT_Buffer * in);
-typedef int (fn_write) (void *args, LZ4MT_Buffer * out);
-
-typedef struct {
-	fn_read *fn_read;
-	void *arg_read;
-	fn_write *fn_write;
-	void *arg_write;
-} LZ4MT_RdWr_t;
-
-typedef struct LZ4MT_CCtx_s LZ4MT_CCtx;
-
-/* 1) allocate new cctx */
-LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize);
-
-/* 2) threaded compression */
-size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr);
-
-/* 3) get some statistic */
-size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx);
-size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx);
-size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx);
-
-/* 4) free cctx */
-void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx);
-```
-
-### Decompression
-```
-typedef struct LZ4MT_DCtx_s LZ4MT_DCtx;
-
-/* 1) allocate new cctx */
-LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize);
-
-/* 2) threaded compression */
-size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr);
-
-/* 3) get some statistic */
-size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx);
-size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx);
-size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx);
-
-/* 4) free cctx */
-void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx);
-```
+- see lib
 
 ## Benchmarks
 
@@ -283,9 +181,5 @@ Level|Threads|InSize|OutSize|Frames|Real|User|Sys|MaxMem
 [LZ5]:https://github.com/inikep/lz5
 [ZStandard]:http://facebook.github.io/zstd/
 
-# TODO
-
-- threaded decompression for zstd not ready
-- document the header files correctly ... just some first start currently
 
-/TR 2016-09-25
+/TR 2016-09-28
diff --git a/lib/README.md b/lib/README.md
@@ -0,0 +1,66 @@
+
+# Multithreading Library for [LZ4], [LZ5] and [ZStandard]
+
+
+### Compression
+```
+typedef struct {
+	void *buf;		/* ptr to data */
+	size_t size;		/* current filled in buf */
+	size_t allocated;	/* length of buf */
+} LZ4MT_Buffer;
+
+/**
+ * reading and writing functions
+ * - you can use stdio functions or plain read/write ...
+ * - a sample is given in 7-Zip ZS or lz4mt.c
+ */
+typedef int (fn_read) (void *args, LZ4MT_Buffer * in);
+typedef int (fn_write) (void *args, LZ4MT_Buffer * out);
+
+typedef struct {
+	fn_read *fn_read;
+	void *arg_read;
+	fn_write *fn_write;
+	void *arg_write;
+} LZ4MT_RdWr_t;
+
+typedef struct LZ4MT_CCtx_s LZ4MT_CCtx;
+
+/* 1) allocate new cctx */
+LZ4MT_CCtx *LZ4MT_createCCtx(int threads, int level, int inputsize);
+
+/* 2) threaded compression */
+size_t LZ4MT_CompressCCtx(LZ4MT_CCtx * ctx, LZ4MT_RdWr_t * rdwr);
+
+/* 3) get some statistic */
+size_t LZ4MT_GetFramesCCtx(LZ4MT_CCtx * ctx);
+size_t LZ4MT_GetInsizeCCtx(LZ4MT_CCtx * ctx);
+size_t LZ4MT_GetOutsizeCCtx(LZ4MT_CCtx * ctx);
+
+/* 4) free cctx */
+void LZ4MT_freeCCtx(LZ4MT_CCtx * ctx);
+```
+
+### Decompression
+```
+typedef struct LZ4MT_DCtx_s LZ4MT_DCtx;
+
+/* 1) allocate new cctx */
+LZ4MT_DCtx *LZ4MT_createDCtx(int threads, int inputsize);
+
+/* 2) threaded compression */
+size_t LZ4MT_DecompressDCtx(LZ4MT_DCtx * ctx, LZ4MT_RdWr_t * rdwr);
+
+/* 3) get some statistic */
+size_t LZ4MT_GetFramesDCtx(LZ4MT_DCtx * ctx);
+size_t LZ4MT_GetInsizeDCtx(LZ4MT_DCtx * ctx);
+size_t LZ4MT_GetOutsizeDCtx(LZ4MT_DCtx * ctx);
+
+/* 4) free cctx */
+void LZ4MT_freeDCtx(LZ4MT_DCtx * ctx);
+```
+
+## Todo
+
+- add Makefile
diff --git a/list.h → lib/list.h b/list.h → lib/list.h
diff --git a/lz4mt.h → lib/lz4mt.h b/lz4mt.h → lib/lz4mt.h
diff --git a/lz4mt_common.c → lib/lz4mt_common.c b/lz4mt_common.c → lib/lz4mt_common.c
@@ -24,14 +24,6 @@ size_t lz4mt_errcode;
  */
 unsigned LZ4MT_isError(size_t code)
 {
-#if 0
-	printf("no_error            = %zu\n", ERROR(no_error));
-	printf("memory_allocation   = %zu\n", ERROR(memory_allocation));
-	printf("compression_library = %zu\n", ERROR(compression_library));
-	printf("maxCode             = %zu\n", ERROR(maxCode));
-	printf("result              = %zu\n", code);
-	printf("ok ZSTD_decompressStream() result=%zu >=%d max=%zu\n", code, code > ERROR(maxCode), ERROR(maxCode));
-#endif
 	return (code > ERROR(maxCode));
 }
 

diff --git a/lz4mt_compress.c → lib/lz4mt_compress.c b/lz4mt_compress.c → lib/lz4mt_compress.c
diff --git a/lz4mt_decompress.c → lib/lz4mt_decompress.c b/lz4mt_decompress.c → lib/lz4mt_decompress.c
diff --git a/lz5mt.h → lib/lz5mt.h b/lz5mt.h → lib/lz5mt.h
diff --git a/lz5mt_common.c → lib/lz5mt_common.c b/lz5mt_common.c → lib/lz5mt_common.c
diff --git a/lz5mt_compress.c → lib/lz5mt_compress.c b/lz5mt_compress.c → lib/lz5mt_compress.c
diff --git a/lz5mt_decompress.c → lib/lz5mt_decompress.c b/lz5mt_decompress.c → lib/lz5mt_decompress.c
diff --git a/mem.h → lib/mem.h b/mem.h → lib/mem.h
diff --git a/threading.c → lib/threading.c b/threading.c → lib/threading.c
diff --git a/threading.h → lib/threading.h b/threading.h → lib/threading.h
diff --git a/zstdmt.h → lib/zstdmt.h b/zstdmt.h → lib/zstdmt.h
diff --git a/zstdmt_common.c → lib/zstdmt_common.c b/zstdmt_common.c → lib/zstdmt_common.c
diff --git a/zstdmt_compress.c → lib/zstdmt_compress.c b/zstdmt_compress.c → lib/zstdmt_compress.c
diff --git a/zstdmt_decompress.c → lib/zstdmt_decompress.c b/zstdmt_decompress.c → lib/zstdmt_decompress.c
diff --git a/multistream/README.md b/multistream/README.md
@@ -1,20 +1,6 @@
 
 # zstdmt multistream testing, obsolete
 
-zstdmt depends currently on posix threads for multi-threading and is
-only tested on linux
-
-I will add support for windows and CreateThread() together with
-WaitForMultipleObjects() / WaitForSingleObject() later...
-
-## Building for Linux with gcc
-
- - Run `make`.
-
-## Building for Linux with Clang
-
- - Run `make CC=clang`
-
 ## Overview
 
  - `./zstd-zstd` - single threaded single stream mode (normal zstd stream)
@@ -24,8 +10,7 @@ WaitForMultipleObjects() / WaitForSingleObject() later...
 
 ## Todo
 
- - add support for windows
- - fix and document return codes in own mini libraries
+ - nothing, this stuff is obsolte, just kept for history
 
 ## See also
 

diff --git a/unix/Makefile b/unix/Makefile
@@ -0,0 +1,46 @@
+
+CC	= gcc
+STRIP	= strip
+#CC	= clang
+
+CFLAGS	= -W -pthread -Wall -pipe
+CFLAGS	+= -fomit-frame-pointer
+CFLAGS	+= -I ../lib
+LDFLAGS	= -lpthread
+# -static
+
+#CFLAGS += -DDEBUGME
+#CFLAGS += -g
+CFLAGS += -O3
+#CFLAGS += -march=native
+#CFLAGS += -Wno-unused-but-set-variable
+#CFLAGS += -Wno-unused-variable
+#CFLAGS += -Wno-unused-function
+
+PRGS	= zstdmt lz4mt lz5mt
+
+COMMON	= util.c
+LIBLZ4	= ../lib/threading.c ../lib/lz4mt_common.c ../lib/lz4mt_compress.c ../lib/lz4mt_decompress.c
+LIBLZ5	= ../lib/threading.c ../lib/lz5mt_common.c ../lib/lz5mt_compress.c ../lib/lz5mt_decompress.c
+LIBZSTD	= ../lib/threading.c ../lib/zstdmt_common.c ../lib/zstdmt_compress.c ../lib/zstdmt_decompress.c
+
+all:	$(PRGS)
+again:	clean $(PRGS)
+
+lz4mt:
+	$(CC) $(CFLAGS) $(LDFLAGS) -llz4 -o $@ lz4mt.c $(LIBLZ4) $(COMMON)
+	$(STRIP) $@
+
+lz5mt:
+	$(CC) $(CFLAGS) $(LDFLAGS) -llz5 -o $@ lz5mt.c $(LIBLZ5) $(COMMON)
+	$(STRIP) $@
+
+zstdmt:
+	$(CC) $(CFLAGS) $(LDFLAGS) -lzstd -o $@ zstdmt.c $(LIBZSTD) $(COMMON)
+	$(STRIP) $@
+
+clean:
+	rm -rf $(PRGS)
+
+.c.c:
+	$(CC) $(CFLAGS) -c $< -o $@