Skip to content

Commit

Permalink
Merge pull request #182 from sPHENIX-Collaboration/bz3compression
Browse files Browse the repository at this point in the history
Bz2compression
  • Loading branch information
mpurschke authored Aug 28, 2024
2 parents c6202c3 + dedba9a commit bbddd86
Show file tree
Hide file tree
Showing 10 changed files with 223 additions and 31 deletions.
3 changes: 2 additions & 1 deletion newbasic/BufferConstants.h
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
#define GZBUFFERMARKER 0xfffffafeU
#define LZO1XBUFFERMARKER 0xffffbbfeU
#define LZO1CBUFFERMARKER 0xffffbcfeU
#define ONCSLZO1XBUFFERMARKER 0xffffbbc0U
#define LZO2ABUFFERMARKER 0xffffbdfdU
#define BZ2BUFFERMARKER 0xffffbefaU

#define BUFFERBLOCKSIZE 8192U

Expand Down
28 changes: 15 additions & 13 deletions newbasic/Makefile.am
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ installedheaders = \
eventReceiverClient.h \
gzbuffer.h \
lzobuffer.h \
bz2buffer.h \
prdfBuffer.h \
oBuffer.h \
ophBuffer.h \
Expand Down Expand Up @@ -237,6 +238,7 @@ allsources = \
PHmd5Value.cc \
gzbuffer.cc \
lzobuffer.cc \
bz2buffer.cc \
oEvent.cc \
ospEvent.cc \
oBuffer.cc \
Expand Down Expand Up @@ -402,22 +404,22 @@ eventClient_SOURCES = eventClient.cc
lastEvent_SOURCES = lastEvent.cc


dpipe_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@
dlist_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@
ddump_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@
eventcombiner_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@
dpipe_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@ -lbz2
dlist_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@ -lbz2
ddump_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@ -lbz2
eventcombiner_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@ -lbz2


changeid_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@
changehitformat_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@
prdf2prdf_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@
prdfcheck_LDADD = libNoRootEvent.la -ldl @LZOLIB@
prdfsplit_LDADD = libNoRootEvent.la -ldl @LZOLIB@
changeid_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@ -lbz2
changehitformat_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@ -lbz2
prdf2prdf_LDADD = libNoRootEvent.la libmessage.la -ldl @LZOLIB@ -lbz2
prdfcheck_LDADD = libNoRootEvent.la -ldl @LZOLIB@ -lbz2
prdfsplit_LDADD = libNoRootEvent.la -ldl @LZOLIB@ -lbz2

eventServer_LDADD = libNoRootEvent.la -ldl -lpthread @LZOLIB@
eventServer_classic_LDADD = libNoRootEvent.la -ldl -lpthread @LZOLIB@
eventClient_LDADD = libNoRootEvent.la -ldl @LZOLIB@
lastEvent_LDADD = libNoRootEvent.la -ldl @LZOLIB@
eventServer_LDADD = libNoRootEvent.la -ldl -lpthread @LZOLIB@ -lbz2
eventServer_classic_LDADD = libNoRootEvent.la -ldl -lpthread @LZOLIB@ -lbz2
eventClient_LDADD = libNoRootEvent.la -ldl @LZOLIB@ -lbz2
lastEvent_LDADD = libNoRootEvent.la -ldl @LZOLIB@ -lbz2


libmessage_la_SOURCES = \
Expand Down
17 changes: 15 additions & 2 deletions newbasic/buffer.cc
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
#include "buffer.h"
#include "gzbuffer.h"
#include "lzobuffer.h"
#include "bz2buffer.h"
#include "prdfBuffer.h"
#include "oncsBuffer.h"

Expand All @@ -23,13 +24,25 @@ int buffer::makeBuffer( PHDWORD *bp, const int allocatedsize, buffer **bptr)
return 0;
}

else if ( bp[1]== LZO1XBUFFERMARKER || buffer::u4swap(bp[1])== LZO1XBUFFERMARKER
|| bp[1]== LZO1CBUFFERMARKER || buffer::u4swap(bp[1])== LZO1CBUFFERMARKER )
else if ( bp[1] == BUFFERMARKER
|| bp[1]== LZO1XBUFFERMARKER
|| bp[1]== LZO1CBUFFERMARKER
|| bp[1]== LZO2ABUFFERMARKER
|| buffer::u4swap(bp[1]) == BUFFERMARKER
|| buffer::u4swap(bp[1])== LZO1XBUFFERMARKER
|| buffer::u4swap(bp[1])== LZO1CBUFFERMARKER
|| buffer::u4swap(bp[1])== LZO2ABUFFERMARKER )
{
*bptr = new lzobuffer ( bp, allocatedsize );
return 0;
}

else if ( bp[1]== BZ2BUFFERMARKER || buffer::u4swap(bp[1])== BZ2BUFFERMARKER )
{
*bptr = new bz2buffer ( bp, allocatedsize );
return 0;
}

else if ( bp[1]== ONCSBUFFERMARKER || buffer::u4swap(bp[1])== ONCSBUFFERMARKER )
{
*bptr = new oncsBuffer ( bp, allocatedsize );
Expand Down
88 changes: 88 additions & 0 deletions newbasic/bz2buffer.cc
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
#include "bz2buffer.h"
#include "oncsBuffer.h"
#include <bzlib.h>


// the constructor first ----------------
bz2buffer::bz2buffer (PHDWORD *array , const int length )

{

is_good =1;
bufferarray=0;
theBuffer=0;

unsigned int bytes;
unsigned int outputlength_in_bytes;
if (array[1] == BZ2BUFFERMARKER)
{
bytes = array[0]-4*BUFFERHEADERLENGTH;
outputlength_in_bytes = array[3];
}
else if ( u4swap(array[1]) == BZ2BUFFERMARKER )
{
bytes = i4swap(array[0])-16;
outputlength_in_bytes = i4swap(array[3]);
}

else
{
COUT << __FILE__ << " " << __LINE__ << " wrong buffer marker = " << std::hex << array[1] << std::dec << std::endl;
is_good = 0;
return;
}


unsigned int outputlength = (outputlength_in_bytes+3)/4;
bufferarray = new PHDWORD[outputlength];

unsigned int olen = 4*outputlength;

int status = BZ2_bzBuffToBuffDecompress ( (char *) bufferarray, &olen,
( char *) &array[4], bytes, 0, 0 );

if ( status || olen != outputlength_in_bytes)
{
COUT << __FILE__ << " " << __LINE__ << " wrong-sized buffer: " << olen << " should be " << outputlength_in_bytes << std::endl;
is_good = 0;
// delete [] bufferarray;
// bufferarray = 0;
// return;
}
if ( bufferarray[1]== BUFFERMARKER || buffer::u4swap(bufferarray[1])== BUFFERMARKER )
{
theBuffer = new prdfBuffer(bufferarray, outputlength);
}
else if ( bufferarray[1]== ONCSBUFFERMARKER || buffer::u4swap(bufferarray[1])== ONCSBUFFERMARKER )
{
theBuffer = new oncsBuffer(bufferarray, outputlength);
}
else
{
theBuffer = 0;
}
}

// ---------------------------------------------------------
Event * bz2buffer::getEvent()
{
if ( theBuffer) return theBuffer->getEvent();
return 0;
}

// ---------------------------------------------------------
int bz2buffer::getBufferSequence() const
{
if ( !theBuffer) return 0;
return theBuffer->getBufferSequence();
}

// ---------------------------------------------------------

bz2buffer::~bz2buffer()
{
if ( theBuffer) delete theBuffer;
if ( bufferarray) delete [] bufferarray;
}


32 changes: 32 additions & 0 deletions newbasic/bz2buffer.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#ifndef __BZ2BUFFER_H
#define __BZ2BUFFER_H

#include "prdfBuffer.h"

#ifndef __CINT__
class WINDOWSEXPORT bz2buffer : public prdfBuffer{
#else
class bz2buffer : public prdfBuffer{
#endif

public:

//** Constructors

bz2buffer( PHDWORD *array, const int length);
~bz2buffer();

Event * getEvent();

int getBufferSequence() const;

protected:

PHDWORD *bufferarray;
buffer *theBuffer;

int _broken;

};

#endif
28 changes: 20 additions & 8 deletions newbasic/fileEventiterator.cc
Original file line number Diff line number Diff line change
Expand Up @@ -192,14 +192,24 @@ int fileEventiterator::read_next_buffer()

// get the buffer length into a dedicated variable
if (initialbuffer[1] == BUFFERMARKER || initialbuffer[1]== GZBUFFERMARKER
|| initialbuffer[1]== LZO1XBUFFERMARKER || initialbuffer[1]== LZO1CBUFFERMARKER || initialbuffer[1]== ONCSBUFFERMARKER)
|| initialbuffer[1]== LZO1XBUFFERMARKER
|| initialbuffer[1]== LZO1CBUFFERMARKER
|| initialbuffer[1]== LZO2ABUFFERMARKER
|| initialbuffer[1]== BZ2BUFFERMARKER
|| initialbuffer[1]== ONCSBUFFERMARKER)
{
buffer_size = initialbuffer[0];
}
else
{
unsigned int marker = buffer::u4swap(initialbuffer[1]);
if (marker == BUFFERMARKER || marker == GZBUFFERMARKER || marker == LZO1XBUFFERMARKER || marker == LZO1CBUFFERMARKER || marker == ONCSBUFFERMARKER)
if (marker == BUFFERMARKER
|| marker == GZBUFFERMARKER
|| marker == LZO1XBUFFERMARKER
|| marker == LZO1CBUFFERMARKER
|| marker == LZO2ABUFFERMARKER
|| marker == BZ2BUFFERMARKER
|| marker == ONCSBUFFERMARKER)
{
buffer_size = buffer::u4swap(initialbuffer[0]);
}
Expand Down Expand Up @@ -256,12 +266,14 @@ int fileEventiterator::read_next_buffer()

// and initialize the current_index to be the first event

if ( ( initialbuffer[1]== GZBUFFERMARKER ||
buffer::u4swap(initialbuffer[1])== GZBUFFERMARKER ||
initialbuffer[1]== LZO1XBUFFERMARKER ||
buffer::u4swap(initialbuffer[1])== LZO1XBUFFERMARKER ||
initialbuffer[1]== LZO1CBUFFERMARKER ||
buffer::u4swap(initialbuffer[1])== LZO1CBUFFERMARKER )
if ( ( initialbuffer[1]== GZBUFFERMARKER
|| buffer::u4swap(initialbuffer[1])== GZBUFFERMARKER
|| initialbuffer[1]== LZO1XBUFFERMARKER
|| buffer::u4swap(initialbuffer[1])== LZO1XBUFFERMARKER
|| initialbuffer[1]== LZO1CBUFFERMARKER
|| buffer::u4swap(initialbuffer[1])== LZO1CBUFFERMARKER
|| initialbuffer[1]== LZO2ABUFFERMARKER
|| buffer::u4swap(initialbuffer[1])== LZO2ABUFFERMARKER )
&& errorinread )
{
bptr = 0;
Expand Down
17 changes: 14 additions & 3 deletions newbasic/lzobuffer.cc
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,16 @@ lzobuffer::lzobuffer (PHDWORD *array , const int length )

lzo_uint bytes;
lzo_uint outputlength_in_bytes;
if (array[1] == LZO1XBUFFERMARKER || array[1] == LZO1CBUFFERMARKER)
if (array[1] == LZO1XBUFFERMARKER ||
array[1] == LZO1CBUFFERMARKER ||
array[1] == LZO2ABUFFERMARKER )
{
bytes = array[0]-4*BUFFERHEADERLENGTH;
outputlength_in_bytes = array[3];
}
else if ( u4swap(array[1]) == LZO1XBUFFERMARKER || u4swap(array[1]) == LZO1CBUFFERMARKER)
else if ( u4swap(array[1]) == LZO1XBUFFERMARKER
|| u4swap(array[1]) == LZO1CBUFFERMARKER
|| u4swap(array[1]) == LZO2ABUFFERMARKER)
{
bytes = i4swap(array[0])-16;
outputlength_in_bytes = i4swap(array[3]);
Expand All @@ -59,13 +63,20 @@ lzobuffer::lzobuffer (PHDWORD *array , const int length )
lzo1x_decompress_safe ( (lzo_byte *) &array[4], bytes,
(lzo_byte *) bufferarray, &olen, NULL );
}
else
else if (array[1] == LZO1CBUFFERMARKER || u4swap(array[1]) == LZO1CBUFFERMARKER )
{

lzo1c_decompress_safe ( (lzo_byte *) &array[4], bytes,
(lzo_byte *) bufferarray, &olen, NULL );

}
else
{

lzo2a_decompress_safe ( (lzo_byte *) &array[4], bytes,
(lzo_byte *) bufferarray, &olen, NULL );

}

// std::cout << __FILE__ << " " << __LINE__ << " array length after is " << array[-1] << std::endl;

Expand Down
1 change: 1 addition & 0 deletions newbasic/lzobuffer.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "prdfBuffer.h"
#include <lzo/lzo1x.h>
#include <lzo/lzo1c.h>
#include <lzo/lzo2a.h>


#ifndef __CINT__
Expand Down
32 changes: 30 additions & 2 deletions newbasic/prdfcheck.cc
Original file line number Diff line number Diff line change
Expand Up @@ -56,15 +56,21 @@ main(int argc, char *argv[])
buffer[1] == LZO1XBUFFERMARKER ||
buffer::u4swap(buffer[1]) == LZO1XBUFFERMARKER ||
buffer[1] == LZO1CBUFFERMARKER ||
buffer::u4swap(buffer[1]) == LZO1CBUFFERMARKER )
buffer::u4swap(buffer[1]) == LZO1CBUFFERMARKER ||
buffer[1] == LZO2ABUFFERMARKER ||
buffer::u4swap(buffer[1]) == LZO2ABUFFERMARKER ||
buffer[1] == BZ2BUFFERMARKER ||
buffer::u4swap(buffer[1]) == BZ2BUFFERMARKER )
{


if ( buffer::u4swap(buffer[1]) == BUFFERMARKER ||
buffer::u4swap(buffer[1]) == ONCSBUFFERMARKER ||
buffer::u4swap(buffer[1]) == GZBUFFERMARKER ||
buffer::u4swap(buffer[1]) == LZO1XBUFFERMARKER ||
buffer::u4swap(buffer[1]) == LZO1CBUFFERMARKER )
buffer::u4swap(buffer[1]) == LZO1CBUFFERMARKER ||
buffer::u4swap(buffer[1]) == LZO2ABUFFERMARKER ||
buffer::u4swap(buffer[1]) == BZ2BUFFERMARKER )
{
needs_swap = 1;
}
Expand Down Expand Up @@ -102,6 +108,8 @@ main(int argc, char *argv[])
else if ( buffer[1] == LZO1XBUFFERMARKER ||
buffer::u4swap(buffer[1]) == LZO1XBUFFERMARKER ||
buffer[1] == LZO1CBUFFERMARKER ||
buffer::u4swap(buffer[1]) == LZO1CBUFFERMARKER ||
buffer[1] == LZO2ABUFFERMARKER ||
buffer::u4swap(buffer[1]) == LZO1CBUFFERMARKER )
{
std::cout << "LZO Marker ";
Expand All @@ -121,6 +129,26 @@ main(int argc, char *argv[])
}
}

else if ( buffer[1] == BZ2BUFFERMARKER ||
buffer::u4swap(buffer[1]) == BZ2BUFFERMARKER )
{
std::cout << "BZ2 Marker ";
std::cout << " Or.length: " << buffer[3];
float ratio = 100.*buffer[0]/buffer[3];
std::cout << " " << ratio << "%";

int e = buffer[2] & 0xffff;
int atp = (buffer[2] >> 16) & 0xffff;
if ( atp)
{
std::cout << " events: " << e << " from ATP " << atp << std::endl;
}
else
{
std::cout << std::endl;
}
}


while ( ip < length)
{
Expand Down
8 changes: 6 additions & 2 deletions newbasic/prdfsplit.cc
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,19 @@ int check_buffermarker ( const unsigned int bm)
if ( bm == BUFFERMARKER ||
bm == GZBUFFERMARKER ||
bm == LZO1XBUFFERMARKER ||
bm == LZO1CBUFFERMARKER )
bm == LZO1CBUFFERMARKER ||
bm == LZO2ABUFFERMARKER ||
bm == BZ2BUFFERMARKER )
{
return 1;
}

else if ( buffer::u4swap(bm) == BUFFERMARKER ||
buffer::u4swap(bm) == GZBUFFERMARKER ||
buffer::u4swap(bm) == LZO1XBUFFERMARKER ||
buffer::u4swap(bm) == LZO1CBUFFERMARKER )
buffer::u4swap(bm) == LZO1CBUFFERMARKER ||
buffer::u4swap(bm) == LZO2ABUFFERMARKER ||
buffer::u4swap(bm) == BZ2BUFFERMARKER )
{
return -1;
}
Expand Down

0 comments on commit bbddd86

Please sign in to comment.