diff --git a/src/main/scala/com/sparkfits/FitsLib.scala b/src/main/scala/com/sparkfits/FitsLib.scala index 03c1ec9..3f415f9 100644 --- a/src/main/scala/com/sparkfits/FitsLib.scala +++ b/src/main/scala/com/sparkfits/FitsLib.scala @@ -138,6 +138,9 @@ object FitsLib { */ class Fits(hdfsPath : Path, conf : Configuration, hduIndex : Int) { + // Memory of the pointers during HDU initialization + val pointers = Array.newBuilder[(String, Long)] + // Open the data val fs = hdfsPath.getFileSystem(conf) val data = fs.open(hdfsPath) @@ -179,6 +182,7 @@ object FitsLib { // Check whether we know the HDU type. val hduType = getHduType val hdu: HDU = hduType match { + case "ZIMAGE" => handleZImage case "BINTABLE" => handleBintable case "IMAGE" => handleImage case "TABLE" => AnyHDU() @@ -199,6 +203,17 @@ object FitsLib { FitsHduImage.ImageHDU(blockHeader) } + /** + * Give access to methods concerning ZImage HDU. + * + * @return (ZImageHDU) + * + */ + def handleZImage : FitsHduZImage.ZImageHDU = { + // Grab only columns specified by the user + FitsHduZImage.ZImageHDU(blockHeader) + } + /** * Give access to methods concerning BinTable HDU. * @@ -291,18 +306,22 @@ object FitsLib { def getHduType : String = { // Get the header NAMES - val colNames = parseHeader(blockHeader) + val keyValues = parseHeader(blockHeader) - // Check if the HDU is empty, a table or an image - val isBintable = colNames.filter( + // Check if the HDU is empty, or some specific Fits type + val isZImage = keyValues.filter( + x=>x._1.contains("ZIMAGE") && (x._2.trim == "T")).values.toList.size > 0 + val isBintable = keyValues.filter( x=>x._2.contains("BINTABLE")).values.toList.size > 0 - val isTable = colNames.filter( + val isTable = keyValues.filter( x=>x._2.contains("TABLE")).values.toList.size > 0 - val isImage = colNames.filter( + val isImage = keyValues.filter( x=>x._2.contains("IMAGE")).values.toList.size > 0 val isEmpty = empty_hdu - val fitstype = if (isBintable) { + val fitstype = if (isZImage) { + "ZIMAGE" + } else if (isBintable) { "BINTABLE" } else if (isTable) { "TABLE" @@ -359,18 +378,29 @@ object FitsLib { var hasData : Boolean = false var datalen = 0L + var last_pointer = 0L + // Loop over all HDU, and exit. do { + pointers += s"absstart$currentHduIndex" -> (data.getPos) + pointers += s"headerstart$currentHduIndex" -> (data.getPos - last_pointer) + last_pointer = data.getPos + val localHeader = readFullHeaderBlocks - // If the header cannot be read, + // Test if the header cannot be read, if (localHeader.size > 0) { hasData = true // Size of the data block in Bytes. // Skip Data if None (typically HDU=0) datalen = Try { - getDataLen(parseHeader(localHeader)) + val keyValues = parseHeader(localHeader) + if (keyValues.contains("PCOUNT")){ + // pointers += keyValues("PCOUNT").toLong + (keyValues("NAXIS").toLong * keyValues("NAXIS1").toLong * keyValues("NAXIS2").toLong) + pointers += s"PCOUNT$currentHduIndex" -> keyValues("PCOUNT").toLong + } + getDataLen(keyValues) }.getOrElse(0L) // Store the offset to the next HDU @@ -383,6 +413,12 @@ object FitsLib { datalen + FITSBLOCK_SIZE_BYTES - (datalen % FITSBLOCK_SIZE_BYTES) } + val n = data.getPos - last_pointer + pointers += s"absdatastart$currentHduIndex" -> data.getPos + pointers += s"datastart$currentHduIndex" -> n + pointers += s"data$currentHduIndex" -> skipBytes + last_pointer = data.getPos + data.seek(data.getPos + skipBytes) // Move to the another HDU if needed diff --git a/src/test/scala/com/sparkfits/FitsLibTest.scala b/src/test/scala/com/sparkfits/FitsLibTest.scala index 28fa29d..853549c 100644 --- a/src/test/scala/com/sparkfits/FitsLibTest.scala +++ b/src/test/scala/com/sparkfits/FitsLibTest.scala @@ -86,6 +86,40 @@ class FitsLibTest extends FunSuite with BeforeAndAfterAll { ) } + + test("FitsLib test: Can you initialise correctly methods of a Image HDU?") { + val tablefile = new Path("src/test/resources/toTest/tst0009.fits") + val fB1 = new Fits(tablefile, conf, 2) + val p = fB1.pointers.result.size + val keyValue = FitsLib.parseHeader(fB1.blockHeader) + assert( + fB1.hdu.implemented == true && + fB1.pointers.result.size == s"p=$p" + ) + } + + + test("FitsLib test: Can you initialise correctly methods of a ZImage HDU?") { + val file = new Path("hdfs://134.158.75.222:8020//lsst/images/a.fits.fz") + val fB1 = new Fits(file, conf, 1) + val p = fB1.pointers.result.size + + val keyValue = FitsLib.parseHeader(fB1.blockHeader) + assert( + fB1.hdu.implemented == true && + fB1.hdu.getNRows(keyValue) == 1024L && + fB1.hdu.getSizeRowBytes(keyValue) == 1024 && + /* + fB1.hdu.getNCols(keyValue) == 1L && + fB1.hdu.getColTypes(keyValue) == null && + fB1.hdu.listOfStruct != null && + fB1.hdu.getRow(Array(0)) != null && + fB1.hdu.getElementFromBuffer(Array(0), "") != null && + */ + fB1.pointers.result.size == s"p=$p" + ) + } + // Check that the HDU asked is below the max HDU index. test("FitsLib test: Can you compute correctly the boundaries of a HDU?") { val fB1 = new Fits(file, conf, 1) diff --git a/src/test/scala/com/sparkfits/ReadFitsTest.scala b/src/test/scala/com/sparkfits/ReadFitsTest.scala index c3bcee7..b40ab47 100644 --- a/src/test/scala/com/sparkfits/ReadFitsTest.scala +++ b/src/test/scala/com/sparkfits/ReadFitsTest.scala @@ -89,6 +89,15 @@ class ReadFitsTest extends FunSuite with BeforeAndAfterAll { assert(count == 155) } + test("HDU type test: test if HDU is an zimage?") { + val fn_image = "hdfs://134.158.75.222:8020//lsst/images/a.fits.fz" + val results = spark.read.format("com.sparkfits") + .option("hdu", 1) + .load(fn_image) + val count = results.count() + assert(count == 36) + } + // Test if the user provides the data type in the HDU test("HDU type test: Return an empty DF if the HDU is a Table? (not implemented yet)") { val fn_table = "src/test/resources/toTest/tst0009.fits"