diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5cdmBuilder.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5cdmBuilder.kt index dd3131bd..e38fa22b 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5cdmBuilder.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/H5cdmBuilder.kt @@ -185,7 +185,6 @@ internal open class DataContainerAttribute( val mdlClassCount = getDataLayoutCounts() -// TODO ?? none of the version4 layouts are implemented internal class DataContainerVariable( override val name: String, override val h5type: H5TypeInfo, diff --git a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt index 4adf7a5d..1cd01aa1 100644 --- a/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt +++ b/core/src/commonMain/kotlin/com/sunya/netchdf/hdf5/Hdf5File.kt @@ -31,6 +31,11 @@ class Hdf5File(val filename : String, strict : Boolean = false) : Netchdf { override fun type() = header.formatType() override val size : Long get() = raf.size() + fun layoutName(v: Variable<*>): String { + val vinfo = (v.spObject as DataContainerVariable) + return if (vinfo.mdl != null) vinfo.mdl.javaClass.simpleName else "none" + } + override fun readArrayData(v2: Variable, section: SectionPartial?): ArrayTyped { if (v2.nelems == 0L) { return ArrayEmpty(v2.shape.toIntArray(), v2.datatype) diff --git a/core/src/commonTest/data/netcdf4/ds.mint.nc b/core/src/commonTest/data/netcdf4/ds.mint.nc new file mode 100755 index 00000000..beeba6bd Binary files /dev/null and b/core/src/commonTest/data/netcdf4/ds.mint.nc differ diff --git a/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/Btree1extTest.kt b/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/Btree1extTest.kt index 974fc46d..494e6267 100644 --- a/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/Btree1extTest.kt +++ b/core/src/commonTest/kotlin/com/sunya/netchdf/hdf5/Btree1extTest.kt @@ -2,27 +2,30 @@ package com.sunya.netchdf.hdf5 -import com.sunya.cdm.api.computeSize +import com.sunya.cdm.api.ArraySection import com.sunya.cdm.api.toLongArray import com.sunya.cdm.iosp.OkioFile import com.sunya.cdm.iosp.OpenFileIF -import com.sunya.cdm.layout.Tiling import com.sunya.cdm.util.InternalLibraryApi +import com.sunya.netchdf.testutil.nano +import com.sunya.netchdf.testutil.testData +import kotlin.system.measureNanoTime import kotlin.test.Test class Btree1extTest { + // 227322 == /home/all/testdata/cdmUnitTest/formats/netcdf4/ds.mint.nc#Minimum_temperature_surface_12_Hour_Minimum#DataLayoutBTreeVer1 } files + @Test fun testBTree1ext() { - val filename = "/home/all/testdata/cdmUnitTest/formats/netcdf4/hiig_forec_20140208.nc" - val varname = "salt" + val filename = testData + "netcdf4/ds.mint.nc" + val varname = "Minimum_temperature_surface_12_Hour_Minimum" Hdf5File(filename).use { myfile -> println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes") - // println(myfile.cdl()) + println(myfile.cdl()) val h5 = myfile.header - var countChunks = 0 val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } ?: throw RuntimeException("cant find $varname") @@ -36,18 +39,11 @@ class Btree1extTest { require(myvar.spObject is DataContainerVariable) val vinfo = myvar.spObject - require(vinfo.mdl is DataLayoutBTreeVer1) + require(vinfo.mdl is DataLayoutBTreeVer1) {"must use DataLayoutBTreeVer1"} val mdl = vinfo.mdl val chunkShape = mdl.chunkDims - val tiling = Tiling(varShape, chunkShape.toLongArray()) - val nchunks = tiling.tileShape.computeSize() // a thread-safe accessor of the btree - // val raf: OpenFileExtended, - // val rootNodeAddress: Long, - // val nodeType : Int, // 0 = group/symbol table, 1 = raw data chunks - // varShape: LongArray, - // chunkShape: LongArray, val bTreeExt = BTree1ext(rafext, mdl.btreeAddress, 1, varShape, chunkShape.toLongArray()) val rootNode = bTreeExt.rootNode() @@ -55,4 +51,28 @@ class Btree1extTest { } } + @Test + fun testH5readConcurrent() { + val filename = testData + "netcdf4/ds.mint.nc" + val varname = "Minimum_temperature_surface_12_Hour_Minimum" + Hdf5File(filename).use { myfile -> + println("${myfile.type()} $filename ${myfile.size / 1000.0 / 1000.0} Mbytes") + + val myvar = myfile.rootGroup().allVariables().find { it.fullname() == varname } + ?: throw RuntimeException("cant find $varname") + + println("nthreads, time in secs") + + for (nthreads in listOf(1, 2, 4, 8, 10, 16, 20, 24, 32, 40, 48)) { + val time = measureNanoTime { + val reader = H5readConcurrent(myfile, myvar) + reader.readChunks(nthreads) { asect: ArraySection<*> -> + // println(" section = ${asect.section}") + } + } + println("$nthreads, ${time * nano}") + } + } + } + } diff --git a/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/ReadCommon.kt b/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/ReadCommon.kt index 6a6ea341..a18cfb46 100644 --- a/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/ReadCommon.kt +++ b/core/src/commonTest/kotlin/com/sunya/netchdf/testutil/ReadCommon.kt @@ -8,6 +8,8 @@ import kotlin.test.assertTrue ////////////////////////////////////////////////////////////////////////////////////////////////////// +const val nano = 1.0e-9 + fun showNetchdfHeader(filename: String) { println(filename) openNetchdfFile(filename).use { myfile -> diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/CountVersions.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/CountVersions.kt index 5feafab5..10739aa0 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/CountVersions.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/CountVersions.kt @@ -1,5 +1,6 @@ package com.sunya.netchdf +import com.sunya.netchdf.hdf5.Hdf5File import com.sunya.netchdf.testfiles.H4Files import com.sunya.netchdf.testfiles.H5Files import com.sunya.netchdf.testfiles.JhdfFiles @@ -112,8 +113,13 @@ class CountVersions { if (ncfile == null) { println("Not a netchdf file=$filename ") } else { + val hdf5File = if (ncfile.type() in listOf("netcdf4", "hdf")) { + ncfile as Hdf5File + } else null + ncfile.rootGroup().allVariables().forEach { v -> - varSizes["$filename#${v.name}" ] = v.nelems + val layout = hdf5File?.layoutName(v) ?: "" + varSizes["$filename#${v.name}#$layout"] = v.nelems } } } diff --git a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt index 6dcbcabe..025ee945 100644 --- a/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt +++ b/testfiles/src/test/kotlin/com/sunya/netchdf/hdf5/H5readConcurrentTest.kt @@ -45,7 +45,6 @@ class H5readConcurrentTest { } } - @Test fun testH5readConcurrent() { val filename = "/home/all/testdata/cdmUnitTest/formats/netcdf4/hiig_forec_20140208.nc"