limit validity Bytebuffers after query submit. In this commit we use the limit method to limit the number of bytes in the validity ByteBuffers in READ mode. This will use less memory

Author: DimitrisStaratzis

src/main/java/io/tiledb/java/api/Query.java

@@ -133,27 +133,35 @@ public QueryStatus submit() throws TileDBError {
     // Set the actual number of bytes received to each ByteBuffer
     for (String attribute : byteBuffers_.keySet()) {
       boolean isVar;
+      boolean isNullable = false;
+      Datatype datatype;
 
       try (ArraySchema arraySchema = array.getSchema()) {
         if (arraySchema.hasAttribute(attribute)) {
           try (Attribute attr = arraySchema.getAttribute(attribute)) {
             isVar = attr.isVar();
+            isNullable = attr.getNullable();
+            datatype = attr.getType();
           }
         } else {
           try (Dimension dim = arraySchema.getDomain().getDimension(attribute)) {
             isVar = dim.isVar();
+            datatype = dim.getType();
           }
         }
       }
 
+      int nbytes = this.buffer_sizes_.get(attribute).getSecond().getitem(0).intValue();
+      this.byteBuffers_.get(attribute).getSecond().limit(nbytes);
+
       if (isVar) {
         int offset_nbytes = this.buffer_sizes_.get(attribute).getFirst().getitem(0).intValue();
-        int data_nbytes = this.buffer_sizes_.get(attribute).getSecond().getitem(0).intValue();
         this.byteBuffers_.get(attribute).getFirst().limit(offset_nbytes);
-        this.byteBuffers_.get(attribute).getSecond().limit(data_nbytes);
-      } else {
-        int nbytes = this.buffer_sizes_.get(attribute).getSecond().getitem(0).intValue();
-        this.byteBuffers_.get(attribute).getSecond().limit(nbytes);
+      }
+
+      if (isNullable) {
+        int validity_nbytes = nbytes / datatype.getNativeSize();
+        this.validityByteMapsByteBuffers_.get(attribute).limit(validity_nbytes);
       }
     }
 
@@ -177,27 +185,35 @@ public QueryStatus submitAndFinalize() throws TileDBError {
     // Set the actual number of bytes received to each ByteBuffer
     for (String attribute : byteBuffers_.keySet()) {
       boolean isVar;
+      boolean isNullable = false;
+      Datatype datatype;
 
       try (ArraySchema arraySchema = array.getSchema()) {
         if (arraySchema.hasAttribute(attribute)) {
           try (Attribute attr = arraySchema.getAttribute(attribute)) {
             isVar = attr.isVar();
+            isNullable = attr.getNullable();
+            datatype = attr.getType();
           }
         } else {
           try (Dimension dim = arraySchema.getDomain().getDimension(attribute)) {
             isVar = dim.isVar();
+            datatype = dim.getType();
           }
         }
       }
 
+      int nbytes = this.buffer_sizes_.get(attribute).getSecond().getitem(0).intValue();
+      this.byteBuffers_.get(attribute).getSecond().limit(nbytes);
+
       if (isVar) {
         int offset_nbytes = this.buffer_sizes_.get(attribute).getFirst().getitem(0).intValue();
-        int data_nbytes = this.buffer_sizes_.get(attribute).getSecond().getitem(0).intValue();
         this.byteBuffers_.get(attribute).getFirst().limit(offset_nbytes);
-        this.byteBuffers_.get(attribute).getSecond().limit(data_nbytes);
-      } else {
-        int nbytes = this.buffer_sizes_.get(attribute).getSecond().getitem(0).intValue();
-        this.byteBuffers_.get(attribute).getSecond().limit(nbytes);
+      }
+
+      if (isNullable) {
+        int validity_nbytes = nbytes / datatype.getNativeSize();
+        this.validityByteMapsByteBuffers_.get(attribute).limit(validity_nbytes);
       }
     }
 

src/test/java/io/tiledb/java/api/QueryTest.java

@@ -1238,6 +1238,15 @@ public void denseArrayNIOReadTest() throws Exception {
         // Submit query
         query.submit();
 
+        Assert.assertEquals(4, a1byteMap.limit());
+
+        try {
+          a1byteMap.get(5);
+          Assert.fail();
+        } catch (IndexOutOfBoundsException e) {
+          // do nothing. This is expected
+        }
+
         int[] dim1 = new int[4];
         int[] dim2 = new int[4];
         byte[] a1 = new byte[4];
@@ -1401,6 +1410,24 @@ public void sparseArrayNIOReadTest() throws Exception {
         // Submit query
         query.submit();
 
+        Assert.assertEquals(5, a1byteMap.limit());
+
+        try {
+          a1byteMap.get(6);
+          Assert.fail();
+        } catch (IndexOutOfBoundsException e) {
+          // do nothing. This is expected
+        }
+
+        Assert.assertEquals(10, a2byteMapBuffer.limit());
+
+        try {
+          a2byteMapBuffer.get(11);
+          Assert.fail();
+        } catch (IndexOutOfBoundsException e) {
+          // do nothing. This is expected
+        }
+
         Assert.assertEquals(
             5L,
             (long)
@@ -1436,6 +1463,7 @@ public void sparseArrayNIOReadTest() throws Exception {
 
         int idx = 0;
         int bytesIdx = 0;
+
         while (a1Buffer.hasRemaining()) {
           a1Values[idx] = a1Buffer.getInt();
           a1ByteMapValues[idx] = a1byteMap.get();