From d90c7d5d674cf006cc27e5080be3f2029162f394 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 2 Dec 2023 06:26:17 +0800 Subject: [PATCH 1/8] fix calculate checkSum when using Java9IntHash --- .../main/java/com/scurrilous/circe/checksum/Java9IntHash.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java index 622c66d9ee9..9e26f8d7631 100644 --- a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java +++ b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java @@ -109,11 +109,13 @@ public int resume(int current, ByteBuf buffer, int offset, int len) { } else { byte[] b = TL_BUFFER.get(); int toRead = len; + int loopOffset = offset; while (toRead > 0) { int length = Math.min(toRead, b.length); - buffer.slice(offset, len).readBytes(b, 0, length); + buffer.slice(loopOffset, len).readBytes(b, 0, length); negCrc = resume(negCrc, b, 0, length); toRead -= length; + loopOffset += length; } } From 8a4c55854b625d866cde28e2e41314ab2e5483b2 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 2 Dec 2023 07:02:59 +0800 Subject: [PATCH 2/8] - --- .../main/java/com/scurrilous/circe/checksum/Java9IntHash.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java index 9e26f8d7631..85c140f3f8b 100644 --- a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java +++ b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java @@ -112,7 +112,7 @@ public int resume(int current, ByteBuf buffer, int offset, int len) { int loopOffset = offset; while (toRead > 0) { int length = Math.min(toRead, b.length); - buffer.slice(loopOffset, len).readBytes(b, 0, length); + buffer.slice(loopOffset, length).readBytes(b, 0, length); negCrc = resume(negCrc, b, 0, length); toRead -= length; loopOffset += length; From e197574f7ff234c8c73087763039e86ccdff2aed Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 2 Dec 2023 07:27:04 +0800 Subject: [PATCH 3/8] improve performence --- .../java/com/scurrilous/circe/checksum/Java9IntHash.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java index 85c140f3f8b..6e4f4f6d71c 100644 --- a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java +++ b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java @@ -19,6 +19,7 @@ package com.scurrilous.circe.checksum; import io.netty.buffer.ByteBuf; +import io.netty.buffer.CompositeByteBuf; import io.netty.util.concurrent.FastThreadLocal; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; @@ -106,6 +107,12 @@ public int resume(int current, ByteBuf buffer, int offset, int len) { } else if (buffer.hasArray()) { int arrayOffset = buffer.arrayOffset() + offset; negCrc = resume(negCrc, buffer.array(), arrayOffset, len); + } else if (buffer instanceof CompositeByteBuf) { + CompositeByteBuf compositeByteBuf = (CompositeByteBuf) buffer; + for (int i = 0; i < compositeByteBuf.numComponents(); i ++) { + negCrc = resume(~negCrc, compositeByteBuf.component(i)); + } + return ~negCrc; } else { byte[] b = TL_BUFFER.get(); int toRead = len; From 5440cd19e2f025fd77987d37942c59688c711887 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 2 Dec 2023 07:30:47 +0800 Subject: [PATCH 4/8] add test --- .../circe/checksum/Java9IntHashTest.java | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java diff --git a/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java new file mode 100644 index 00000000000..a0f592f03d8 --- /dev/null +++ b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java @@ -0,0 +1,113 @@ +package com.scurrilous.circe.checksum; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; +import io.netty.buffer.CompositeByteBuf; +import io.netty.buffer.DuplicatedByteBuf; +import java.util.Random; +import lombok.extern.slf4j.Slf4j; +import org.junit.Assert; +import org.junit.Test; + +@Slf4j +public class Java9IntHashTest { + + @Test + public void calculateCheckSum() { + Random random = new Random(); + int hugeDataLen = 4096 * 3; + byte[] hugeData = new byte[hugeDataLen]; + for (int i = 0; i < hugeDataLen; i ++) { + hugeData[i] = (byte) (random.nextInt() % 127); + } + + // b_total = b1 + b2 + b3; + ByteBuf bTotal = ByteBufAllocator.DEFAULT.heapBuffer(6 + hugeDataLen); + bTotal.writeBytes(new byte[]{1,2,3,4,5,6}); + bTotal.writeBytes(hugeData); + ByteBuf b1 = ByteBufAllocator.DEFAULT.heapBuffer(3); + b1.writeBytes(new byte[]{1,2,3}); + ByteBuf b2 = ByteBufAllocator.DEFAULT.heapBuffer(3); + b2.writeBytes(new byte[]{4,5,6}); + ByteBuf b3 = ByteBufAllocator.DEFAULT.heapBuffer(hugeDataLen); + b2.writeBytes(hugeData); + + // Calculate: case-1. + int checksumRes1 = Crc32cIntChecksum.computeChecksum(bTotal); + log.info("checksumRes1: {}", checksumRes1); + + // Calculate: case-2. + int b1CheckSum = Crc32cIntChecksum.computeChecksum(b1); + log.info("b1CheckSum: {}", b1CheckSum); + int checksumRes2 = Crc32cIntChecksum.resumeChecksum(b1CheckSum, + new CompositeByteBuf(ByteBufAllocator.DEFAULT, false, 2, b2, b3)); + log.info("checksumRes2: {}", checksumRes2); + + // Verify: the results of both ways to calculate the checksum are same. + Assert.assertEquals(checksumRes1, checksumRes2); + + // cleanup. + bTotal.release(); + b1.release(); + b2.release(); + b3.release(); + } + + @Test + public void calculateCheckSum2() { + Random random = new Random(); + int hugeDataLen = 4096 * 3; + byte[] hugeData = new byte[hugeDataLen]; + for (int i = 0; i < hugeDataLen; i ++) { + hugeData[i] = (byte) (random.nextInt() % 127); + } + + // b_total = b1 + b2 + b3; + ByteBuf bTotal = ByteBufAllocator.DEFAULT.heapBuffer(6 + hugeDataLen); + bTotal.writeBytes(new byte[]{1,2,3,4,5,6}); + bTotal.writeBytes(hugeData); + ByteBuf b1 = ByteBufAllocator.DEFAULT.heapBuffer(3); + b1.writeBytes(new byte[]{1,2,3}); + ByteBuf b2 = ByteBufAllocator.DEFAULT.heapBuffer(3); + b2.writeBytes(new byte[]{4,5,6}); + ByteBuf b3 = ByteBufAllocator.DEFAULT.heapBuffer(hugeDataLen); + b2.writeBytes(hugeData); + + // Calculate: case-1. + int checksumRes1 = Crc32cIntChecksum.computeChecksum(bTotal); + log.info("checksumRes1: {}", checksumRes1); + + // Calculate: case-2. + int b1CheckSum = Crc32cIntChecksum.computeChecksum(b1); + log.info("b1CheckSum: {}", b1CheckSum); + int checksumRes2 = Crc32cIntChecksum.resumeChecksum(b1CheckSum, + new NoArrayNoMemoryAddrByteBuff(new CompositeByteBuf(ByteBufAllocator.DEFAULT, false, 2, b2, b3))); + log.info("checksumRes2: {}", checksumRes2); + + // Verify: the results of both ways to calculate the checksum are same. + Assert.assertEquals(checksumRes1, checksumRes2); + + // cleanup. + bTotal.release(); + b1.release(); + b2.release(); + b3.release(); + } + + public static class NoArrayNoMemoryAddrByteBuff extends DuplicatedByteBuf { + + public NoArrayNoMemoryAddrByteBuff(ByteBuf buffer) { + super(buffer); + } + + @Override + public boolean hasArray(){ + return false; + } + + @Override + public boolean hasMemoryAddress(){ + return false; + } + } +} \ No newline at end of file From 3f453694c17ddcc57387bea392e2232a5b3312b7 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 2 Dec 2023 07:43:24 +0800 Subject: [PATCH 5/8] edit test --- .../circe/checksum/Java9IntHashTest.java | 54 ++++++++----------- 1 file changed, 22 insertions(+), 32 deletions(-) diff --git a/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java index a0f592f03d8..ee76c057274 100644 --- a/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java +++ b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java @@ -12,8 +12,7 @@ @Slf4j public class Java9IntHashTest { - @Test - public void calculateCheckSum() { + private ByteBuf[] generateByteBuffers() { Random random = new Random(); int hugeDataLen = 4096 * 3; byte[] hugeData = new byte[hugeDataLen]; @@ -32,16 +31,24 @@ public void calculateCheckSum() { ByteBuf b3 = ByteBufAllocator.DEFAULT.heapBuffer(hugeDataLen); b2.writeBytes(hugeData); + return new ByteBuf[]{bTotal, b1, new CompositeByteBuf(ByteBufAllocator.DEFAULT, false, 2, b2, b3)}; + } + + @Test + public void calculateCheckSum() { + // byteBuffers[0] = byteBuffers[1] + byteBuffers[2]. + // byteBuffers[2] is a composite ByteBuf. + ByteBuf[] byteBuffers = generateByteBuffers(); + ByteBuf bTotal = byteBuffers[0]; + ByteBuf b1 = byteBuffers[1]; + ByteBuf b2 = byteBuffers[2]; + // Calculate: case-1. int checksumRes1 = Crc32cIntChecksum.computeChecksum(bTotal); - log.info("checksumRes1: {}", checksumRes1); // Calculate: case-2. int b1CheckSum = Crc32cIntChecksum.computeChecksum(b1); - log.info("b1CheckSum: {}", b1CheckSum); - int checksumRes2 = Crc32cIntChecksum.resumeChecksum(b1CheckSum, - new CompositeByteBuf(ByteBufAllocator.DEFAULT, false, 2, b2, b3)); - log.info("checksumRes2: {}", checksumRes2); + int checksumRes2 = Crc32cIntChecksum.resumeChecksum(b1CheckSum, b2); // Verify: the results of both ways to calculate the checksum are same. Assert.assertEquals(checksumRes1, checksumRes2); @@ -50,39 +57,23 @@ public void calculateCheckSum() { bTotal.release(); b1.release(); b2.release(); - b3.release(); } @Test - public void calculateCheckSum2() { - Random random = new Random(); - int hugeDataLen = 4096 * 3; - byte[] hugeData = new byte[hugeDataLen]; - for (int i = 0; i < hugeDataLen; i ++) { - hugeData[i] = (byte) (random.nextInt() % 127); - } - - // b_total = b1 + b2 + b3; - ByteBuf bTotal = ByteBufAllocator.DEFAULT.heapBuffer(6 + hugeDataLen); - bTotal.writeBytes(new byte[]{1,2,3,4,5,6}); - bTotal.writeBytes(hugeData); - ByteBuf b1 = ByteBufAllocator.DEFAULT.heapBuffer(3); - b1.writeBytes(new byte[]{1,2,3}); - ByteBuf b2 = ByteBufAllocator.DEFAULT.heapBuffer(3); - b2.writeBytes(new byte[]{4,5,6}); - ByteBuf b3 = ByteBufAllocator.DEFAULT.heapBuffer(hugeDataLen); - b2.writeBytes(hugeData); + public void calculateCheckSumUsingNoArrayNoMemoryAddrData() { + // byteBuffers[0] = byteBuffers[1] + byteBuffers[2]. + // byteBuffers[2] is a composite ByteBuf. + ByteBuf[] byteBuffers = generateByteBuffers(); + ByteBuf bTotal = byteBuffers[0]; + ByteBuf b1 = byteBuffers[1]; + ByteBuf b2 = new NoArrayNoMemoryAddrByteBuff(byteBuffers[2]); // Calculate: case-1. int checksumRes1 = Crc32cIntChecksum.computeChecksum(bTotal); - log.info("checksumRes1: {}", checksumRes1); // Calculate: case-2. int b1CheckSum = Crc32cIntChecksum.computeChecksum(b1); - log.info("b1CheckSum: {}", b1CheckSum); - int checksumRes2 = Crc32cIntChecksum.resumeChecksum(b1CheckSum, - new NoArrayNoMemoryAddrByteBuff(new CompositeByteBuf(ByteBufAllocator.DEFAULT, false, 2, b2, b3))); - log.info("checksumRes2: {}", checksumRes2); + int checksumRes2 = Crc32cIntChecksum.resumeChecksum(b1CheckSum, b2); // Verify: the results of both ways to calculate the checksum are same. Assert.assertEquals(checksumRes1, checksumRes2); @@ -91,7 +82,6 @@ public void calculateCheckSum2() { bTotal.release(); b1.release(); b2.release(); - b3.release(); } public static class NoArrayNoMemoryAddrByteBuff extends DuplicatedByteBuf { From c184a1b11aab6618f271dba87cb8ebd828627391 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 2 Dec 2023 08:00:45 +0800 Subject: [PATCH 6/8] rename test --- .../java/com/scurrilous/circe/checksum/Java9IntHashTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java index ee76c057274..b9b58cacd61 100644 --- a/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java +++ b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java @@ -35,7 +35,7 @@ private ByteBuf[] generateByteBuffers() { } @Test - public void calculateCheckSum() { + public void calculateCheckSumUsingCompositeByteBuf() { // byteBuffers[0] = byteBuffers[1] + byteBuffers[2]. // byteBuffers[2] is a composite ByteBuf. ByteBuf[] byteBuffers = generateByteBuffers(); @@ -60,7 +60,7 @@ public void calculateCheckSum() { } @Test - public void calculateCheckSumUsingNoArrayNoMemoryAddrData() { + public void calculateCheckSumUsingNoArrayNoMemoryAddrByteBuf() { // byteBuffers[0] = byteBuffers[1] + byteBuffers[2]. // byteBuffers[2] is a composite ByteBuf. ByteBuf[] byteBuffers = generateByteBuffers(); From d2ca22c7592e7cf525d60a90f905cb793e15f4cd Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sat, 2 Dec 2023 15:39:20 +0800 Subject: [PATCH 7/8] fix license --- .../circe/checksum/Java9IntHashTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java index b9b58cacd61..85a00ee73f9 100644 --- a/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java +++ b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ package com.scurrilous.circe.checksum; import io.netty.buffer.ByteBuf; From 53cb6a64a73b3d5f3cf093c879a12582a88d6278 Mon Sep 17 00:00:00 2001 From: fengyubiao Date: Sun, 3 Dec 2023 15:41:32 +0800 Subject: [PATCH 8/8] address comments --- .../java/com/scurrilous/circe/checksum/Java9IntHash.java | 5 +++-- .../java/com/scurrilous/circe/checksum/Java9IntHashTest.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java index 6e4f4f6d71c..31af153666e 100644 --- a/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java +++ b/circe-checksum/src/main/java/com/scurrilous/circe/checksum/Java9IntHash.java @@ -109,10 +109,11 @@ public int resume(int current, ByteBuf buffer, int offset, int len) { negCrc = resume(negCrc, buffer.array(), arrayOffset, len); } else if (buffer instanceof CompositeByteBuf) { CompositeByteBuf compositeByteBuf = (CompositeByteBuf) buffer; + int loopedCurrent = current; for (int i = 0; i < compositeByteBuf.numComponents(); i ++) { - negCrc = resume(~negCrc, compositeByteBuf.component(i)); + loopedCurrent = resume(loopedCurrent, compositeByteBuf.component(i)); } - return ~negCrc; + return loopedCurrent; } else { byte[] b = TL_BUFFER.get(); int toRead = len; diff --git a/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java index 85a00ee73f9..3fb57b4a1aa 100644 --- a/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java +++ b/circe-checksum/src/test/java/com/scurrilous/circe/checksum/Java9IntHashTest.java @@ -47,7 +47,7 @@ private ByteBuf[] generateByteBuffers() { ByteBuf b2 = ByteBufAllocator.DEFAULT.heapBuffer(3); b2.writeBytes(new byte[]{4,5,6}); ByteBuf b3 = ByteBufAllocator.DEFAULT.heapBuffer(hugeDataLen); - b2.writeBytes(hugeData); + b3.writeBytes(hugeData); return new ByteBuf[]{bTotal, b1, new CompositeByteBuf(ByteBufAllocator.DEFAULT, false, 2, b2, b3)}; }