diff --git a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamUploader.java b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamUploader.java index 18aaa6a807a7cf..fd53932fce7f78 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/ByteStreamUploader.java +++ b/src/main/java/com/google/devtools/build/lib/remote/ByteStreamUploader.java @@ -21,6 +21,7 @@ import static java.util.concurrent.TimeUnit.SECONDS; import build.bazel.remote.execution.v2.Digest; +import build.bazel.remote.execution.v2.DigestFunction; import com.google.bytestream.ByteStreamGrpc; import com.google.bytestream.ByteStreamGrpc.ByteStreamFutureStub; import com.google.bytestream.ByteStreamGrpc.ByteStreamStub; @@ -69,6 +70,7 @@ final class ByteStreamUploader { private final CallCredentialsProvider callCredentialsProvider; private final long callTimeoutSecs; private final RemoteRetrier retrier; + private final DigestFunction.Value digestFunction; @Nullable private final Semaphore openedFilePermits; @@ -89,7 +91,8 @@ final class ByteStreamUploader { CallCredentialsProvider callCredentialsProvider, long callTimeoutSecs, RemoteRetrier retrier, - int maximumOpenFiles) { + int maximumOpenFiles, + DigestFunction.Value digestFunction) { checkArgument(callTimeoutSecs > 0, "callTimeoutSecs must be gt 0."); this.instanceName = instanceName; this.channel = channel; @@ -97,6 +100,7 @@ final class ByteStreamUploader { this.callTimeoutSecs = callTimeoutSecs; this.retrier = retrier; this.openedFilePermits = maximumOpenFiles != -1 ? new Semaphore(maximumOpenFiles) : null; + this.digestFunction = digestFunction; } @VisibleForTesting @@ -175,11 +179,34 @@ public ListenableFuture uploadBlobAsync( MoreExecutors.directExecutor()); } - private static String buildUploadResourceName( + private boolean isOldStyleDigestFunction() { + // Old-style digest functions (SHA256, etc) are distinguishable by the length + // of their hash alone and do not require extra specification, but newer + // digest functions (which may have the same length hashes as the older + // functions!) must be explicitly specified in the upload resource name. + return digestFunction.getNumber() <= 7; + } + + private String buildUploadResourceName( String instanceName, UUID uuid, Digest digest, boolean compressed) { - String template = - compressed ? "uploads/%s/compressed-blobs/zstd/%s/%d" : "uploads/%s/blobs/%s/%d"; - String resourceName = format(template, uuid, digest.getHash(), digest.getSizeBytes()); + + String resourceName; + + if (isOldStyleDigestFunction()) { + String template = + compressed ? "uploads/%s/compressed-blobs/zstd/%s/%d" : "uploads/%s/blobs/%s/%d"; + resourceName = format(template, uuid, digest.getHash(), digest.getSizeBytes()); + } else { + String template = + compressed ? "uploads/%s/compressed-blobs/zstd/%s/%s/%d" : "uploads/%s/blobs/%s/%s/%d"; + resourceName = + format( + template, + uuid, + digestFunction.getValueDescriptor().getName().toLowerCase(), + digest.getHash(), + digest.getSizeBytes()); + } if (!Strings.isNullOrEmpty(instanceName)) { resourceName = instanceName + "/" + resourceName; } diff --git a/src/main/java/com/google/devtools/build/lib/remote/GrpcCacheClient.java b/src/main/java/com/google/devtools/build/lib/remote/GrpcCacheClient.java index 470d3845dc3273..13830d2f6ee2a5 100644 --- a/src/main/java/com/google/devtools/build/lib/remote/GrpcCacheClient.java +++ b/src/main/java/com/google/devtools/build/lib/remote/GrpcCacheClient.java @@ -107,7 +107,8 @@ public GrpcCacheClient( callCredentialsProvider, options.remoteTimeout.getSeconds(), retrier, - options.maximumOpenFiles); + options.maximumOpenFiles, + digestUtil.getDigestFunction()); maxMissingBlobsDigestsPerMessage = computeMaxMissingBlobsDigestsPerMessage(); Preconditions.checkState( maxMissingBlobsDigestsPerMessage > 0, "Error: gRPC message size too small."); diff --git a/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java b/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java index 2b713adff59075..b146b3bbe9be24 100644 --- a/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java +++ b/src/main/java/com/google/devtools/build/lib/util/Fingerprint.java @@ -14,9 +14,10 @@ package com.google.devtools.build.lib.util; -import static java.nio.charset.StandardCharsets.UTF_8; -import com.google.common.io.ByteStreams; +import com.google.common.hash.Funnels; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; import com.google.devtools.build.lib.vfs.DigestHashFunction; import com.google.devtools.build.lib.vfs.Path; import com.google.devtools.build.lib.vfs.PathFragment; @@ -24,16 +25,14 @@ import com.google.protobuf.ByteString; import com.google.protobuf.CodedOutputStream; import java.io.IOException; -import java.security.DigestException; -import java.security.DigestOutputStream; -import java.security.MessageDigest; +import java.nio.charset.StandardCharsets; import java.util.Collection; import java.util.Map; import java.util.UUID; import javax.annotation.Nullable; /** - * Simplified wrapper for using {@link MessageDigest} to generate fingerprints. + * Simplified wrapper for using {@link HashFunction} to generate fingerprints. * *

A fingerprint is a cryptographic hash of a message that encodes the representation of an * object. Two objects of the same type have the same fingerprint if and only if they are equal. @@ -64,18 +63,14 @@ public final class Fingerprint { // Make novel use of a CodedOutputStream, which is good at efficiently serializing data. By // flushing at the end of each digest we can continue to use the stream. - private final CodedOutputStream codedOut; - private final MessageDigest messageDigest; + private final HashFunction hashFunction; + private Hasher hasher; + private CodedOutputStream codedOut; /** Creates and initializes a new instance. */ public Fingerprint(DigestHashFunction digestFunction) { - messageDigest = digestFunction.cloneOrCreateMessageDigest(); - // This is a lot of indirection, but CodedOutputStream does a reasonable job of converting - // strings to bytes without creating a whole bunch of garbage, which pays off. - codedOut = - CodedOutputStream.newInstance( - new DigestOutputStream(ByteStreams.nullOutputStream(), messageDigest), - /*bufferSize=*/ 1024); + hashFunction = digestFunction.getHashFunction(); + reset(); } public Fingerprint() { @@ -84,6 +79,14 @@ public Fingerprint() { this(DigestHashFunction.SHA256); } + private void reset() { + hasher = hashFunction.newHasher(); + // This is a lot of indirection, but CodedOutputStream does a reasonable job of converting + // strings to bytes without creating a whole bunch of garbage, which pays off. + codedOut = + CodedOutputStream.newInstance(Funnels.asOutputStream(hasher), /* bufferSize= */ 1024); + } + /** * Completes the hash computation by doing final operations and resets the underlying state, * allowing this instance to be used again. @@ -97,7 +100,9 @@ public byte[] digestAndReset() { } catch (IOException e) { throw new IllegalStateException("failed to flush", e); } - return messageDigest.digest(); + byte[] digest = hasher.hash().asBytes(); + reset(); + return digest; } /** @@ -112,12 +117,12 @@ public byte[] digestAndReset() { public void digestAndReset(byte[] buf, int offset, int len) { try { codedOut.flush(); - messageDigest.digest(buf, offset, len); } catch (IOException e) { throw new IllegalStateException("failed to flush", e); - } catch (DigestException e) { - throw new IllegalStateException("failed to digest", e); } + + hasher.hash().writeBytesTo(buf, offset, len); + reset(); } /** Same as {@link #digestAndReset()}, except returns the digest in hex string form. */ @@ -343,7 +348,9 @@ public static String getHexDigest(String input) { // use the value from DigestHashFunction.getDefault(). However, this gets called during class // loading in a few places, before setDefault() has been called, so these call-sites should be // removed before this can be done safely. - return hexDigest( - DigestHashFunction.SHA256.cloneOrCreateMessageDigest().digest(input.getBytes(UTF_8))); + return DigestHashFunction.SHA256 + .getHashFunction() + .hashString(input, StandardCharsets.UTF_8) + .toString(); } } diff --git a/src/main/java/com/google/devtools/build/lib/vfs/BUILD b/src/main/java/com/google/devtools/build/lib/vfs/BUILD index f80997f0305041..22dd4272cafd36 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/BUILD +++ b/src/main/java/com/google/devtools/build/lib/vfs/BUILD @@ -66,6 +66,7 @@ java_library( ":pathfragment", "//src/main/java/com/google/devtools/build/lib/clock", "//src/main/java/com/google/devtools/build/lib/concurrent", + "//src/main/java/com/google/devtools/build/lib/jni", "//src/main/java/com/google/devtools/build/lib/io:file_symlink_exception", "//src/main/java/com/google/devtools/build/lib/profiler", "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec", @@ -77,6 +78,7 @@ java_library( "//third_party:guava", "//third_party:jsr305", "//third_party/protobuf:protobuf_java", + "@maven//:com_google_errorprone_error_prone_annotations", ], ) diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Blake3HashFunction.java b/src/main/java/com/google/devtools/build/lib/vfs/Blake3HashFunction.java new file mode 100644 index 00000000000000..d9dbe9b421bf47 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/Blake3HashFunction.java @@ -0,0 +1,65 @@ +package com.google.devtools.build.lib.vfs; + +import static com.google.common.base.Preconditions.checkArgument; +import static com.google.common.base.Preconditions.checkPositionIndexes; + +import com.google.common.hash.Funnel; +import com.google.common.hash.HashCode; +import com.google.common.hash.HashFunction; +import com.google.common.hash.Hasher; +import com.google.errorprone.annotations.Immutable; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +@Immutable +public final class Blake3HashFunction implements HashFunction { + public int bits() { + return 256; + } + + public Hasher newHasher() { + return new Blake3Hasher(); + } + + /* The following methods implement the {Hasher} interface. */ + + public HashCode hashObject(T instance, Funnel funnel) { + return newHasher().putObject(instance, funnel).hash(); + } + + public HashCode hashUnencodedChars(CharSequence input) { + int len = input.length(); + return newHasher(len * 2).putUnencodedChars(input).hash(); + } + + public HashCode hashString(CharSequence input, Charset charset) { + return newHasher().putString(input, charset).hash(); + } + + public HashCode hashInt(int input) { + return newHasher(4).putInt(input).hash(); + } + + public HashCode hashLong(long input) { + return newHasher(8).putLong(input).hash(); + } + + public HashCode hashBytes(byte[] input) { + return hashBytes(input, 0, input.length); + } + + public HashCode hashBytes(byte[] input, int off, int len) { + checkPositionIndexes(off, off + len, input.length); + return newHasher(len).putBytes(input, off, len).hash(); + } + + public HashCode hashBytes(ByteBuffer input) { + return newHasher(input.remaining()).putBytes(input).hash(); + } + + public Hasher newHasher(int expectedInputSize) { + checkArgument( + expectedInputSize >= 0, "expectedInputSize must be >= 0 but was %s", expectedInputSize); + return newHasher(); + } +} diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Blake3Hasher.java b/src/main/java/com/google/devtools/build/lib/vfs/Blake3Hasher.java new file mode 100644 index 00000000000000..afc6b2f4084697 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/Blake3Hasher.java @@ -0,0 +1,179 @@ +package com.google.devtools.build.lib.vfs; + +import static com.google.common.base.Preconditions.checkState; + +import com.google.common.hash.Funnel; +import com.google.common.hash.HashCode; +import com.google.common.hash.Hasher; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.nio.ByteBuffer; +import java.nio.charset.Charset; + +final class Blake3Hasher implements Hasher { + // These constants match the native definitions in: + // https://github.com/BLAKE3-team/BLAKE3/blob/master/c/blake3.h + public static final int KEY_LEN = 32; + public static final int OUT_LEN = 32; + + // To reduce the number of calls made via JNI, buffer up to this many bytes. + // If a call to "hash()" is made and less than this much data has been + // written, a single JNI call will be made that initializes, hashes, and + // cleans up the hasher, rather than making separate calls for each operation. + public static final int ONESHOT_THRESHOLD = 8 * 1024; + private static ThreadLocal threadLocalBuffer = new ThreadLocal(); + private ByteBuffer buffer = null; + + private long hasher = -1; + private boolean isDone; + + public Blake3Hasher() { + isDone = false; + + buffer = threadLocalBuffer.get(); + if (buffer == null) { + buffer = ByteBuffer.allocate(ONESHOT_THRESHOLD); + threadLocalBuffer.set(buffer); + } + } + + private void flush() { + if (hasher == -1) { + hasher = Blake3JNI.allocate_and_initialize_hasher(); + } + + if (buffer.position() > 0) { + Blake3JNI.blake3_hasher_update(hasher, buffer.array(), buffer.position()); + buffer.clear(); + } + } + + private void update(byte[] data, int offset, int length) { + while (length > 0) { + int numToCopy = Math.min(length, buffer.remaining()); + buffer.put(data, offset, numToCopy); + length -= numToCopy; + offset += numToCopy; + + if (buffer.remaining() == 0) { + flush(); + } + } + } + + private void update(byte[] data) { + update(data, 0, data.length); + } + + private byte[] getOutput(int outputLength) throws IllegalArgumentException { + byte[] retByteArray = new byte[outputLength]; + + checkState(!isDone); + isDone = true; + + if (hasher == -1) { + // If no flush has happened yet; oneshot this. + Blake3JNI.oneshot(buffer.array(), buffer.position(), retByteArray, outputLength); + buffer.clear(); + } else { + flush(); + Blake3JNI.blake3_hasher_finalize_and_close(hasher, retByteArray, outputLength); + hasher = -1; + } + return retByteArray; + } + + /* The following methods implement the {Hasher} interface. */ + + @CanIgnoreReturnValue + public Hasher putBytes(ByteBuffer b) { + buffer = b; + return this; + } + + @CanIgnoreReturnValue + public Hasher putBytes(byte[] bytes, int off, int len) { + update(bytes, off, len); + return this; + } + + @CanIgnoreReturnValue + public Hasher putBytes(byte[] bytes) { + update(bytes, 0, bytes.length); + return this; + } + + @CanIgnoreReturnValue + public Hasher putByte(byte b) { + update(new byte[] {b}); + return this; + } + + public HashCode hash() { + return HashCode.fromBytes(getOutput(OUT_LEN)); + } + + @CanIgnoreReturnValue + public final Hasher putBoolean(boolean b) { + return putByte(b ? (byte) 1 : (byte) 0); + } + + @CanIgnoreReturnValue + public final Hasher putDouble(double d) { + return putLong(Double.doubleToRawLongBits(d)); + } + + @CanIgnoreReturnValue + public final Hasher putFloat(float f) { + return putInt(Float.floatToRawIntBits(f)); + } + + @CanIgnoreReturnValue + public Hasher putUnencodedChars(CharSequence charSequence) { + for (int i = 0, len = charSequence.length(); i < len; i++) { + putChar(charSequence.charAt(i)); + } + return this; + } + + @CanIgnoreReturnValue + public Hasher putString(CharSequence charSequence, Charset charset) { + return putBytes(charSequence.toString().getBytes(charset)); + } + + @CanIgnoreReturnValue + public Hasher putShort(short s) { + putByte((byte) s); + putByte((byte) (s >>> 8)); + return this; + } + + @CanIgnoreReturnValue + public Hasher putInt(int i) { + putByte((byte) i); + putByte((byte) (i >>> 8)); + putByte((byte) (i >>> 16)); + putByte((byte) (i >>> 24)); + return this; + } + + @CanIgnoreReturnValue + public Hasher putLong(long l) { + for (int i = 0; i < 64; i += 8) { + putByte((byte) (l >>> i)); + } + return this; + } + + @CanIgnoreReturnValue + public Hasher putChar(char c) { + putByte((byte) c); + putByte((byte) (c >>> 8)); + return this; + } + + @CanIgnoreReturnValue + public Hasher putObject(T instance, Funnel funnel) { + funnel.funnel(instance, this); + return this; + } +} diff --git a/src/main/java/com/google/devtools/build/lib/vfs/Blake3JNI.java b/src/main/java/com/google/devtools/build/lib/vfs/Blake3JNI.java new file mode 100644 index 00000000000000..5fe12555cd0054 --- /dev/null +++ b/src/main/java/com/google/devtools/build/lib/vfs/Blake3JNI.java @@ -0,0 +1,34 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.vfs; + +import com.google.devtools.build.lib.jni.JniLoader; + +final class Blake3JNI { + private Blake3JNI() {} + + static { + JniLoader.loadJni(); + } + + public static final native long allocate_and_initialize_hasher(); + + public static final native void blake3_hasher_update(long self, byte[] input, int input_len); + + public static final native void blake3_hasher_finalize_and_close( + long self, byte[] out, int out_len); + + public static final native void oneshot(byte[] input, int input_len, byte[] out, int out_len); +} diff --git a/src/main/java/com/google/devtools/build/lib/vfs/DigestHashFunction.java b/src/main/java/com/google/devtools/build/lib/vfs/DigestHashFunction.java index e279d7b95b7c39..b1e5ede4c8d6d5 100644 --- a/src/main/java/com/google/devtools/build/lib/vfs/DigestHashFunction.java +++ b/src/main/java/com/google/devtools/build/lib/vfs/DigestHashFunction.java @@ -23,18 +23,10 @@ import com.google.devtools.build.lib.vfs.DigestHashFunction.DigestLength.DigestLengthImpl; import com.google.devtools.common.options.Converter; import com.google.devtools.common.options.OptionsParsingException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; import java.util.HashMap; import java.util.Map.Entry; -/** - * Type of hash function to use for digesting files. - * - *

This tracks parallel {@link java.security.MessageDigest} and {@link HashFunction} interfaces - * for each provided hash, as Bazel uses both - MessageDigest where performance is critical and - * HashFunctions where ease-of-use wins over. - */ +/** Type of hash function to use for digesting files. */ // The underlying HashFunctions are immutable and thread safe. public class DigestHashFunction { // This map must be declared first to make sure that calls to register() have it ready. @@ -67,12 +59,11 @@ public int getDigestMaximumLength() { public static final DigestHashFunction SHA1 = register(Hashing.sha1(), "SHA-1", "SHA1"); public static final DigestHashFunction SHA256 = register(Hashing.sha256(), "SHA-256", "SHA256"); + public static final DigestHashFunction BLAKE3 = register(new Blake3HashFunction(), "BLAKE3"); private final HashFunction hashFunction; private final DigestLength digestLength; private final String name; - private final MessageDigest messageDigestPrototype; - private final boolean messageDigestPrototypeSupportsClone; private final ImmutableList names; private DigestHashFunction( @@ -82,8 +73,6 @@ private DigestHashFunction( checkArgument(!names.isEmpty()); this.name = names.get(0); this.names = names; - this.messageDigestPrototype = getMessageDigestInstance(); - this.messageDigestPrototypeSupportsClone = supportsClone(messageDigestPrototype); } public static DigestHashFunction register( @@ -95,8 +84,7 @@ public static DigestHashFunction register( * Creates a new DigestHashFunction that is registered to be recognized by its name in {@link * DigestFunctionConverter}. * - * @param hashName the canonical name for this hash function - and the name that can be used to - * uncover the MessageDigest. + * @param hashName the canonical name for this hash function. * @param altNames alternative names that will be mapped to this function by the converter but * will not serve as the canonical name for the DigestHashFunction. * @param hash The {@link HashFunction} to register. @@ -104,15 +92,6 @@ public static DigestHashFunction register( */ public static DigestHashFunction register( HashFunction hash, DigestLength digestLength, String hashName, String... altNames) { - try { - MessageDigest.getInstance(hashName); - } catch (NoSuchAlgorithmException e) { - throw new IllegalArgumentException( - "The hash function name provided does not correspond to a valid MessageDigest: " - + hashName, - e); - } - ImmutableList names = ImmutableList.builder().add(hashName).add(altNames).build(); DigestHashFunction hashFunction = new DigestHashFunction(hash, digestLength, names); @@ -149,19 +128,6 @@ public HashFunction getHashFunction() { return hashFunction; } - public MessageDigest cloneOrCreateMessageDigest() { - if (messageDigestPrototypeSupportsClone) { - try { - return (MessageDigest) messageDigestPrototype.clone(); - } catch (CloneNotSupportedException e) { - // We checked at initialization that this could be cloned, so this should never happen. - throw new IllegalStateException("Could not clone message digest", e); - } - } else { - return getMessageDigestInstance(); - } - } - public DigestLength getDigestLength() { return digestLength; } @@ -175,25 +141,6 @@ public String toString() { return name; } - private MessageDigest getMessageDigestInstance() { - try { - return MessageDigest.getInstance(name); - } catch (NoSuchAlgorithmException e) { - // We check when we register() this digest function that the message digest exists. This - // should never happen. - throw new IllegalStateException("message digest " + name + " not available", e); - } - } - - private static boolean supportsClone(MessageDigest toCheck) { - try { - var unused = toCheck.clone(); - return true; - } catch (CloneNotSupportedException e) { - return false; - } - } - public static ImmutableSet getPossibleHashFunctions() { return ImmutableSet.copyOf(hashFunctionRegistry.values()); } diff --git a/src/main/native/BUILD b/src/main/native/BUILD index 2601d3298cfb30..a8e91c130c4c9a 100644 --- a/src/main/native/BUILD +++ b/src/main/native/BUILD @@ -54,9 +54,23 @@ cc_library( includes = ["."], # For jni headers. ) +cc_library( + name = "blake3_jni", + srcs = [ + "blake3_jni.cc", + ":jni.h", + ":jni_md.h", + ], + deps = [ + "@blake3//:blake3", + ], + includes = ["."], # For jni headers. +) + cc_binary( name = "libunix_jni.so", srcs = [ + "blake3_jni.cc", "macros.h", "process.cc", "unix_jni.cc", @@ -81,6 +95,7 @@ cc_binary( visibility = ["//src/main/java/com/google/devtools/build/lib/jni:__pkg__"], deps = [ ":latin1_jni_path", + ":blake3_jni", "//src/main/cpp/util:logging", "//src/main/cpp/util:md5", "//src/main/cpp/util:port", diff --git a/src/main/native/blake3_jni.cc b/src/main/native/blake3_jni.cc new file mode 100644 index 00000000000000..e2c32ccc8af79b --- /dev/null +++ b/src/main/native/blake3_jni.cc @@ -0,0 +1,80 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "external/blake3/c/blake3.h" + +namespace blaze_jni { + +blake3_hasher *hasher_ptr(jlong self) { return (blake3_hasher *)self; } + +jbyte *get_byte_array(JNIEnv *env, jbyteArray java_array) { + return env->GetByteArrayElements(java_array, nullptr); +} + +void release_byte_array(JNIEnv *env, jbyteArray array, jbyte *addr) { + return env->ReleaseByteArrayElements(array, addr, 0); +} + +extern "C" JNIEXPORT jlong JNICALL +Java_com_google_devtools_build_lib_vfs_Blake3JNI_allocate_1and_1initialize_1hasher( + JNIEnv *env, jobject obj) { + blake3_hasher *hasher = new blake3_hasher; + blake3_hasher_init(hasher); + return (jlong)hasher; +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_vfs_Blake3JNI_blake3_1hasher_1update( + JNIEnv *env, jobject obj, jlong self, jbyteArray input, jint input_len) { + + jbyte *input_addr = get_byte_array(env, input); + blake3_hasher_update(hasher_ptr(self), input_addr, input_len); + release_byte_array(env, input, input_addr); +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_vfs_Blake3JNI_oneshot( + JNIEnv *env, jobject obj, jbyteArray input, jint input_len, jbyteArray out, + jint out_len) { + blake3_hasher *hasher = new blake3_hasher; + blake3_hasher_init(hasher); + + jbyte *input_addr = get_byte_array(env, input); + blake3_hasher_update(hasher, input_addr, input_len); + release_byte_array(env, input, input_addr); + + jbyte *out_addr = get_byte_array(env, out); + blake3_hasher_finalize(hasher, (uint8_t *)out_addr, out_len); + release_byte_array(env, out, out_addr); + + delete hasher; +} + +extern "C" JNIEXPORT void JNICALL +Java_com_google_devtools_build_lib_vfs_Blake3JNI_blake3_1hasher_1finalize_1and_1close( + JNIEnv *env, jobject obj, jlong self, jbyteArray out, jint out_len) { + blake3_hasher *hasher = hasher_ptr(self); + + jbyte *out_addr = get_byte_array(env, out); + blake3_hasher_finalize(hasher, (uint8_t *)out_addr, out_len); + release_byte_array(env, out, out_addr); + + delete hasher; +} + +} // namespace blaze_jni diff --git a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java index fd19dddc9e9114..53d433faa03e21 100644 --- a/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java +++ b/src/test/java/com/google/devtools/build/lib/remote/ByteStreamUploaderTest.java @@ -21,6 +21,7 @@ import static org.mockito.ArgumentMatchers.any; import build.bazel.remote.execution.v2.Digest; +import build.bazel.remote.execution.v2.DigestFunction; import build.bazel.remote.execution.v2.RequestMetadata; import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdInputStream; @@ -165,7 +166,8 @@ public void singleBlobUploadShouldWork() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -192,7 +194,8 @@ public void singleChunkCompressedUploadAlreadyExists() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /* maximumOpenFiles= */ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = {'A'}; @@ -232,8 +235,7 @@ public void onError(Throwable throwable) { } @Override - public void onCompleted() { - } + public void onCompleted() {} }; } }); @@ -256,7 +258,8 @@ public void progressiveUploadShouldWork() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, 3, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -372,7 +375,8 @@ public void progressiveCompressedUploadShouldWork() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, 300, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); int chunkSize = 1024; int skipSize = chunkSize + 1; @@ -491,7 +495,8 @@ public void progressiveCompressedUploadSeesAlreadyExistsAtTheEnd() throws Except CallCredentialsProvider.NO_CREDENTIALS, 300, retrier, - /* maximumOpenFiles= */ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); int chunkSize = 1024; byte[] blob = new byte[chunkSize * 2 + 1]; @@ -549,7 +554,8 @@ public void concurrentlyCompletedUploadIsNotRetried() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, 1, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -607,7 +613,8 @@ public void unimplementedQueryShouldRestartUpload() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, 3, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -676,7 +683,8 @@ public void earlyWriteResponseShouldCompleteUpload() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, 3, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -713,7 +721,8 @@ public void incorrectCommittedSizeFailsCompletedUpload() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, 3, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -766,7 +775,8 @@ public void incorrectCommittedSizeDoesNotFailIncompleteUpload() throws Exception CallCredentialsProvider.NO_CREDENTIALS, 300, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -798,7 +808,8 @@ public void multipleBlobsUploadShouldWork() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); int numUploads = 10; Map blobsByHash = Maps.newHashMap(); @@ -830,7 +841,8 @@ public void tooManyFilesIOException_adviseMaximumOpenFilesFlag() throws Exceptio CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE]; Chunker chunker = Mockito.mock(Chunker.class); Digest digest = DIGEST_UTIL.compute(blob); @@ -862,7 +874,8 @@ public void availablePermitsOpenFileSemaphore_fewerPermitsThanUploads_endWithAll CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - maximumOpenFiles); + maximumOpenFiles, + /* digestFunction= */ DigestFunction.Value.SHA256); assertThat(uploader.getOpenedFilePermits().availablePermits()).isEqualTo(999); @@ -900,7 +913,8 @@ public void noMaximumOpenFilesFlags_nullSemaphore() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); assertThat(uploader.getOpenedFilePermits()).isNull(); int numUploads = 10; @@ -936,7 +950,8 @@ public void contextShouldBePreservedUponRetries() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); List toUpload = ImmutableList.of("aaaaaaaaaa", "bbbbbbbbbb", "cccccccccc"); Map chunkers = Maps.newHashMapWithExpectedSize(toUpload.size()); @@ -1066,7 +1081,8 @@ public int maxConcurrency() { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE]; Chunker chunker = Chunker.builder().setInput(blob).setChunkSize(CHUNK_SIZE).build(); @@ -1127,7 +1143,8 @@ public void errorsShouldBeReported() throws IOException, InterruptedException { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE]; Chunker chunker = Chunker.builder().setInput(blob).setChunkSize(CHUNK_SIZE).build(); @@ -1163,7 +1180,8 @@ public void failureInRetryExecutorShouldBeHandled() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); serviceRegistry.addService( new ByteStreamImplBase() { @@ -1202,7 +1220,8 @@ public void resourceNameWithoutInstanceName() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); serviceRegistry.addService( new ByteStreamImplBase() { @@ -1234,6 +1253,50 @@ public void onCompleted() { uploader.uploadBlob(context, digest, chunker); } + @Test + public void resourceWithNewStyleDigestFunction() throws Exception { + RemoteRetrier retrier = + TestUtils.newRemoteRetrier(() -> mockBackoff, (e) -> true, retryService); + ByteStreamUploader uploader = + new ByteStreamUploader( + /* instanceName= */ null, + referenceCountedChannel, + CallCredentialsProvider.NO_CREDENTIALS, + /* callTimeoutSecs= */ 60, + retrier, + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.BLAKE3); + + serviceRegistry.addService( + new ByteStreamImplBase() { + @Override + public StreamObserver write(StreamObserver response) { + return new StreamObserver() { + @Override + public void onNext(WriteRequest writeRequest) { + // Test that the resource name contains the digest function. + assertThat(writeRequest.getResourceName()).contains("blobs/blake3/"); + } + + @Override + public void onError(Throwable throwable) {} + + @Override + public void onCompleted() { + response.onNext(WriteResponse.newBuilder().setCommittedSize(1).build()); + response.onCompleted(); + } + }; + } + }); + + byte[] blob = new byte[1]; + Chunker chunker = Chunker.builder().setInput(blob).setChunkSize(CHUNK_SIZE).build(); + Digest digest = DIGEST_UTIL.compute(blob); + + uploader.uploadBlob(context, digest, chunker); + } + @Test public void nonRetryableStatusShouldNotBeRetried() throws Exception { RemoteRetrier retrier = @@ -1246,7 +1309,8 @@ public void nonRetryableStatusShouldNotBeRetried() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); AtomicInteger numCalls = new AtomicInteger(); @@ -1299,7 +1363,8 @@ public void refresh() throws IOException { callCredentialsProvider, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -1355,7 +1420,8 @@ public void refresh() throws IOException { callCredentialsProvider, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); @@ -1425,7 +1491,8 @@ public void failureAfterUploadCompletes() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - -1); + -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE - 1]; new Random().nextBytes(blob); @@ -1484,7 +1551,8 @@ public void testCompressedUploads() throws Exception { CallCredentialsProvider.NO_CREDENTIALS, /* callTimeoutSecs= */ 60, retrier, - /*maximumOpenFiles=*/ -1); + /* maximumOpenFiles= */ -1, + /* digestFunction= */ DigestFunction.Value.SHA256); byte[] blob = new byte[CHUNK_SIZE * 2 + 1]; new Random().nextBytes(blob); diff --git a/src/test/java/com/google/devtools/build/lib/vfs/Blake3HasherTest.java b/src/test/java/com/google/devtools/build/lib/vfs/Blake3HasherTest.java new file mode 100644 index 00000000000000..9c101ddfa25545 --- /dev/null +++ b/src/test/java/com/google/devtools/build/lib/vfs/Blake3HasherTest.java @@ -0,0 +1,48 @@ +// Copyright 2023 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.google.devtools.build.lib.vfs; + +import static org.junit.Assert.assertEquals; + +import java.nio.charset.StandardCharsets; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; + +/** Tests for {@link Blake3Hasher}. */ +@RunWith(JUnit4.class) +public class Blake3HasherTest { + @Test + public void emptyHash() { + Blake3Hasher h = new Blake3Hasher(); + + byte[] data = new byte[0]; + h.putBytes(data); + + assertEquals( + "af1349b9f5f9a1a6a0404dea36dcc9499bcb25c9adc112b7cc9a93cae41f3262", h.hash().toString()); + } + + @Test + public void helloWorld() { + Blake3Hasher h = new Blake3Hasher(); + + byte[] data = "hello world".getBytes(StandardCharsets.US_ASCII); + h.putBytes(data); + + assertEquals( + "d74981efa70a0c880b8d8c1985d075dbcbf679b99a5f9914e5aaf96b831a9e24", h.hash().toString()); + } +} diff --git a/src/test/java/com/google/devtools/build/lib/vfs/DigestHashFunctionsTest.java b/src/test/java/com/google/devtools/build/lib/vfs/DigestHashFunctionsTest.java deleted file mode 100644 index 6b3b19923d2561..00000000000000 --- a/src/test/java/com/google/devtools/build/lib/vfs/DigestHashFunctionsTest.java +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2018 The Bazel Authors. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -package com.google.devtools.build.lib.vfs; - -import static com.google.common.truth.Truth.assertThat; - -import com.google.common.collect.ImmutableList; -import java.nio.charset.StandardCharsets; -import java.util.Collection; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; -import org.junit.runners.Parameterized.Parameter; -import org.junit.runners.Parameterized.Parameters; - -/** - * Tests different {@link DigestHashFunction} for consistency between the MessageDigests and the - * HashFunctions that it exposes. - */ -@RunWith(Parameterized.class) -public class DigestHashFunctionsTest { - @Parameters(name = "{index}: digestHashFunction={0}") - public static Collection hashFunctions() { - // TODO(b/112537387): Remove the array-ification and return Collection. This - // is possible in Junit4.12, but 4.11 requires the array. Bazel 0.18 will have Junit4.12, so - // this can change then. - return DigestHashFunction.getPossibleHashFunctions() - .stream() - .map(dhf -> new DigestHashFunction[] {dhf}) - .collect(ImmutableList.toImmutableList()); - } - - @Parameter public DigestHashFunction digestHashFunction; - - private void assertHashFunctionAndMessageDigestEquivalentForInput(byte[] input) { - byte[] hashFunctionOutput = digestHashFunction.getHashFunction().hashBytes(input).asBytes(); - byte[] messageDigestOutput = digestHashFunction.cloneOrCreateMessageDigest().digest(input); - assertThat(hashFunctionOutput).isEqualTo(messageDigestOutput); - } - - @Test - public void emptyDigestIsConsistent() { - assertHashFunctionAndMessageDigestEquivalentForInput(new byte[] {}); - } - - @Test - public void shortDigestIsConsistent() { - assertHashFunctionAndMessageDigestEquivalentForInput("Bazel".getBytes(StandardCharsets.UTF_8)); - } -} diff --git a/third_party/blake3/blake3.BUILD b/third_party/blake3/blake3.BUILD index 6e96f051f1412e..aeec30d91b982a 100644 --- a/third_party/blake3/blake3.BUILD +++ b/third_party/blake3/blake3.BUILD @@ -56,10 +56,11 @@ cc_library( "-DBLAKE3_USE_NEON=1", ], "//conditions:default": [ - "-DBLAKE3_NO_SSE2", - "-DBLAKE3_NO_SSE41", "-DBLAKE3_NO_AVX2", "-DBLAKE3_NO_AVX512", + "-DBLAKE3_NO_NEON", + "-DBLAKE3_NO_SSE2", + "-DBLAKE3_NO_SSE41", ], }), includes = ["."],