Skip to content

Commit

Permalink
support bytebuffer updates on streaming hashes
Browse files Browse the repository at this point in the history
In some cases ByteBuffers do not provide access to an underlying
byte array without incurring copies, this is notably the case
for off-heap direct bytebuffers.

This patch provides a way to call update on streaming XXHASH instances
against a ByteBuffer, which avoids the extra copy needed into a byte
array.
  • Loading branch information
pyr committed Feb 2, 2022
1 parent a9c1b3a commit e5e68ba
Show file tree
Hide file tree
Showing 8 changed files with 108 additions and 2 deletions.
7 changes: 6 additions & 1 deletion src/build/source_templates/xxhash32_streaming.template
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import static net.jpountz.xxhash.XXHashConstants.*;
import static net.jpountz.util.${type}Utils.*;
import static net.jpountz.util.SafeUtils.checkRange;
import static java.lang.Integer.rotateLeft;
import java.nio.ByteBuffer;

/**
* Streaming xxhash.
Expand Down Expand Up @@ -60,6 +61,11 @@ final class StreamingXXHash32Java${type} extends AbstractStreamingXXHash32Java {
return h32;
}

@Override
public void update(ByteBuffer buf, int off, int len) {
throw new RuntimeException("unimplemented");
}

@Override
public void update(byte[] buf, int off, int len) {
checkRange(buf, off, len);
Expand Down Expand Up @@ -139,4 +145,3 @@ final class StreamingXXHash32Java${type} extends AbstractStreamingXXHash32Java {
}

}

7 changes: 6 additions & 1 deletion src/build/source_templates/xxhash64_streaming.template
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import static net.jpountz.xxhash.XXHashConstants.*;
import static net.jpountz.util.${type}Utils.*;
import static net.jpountz.util.SafeUtils.checkRange;
import static java.lang.Long.rotateLeft;
import java.nio.ByteBuffer;

/**
* Streaming xxhash.
Expand Down Expand Up @@ -84,6 +85,11 @@ final class StreamingXXHash64Java${type} extends AbstractStreamingXXHash64Java {
return h64;
}

@Override
public void update(ByteBuffer buf, int off, int len) {
throw new RuntimeException("unimplemented");
}

@Override
public void update(byte[] buf, int off, int len) {
checkRange(buf, off, len);
Expand Down Expand Up @@ -163,4 +169,3 @@ final class StreamingXXHash64Java${type} extends AbstractStreamingXXHash64Java {
}

}

10 changes: 10 additions & 0 deletions src/java/net/jpountz/xxhash/StreamingXXHash32.java
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import java.util.zip.Checksum;
import java.io.Closeable;
import java.nio.ByteBuffer;

/*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
Expand Down Expand Up @@ -71,6 +72,15 @@ interface Factory {
*/
public abstract void update(byte[] buf, int off, int len);

/**
* Updates the value of the hash with buf[off:off+len].
*
* @param buf the input data
* @param off the start offset in buf
* @param len the number of bytes to hash
*/
public abstract void update(ByteBuffer buf, int off, int len);

/**
* Resets this instance to the state it had right after instantiation. The
* seed remains unchanged.
Expand Down
16 changes: 16 additions & 0 deletions src/java/net/jpountz/xxhash/StreamingXXHash32JNI.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
package net.jpountz.xxhash;

import java.nio.ByteBuffer;

/*
* Copyright 2020 Adrien Grand and the lz4-java contributors.
*
Expand Down Expand Up @@ -69,6 +71,20 @@ public synchronized void update(byte[] bytes, int off, int len) {
XXHashJNI.XXH32_update(state, bytes, off, len);
}

public synchronized void update(ByteBuffer buf, int off, int len) {
checkState();
if (buf.isDirect()) {
checkRange(buf, off, len);
XXHashJNI.XXH32BB_update(state, buf, off, len);
} else if (buf.hasArray()) {
XXHashJNI.XXH32_update(state, buf.array(), off + buf.arrayOffset(), len);
} else {
// XXX: What to do here?
throw new RuntimeException("unsupported");
}

}

@Override
public synchronized void close() {
if (state != 0) {
Expand Down
10 changes: 10 additions & 0 deletions src/java/net/jpountz/xxhash/StreamingXXHash64.java
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package net.jpountz.xxhash;

import java.util.zip.Checksum;
import java.nio.ByteBuffer;
import java.io.Closeable;

/*
Expand Down Expand Up @@ -71,6 +72,15 @@ interface Factory {
*/
public abstract void update(byte[] buf, int off, int len);

/**
* Updates the value of the hash with buf[off:off+len].
*
* @param buf the input data
* @param off the start offset in buf
* @param len the number of bytes to hash
*/
public abstract void update(ByteBuffer buf, int off, int len);

/**
* Resets this instance to the state it had right after instantiation. The
* seed remains unchanged.
Expand Down
18 changes: 18 additions & 0 deletions src/java/net/jpountz/xxhash/StreamingXXHash64JNI.java
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
package net.jpountz.xxhash;

import static net.jpountz.util.ByteBufferUtils.checkRange;
import static net.jpountz.util.SafeUtils.checkRange;
import java.nio.ByteBuffer;

/*
* Copyright 2020 Linnaea Von Lavia and the lz4-java contributors.
*
Expand Down Expand Up @@ -70,6 +74,20 @@ public synchronized void update(byte[] bytes, int off, int len) {
XXHashJNI.XXH64_update(state, bytes, off, len);
}

@Override
public synchronized void update(ByteBuffer buf, int off, int len) {
checkState();
if (buf.isDirect()) {
checkRange(buf, off, len);
XXHashJNI.XXH64BB_update(state, buf, off, len);
} else if (buf.hasArray()) {
XXHashJNI.XXH64_update(state, buf.array(), off + buf.arrayOffset(), len);
} else {
// XXX: What to do here?
throw new RuntimeException("unsupported");
}
}

@Override
public synchronized void close() {
if (state != 0) {
Expand Down
2 changes: 2 additions & 0 deletions src/java/net/jpountz/xxhash/XXHashJNI.java
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,15 @@ enum XXHashJNI {
static native int XXH32BB(ByteBuffer input, int offset, int len, int seed);
static native long XXH32_init(int seed);
static native void XXH32_update(long state, byte[] input, int offset, int len);
static native void XXH32BB_update(long state, ByteBuffer input, int offset, int len);
static native int XXH32_digest(long state);
static native void XXH32_free(long state);

static native long XXH64(byte[] input, int offset, int len, long seed);
static native long XXH64BB(ByteBuffer input, int offset, int len, long seed);
static native long XXH64_init(long seed);
static native void XXH64_update(long state, byte[] input, int offset, int len);
static native void XXH64BB_update(long state, ByteBuffer input, int offset, int len);
static native long XXH64_digest(long state);
static native void XXH64_free(long state);
}
40 changes: 40 additions & 0 deletions src/jni/net_jpountz_xxhash_XXHashJNI.c
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,26 @@ JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32_1update

}

/*
* Class: net_jpountz_xxhash_XXHashJNI
* Method: XXH32BB_update
* Signature: (JLjava/nio/ByteBuffer;II)V
*/
JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH32BB_1update
(JNIEnv *env, jclass cls, jlong state, jobject src, jint off, jint len) {

char* in;
jlong h64;

in = (char*) (*env)->GetDirectBufferAddress(env, src);
if (in == NULL) {
throw_OOM(env);
return;
}

XXH32_update((XXH32_state_t*) state, in + off, len);
}

/*
* Class: net_jpountz_xxhash_XXHashJNI
* Method: XXH32_digest
Expand Down Expand Up @@ -230,6 +250,26 @@ JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64_1update

}

/*
* Class: net_jpountz_xxhash_XXHashJNI
* Method: XXH64BB_update
* Signature: (JLjava/nio/ByteBuffer;II)V
*/
JNIEXPORT void JNICALL Java_net_jpountz_xxhash_XXHashJNI_XXH64BB_1update
(JNIEnv *env, jclass cls, jlong state, jobject src, jint off, jint len) {

char* in;

in = (char*) (*env)->GetDirectBufferAddress(env, src);
if (in == NULL) {
printf("I AM NULLLLL?\n");
throw_OOM(env);
return;
}

XXH64_update((XXH64_state_t*) state, in + off, len);
}

/*
* Class: net_jpountz_xxhash_XXHashJNI
* Method: XXH64_digest
Expand Down

0 comments on commit e5e68ba

Please sign in to comment.