Merge branch 'fix-gzip-compressor-instead-of-test' into next

build.gradle

@@ -184,6 +184,7 @@ test {
         jvmArgs '--add-opens=java.base/java.lang=ALL-UNNAMED'
         jvmArgs '--add-opens=java.base/java.util=ALL-UNNAMED'
         jvmArgs '--add-opens=java.base/java.io=ALL-UNNAMED'
+        jvmArgs '--add-opens=java.base/java.util.zip=ALL-UNNAMED'
     }
     minHeapSize = "128m"
     maxHeapSize = "512m"

src/freenet/support/compress/GzipCompressor.java

@@ -5,12 +5,16 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
+import java.text.MessageFormat;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.GZIPOutputStream;
 
 import freenet.support.Logger;
 import freenet.support.api.Bucket;
 import freenet.support.api.BucketFactory;
+import freenet.support.api.RandomAccessBucket;
+import freenet.support.api.RandomAccessBuffer;
+import freenet.support.io.ArrayBucket;
 import freenet.support.io.Closer;
 import freenet.support.io.CountedOutputStream;
 
@@ -19,13 +23,15 @@ public class GzipCompressor extends AbstractCompressor {
 	@Override
 	public Bucket compress(Bucket data, BucketFactory bf, long maxReadLength, long maxWriteLength)
 			throws IOException, CompressionOutputSizeException {
-		Bucket output = bf.makeBucket(maxWriteLength);
+		RandomAccessBucket output = bf.makeBucket(maxWriteLength);
 		InputStream is = null;
 		OutputStream os = null;
 		try {
 			is = data.getInputStream();
 			os = output.getOutputStream();
-			compress(is, os, maxReadLength, maxWriteLength);
+			// force OS byte to 0 regardless of Java version (java 16 changed to setting 255 which would break hashes)
+			SingleOffsetReplacingOutputStream osByteFixingOs = new SingleOffsetReplacingOutputStream(os, 9, 0);
+			compress(is, osByteFixingOs, maxReadLength, maxWriteLength);
 			// It is essential that the close()'s throw if there is any problem.
 			is.close(); is = null;
 			os.close(); os = null;

src/freenet/support/compress/SingleOffsetReplacingOutputStream.java

@@ -0,0 +1,53 @@
+package freenet.support.compress;
+
+import java.io.FilterOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * A {@link FilterOutputStream} that replaces a single byte at a specific
+ * offset when being written to. It can be used to change the operating system
+ * byte in the header of a {@link GZIPOutputStream}.
+ * <p>
+ * This class is not safe for usage from multiple threads.
+ */
+public class SingleOffsetReplacingOutputStream extends FilterOutputStream {
+
+	private final int replacementOffset;
+	private final int replacementValue;
+	private int currentOffset = 0;
+
+	public SingleOffsetReplacingOutputStream(OutputStream outputStream, int replacementOffset, int replacementValue) {
+		super(outputStream);
+		this.replacementOffset = replacementOffset;
+		this.replacementValue = replacementValue;
+	}
+
+	@Override
+	public void write(int b) throws IOException {
+		if (currentOffset == replacementOffset) {
+			out.write(replacementValue);
+		} else {
+			out.write(b);
+		}
+		currentOffset++;
+	}
+
+	@Override
+	public void write(byte[] buffer, int offset, int length) throws IOException {
+		if (offsetToReplaceIsInBufferBeingWritten(length)) {
+			out.write(buffer, offset, replacementOffset - currentOffset);
+			out.write(replacementValue);
+			out.write(buffer, offset + (replacementOffset - currentOffset) + 1, length - (replacementOffset - currentOffset) - 1);
+		} else {
+			out.write(buffer, offset, length);
+		}
+		currentOffset += length;
+	}
+
+	private boolean offsetToReplaceIsInBufferBeingWritten(int length) {
+		return (currentOffset <= replacementOffset) && ((currentOffset + length) > replacementOffset);
+	}
+
+}

test/freenet/support/compress/GzipCompressorTest.java

@@ -161,12 +161,6 @@ private byte[] doCompress(byte[] uncompressedData) throws IOException {
 		Bucket outBucket = GZIP.compress(inBucket, factory, 32768, 32768);
 		byte[] outBuffer = BucketTools.toByteArray(outBucket);
 
-		// Newer JVM versions have different OS (Operating System) GZIP member header value
-		// https://www.rfc-editor.org/rfc/rfc1952#section-2.3.1
-		// https://bugs.openjdk.org/browse/JDK-8244706
-		// Set OS byte to zero to produce stable results in this test
-		outBuffer[9] = 0;
-
 		return outBuffer;
 	}
 }

test/freenet/support/compress/SingleOffsetReplacingOutputStreamTest.java

@@ -0,0 +1,117 @@
+package freenet.support.compress;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Random;
+
+import org.junit.Test;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.equalTo;
+
+public class SingleOffsetReplacingOutputStreamTest {
+
+	@Test
+	public void allBytesBeforeTheOffsetAreUnchanged() throws IOException {
+		filterOutputStreamWithOffsetAndReplacementWritingByteArrays(32768, 1, 65536);
+	}
+
+	@Test
+	public void byteAtOffsetZeroCanBeReplaced() throws IOException {
+		filterOutputStreamWithOffsetAndReplacementWritingByteArrays(10, 1, 0);
+	}
+
+	@Test
+	public void byteAtOffsetOneCanBeReplaced() throws IOException {
+		filterOutputStreamWithOffsetAndReplacementWritingByteArrays(10, 1, 1);
+	}
+
+	@Test
+	public void byteAtEndOfBufferCanBeReplaced() throws IOException {
+		filterOutputStreamWithOffsetAndReplacementWritingByteArrays(10, 1, 9);
+	}
+
+	@Test
+	public void byteAtBeginningOfSecondBufferCanBeReplaced() throws IOException {
+		filterOutputStreamWithOffsetAndReplacementWritingByteArrays(10, 2, 10);
+	}
+
+	@Test
+	public void byteInTheMiddleOfSecondBufferCanBeReplaced() throws IOException {
+		filterOutputStreamWithOffsetAndReplacementWritingByteArrays(10, 3, 15);
+	}
+
+	@Test
+	public void byteAtOffsetZeroCanBeReplacedWhenWritingSingleBytes() throws IOException {
+		filterOutputStreamWithOffsetAndReplacementWritingSingleBytes(4096, 16, 0);
+	}
+
+	@Test
+	public void byteInMiddleOfStreamBeReplacedWhenWritingSingleBytes() throws IOException {
+		filterOutputStreamWithOffsetAndReplacementWritingSingleBytes(8192, 8, 12345);
+	}
+
+	@Test
+	public void byteAtEndOfStreamCanBeReplacedWhenWritingSingleBytes() throws IOException {
+		filterOutputStreamWithOffsetAndReplacementWritingSingleBytes(16384, 4, 65535);
+	}
+
+	@Test
+	public void byteAtStartOfBufferCanBeReplacedWhenWritingHalfBuffers() throws IOException {
+		filterOutputStreamWithOffsetAndReplacementWritingHalfBuffers(4096, 16, 0);
+	}
+
+	@Test
+	public void byteAtMiddleOfBufferCanBeReplacedWhenWritingHalfBuffers() throws IOException {
+		filterOutputStreamWithOffsetAndReplacementWritingHalfBuffers(8192, 8, 12345);
+	}
+
+	@Test
+	public void byteAtEndOfBufferCanBeReplacedWhenWritingHalfBuffers() throws IOException {
+		filterOutputStreamWithOffsetAndReplacementWritingHalfBuffers(16384, 4, 65535);
+	}
+
+	private void filterOutputStreamWithOffsetAndReplacementWritingSingleBytes(int blockSize, int numberOfBlocks, int replacementOffset) throws IOException {
+		filterOutputStreamWithOffsetAndReplacement(blockSize, numberOfBlocks, replacementOffset, (buffer, length, repetitions, outputStream) -> {
+			for (int index = 0; index < length * repetitions; index++) {
+				outputStream.write(buffer[index % length]);
+			}
+		});
+	}
+
+	private void filterOutputStreamWithOffsetAndReplacementWritingByteArrays(int blockSize, int numberOfBlocks, int replacementOffset) throws IOException {
+		filterOutputStreamWithOffsetAndReplacement(blockSize, numberOfBlocks, replacementOffset, (buffer, length, repetitions, outputStream) -> {
+			for (int blockIndex = 0; blockIndex < repetitions; blockIndex++) {
+				outputStream.write(buffer, 0, length);
+			}
+		});
+	}
+
+	private void filterOutputStreamWithOffsetAndReplacementWritingHalfBuffers(int blockSize, int numberOfBlocks, int replacementOffset) throws IOException {
+		filterOutputStreamWithOffsetAndReplacement(blockSize, numberOfBlocks, replacementOffset, ((buffer, length, repetitions, outputStream) -> {
+			for (int blockIndex = 0; blockIndex < repetitions; blockIndex++) {
+				outputStream.write(buffer, 0, length / 2);
+				outputStream.write(buffer, length / 2, length - (length / 2));
+			}
+		}));
+	}
+
+	private void filterOutputStreamWithOffsetAndReplacement(int blockSize, int numberOfBlocks, int replacementOffset, BufferToOutputStreamCopierStrategy bufferToOutputStreamCopierStrategy) throws IOException {
+		byte[] bufferToWrite = new byte[blockSize];
+		new Random().nextBytes(bufferToWrite);
+		ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+		SingleOffsetReplacingOutputStream outputStream = new SingleOffsetReplacingOutputStream(byteArrayOutputStream, replacementOffset, (byte) (bufferToWrite[replacementOffset % blockSize] ^ 0xff));
+		bufferToOutputStreamCopierStrategy.copyBufferToOutput(bufferToWrite, blockSize, numberOfBlocks, outputStream);
+		byte[] writtenBuffer = byteArrayOutputStream.toByteArray();
+		for (int offset = 0; offset < blockSize * numberOfBlocks; offset++) {
+			assertThat("offset " + offset, writtenBuffer[offset], equalTo((byte) (bufferToWrite[offset % blockSize] ^ ((offset == replacementOffset) ? 0xff : 0x00))));
+		}
+	}
+
+	@FunctionalInterface
+	private interface BufferToOutputStreamCopierStrategy {
+		void copyBufferToOutput(byte[] buffer, int length, int repetitions, OutputStream outputStream) throws IOException;
+	}
+
+}