Skip to content
This repository has been archived by the owner on Jun 24, 2024. It is now read-only.

Commit

Permalink
Merge pull request #8 from amorris13/master
Browse files Browse the repository at this point in the history
Upstream changes to archive patcher:
  • Loading branch information
andrewhayden authored Oct 18, 2016
2 parents b6093c5 + a982dda commit 0bd5dd5
Show file tree
Hide file tree
Showing 18 changed files with 655 additions and 111 deletions.
2 changes: 2 additions & 0 deletions .classpath
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
<classpathentry kind="src" path="shared/src/test/java"/>
<classpathentry kind="src" path="generator/src/main/java"/>
<classpathentry kind="src" path="generator/src/test/java"/>
<classpathentry kind="src" path="generator/src/test/resources"/>
<classpathentry kind="src" path="applier/src/main/java"/>
<classpathentry kind="src" path="applier/src/test/java"/>
<classpathentry kind="src" path="applier/src/test/resources"/>
<classpathentry kind="src" path="explainer/src/main/java"/>
<classpathentry kind="src" path="explainer/src/test/java"/>
<classpathentry kind="src" path="integrationtest/src/test/java"/>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -89,14 +89,14 @@ public PatchExplainer(Compressor compressor, DeltaGenerator deltaGenerator) {
*
* @param oldFile the old file
* @param newFile the new file
* @param recommendationModifier optionally, a {@link RecommendationModifier} to use during patch
* @param recommendationModifiers optionally, {@link RecommendationModifier}s to use during patch
* planning. If null, a normal patch is generated.
* @return a list of the explanations for each entry that would be
* @throws IOException if unable to read data
* @throws InterruptedException if any thread interrupts this thread
*/
public List<EntryExplanation> explainPatch(
File oldFile, File newFile, RecommendationModifier recommendationModifier)
File oldFile, File newFile, RecommendationModifier... recommendationModifiers)
throws IOException, InterruptedException {
List<EntryExplanation> result = new ArrayList<>();

Expand All @@ -118,11 +118,12 @@ public List<EntryExplanation> explainPatch(
}

Uncompressor uncompressor = new DeflateUncompressor();
PreDiffExecutor executor =
new PreDiffExecutor.Builder()
.readingOriginalFiles(oldFile, newFile)
.withRecommendationModifier(recommendationModifier)
.build();
PreDiffExecutor.Builder builder =
new PreDiffExecutor.Builder().readingOriginalFiles(oldFile, newFile);
for (RecommendationModifier modifier : recommendationModifiers) {
builder.withRecommendationModifier(modifier);
}
PreDiffExecutor executor = builder.build();
PreDiffPlan plan = executor.prepareForDiffing();
try (TempFileHolder oldTemp = new TempFileHolder();
TempFileHolder newTemp = new TempFileHolder();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ public void testExplainPatch_CompressedBytesIdentical() throws Exception {
save(bytes, oldFile);
save(bytes, newFile);
PatchExplainer explainer = new PatchExplainer(null, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile);

EntryExplanation expected =
new EntryExplanation(
Expand All @@ -178,7 +178,7 @@ public void testExplainPatch_CompressedBytesChanged_UncompressedUnchanged() thro
save(oldBytes, oldFile);
save(newBytes, newFile);
PatchExplainer explainer = new PatchExplainer(null, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile);
// The compressed bytes changed, but the uncompressed bytes are the same. Thus the patch size
// should be zero, because the entries are actually identical in the delta-friendly files.
// Additionally no diffing or compression should be performed.
Expand All @@ -201,7 +201,7 @@ public void testExplainPatch_CompressedBytesChanged_UncompressedChanged() throws
FakeCompressor fakeCompressor =
new FakeCompressor(FakeDeltaGenerator.OUTPUT.getBytes("US-ASCII"));
PatchExplainer explainer = new PatchExplainer(fakeCompressor, fakeDeltaGenerator);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile);
// The compressed bytes changed, and so did the uncompressed bytes. The patch size should be
// non-zero because the entries are not identical in the delta-friendly files.
EntryExplanation expected =
Expand Down Expand Up @@ -250,7 +250,7 @@ public void testExplainPatch_BothEntriesUncompressed_BytesUnchanged() throws Exc
save(oldBytes, oldFile);
save(newBytes, newFile);
PatchExplainer explainer = new PatchExplainer(null, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile);
// The uncompressed bytes are the same. Thus the patch size should be zero, because the entries
// are identical in the delta-friendly files. Additionally no diffing or compression should be
// performed.
Expand All @@ -273,7 +273,7 @@ public void testExplainPatch_BothEntriesUncompressed_BytesChanged() throws Excep
FakeCompressor fakeCompressor =
new FakeCompressor(FakeDeltaGenerator.OUTPUT.getBytes("US-ASCII"));
PatchExplainer explainer = new PatchExplainer(fakeCompressor, fakeDeltaGenerator);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile);
// The uncompressed bytes are not the same. Thus the patch size should be non-zero.
EntryExplanation expected =
new EntryExplanation(
Expand All @@ -297,7 +297,7 @@ public void testExplainPatch_CompressedChangedToUncompressed() throws Exception
FakeCompressor fakeCompressor =
new FakeCompressor(FakeDeltaGenerator.OUTPUT.getBytes("US-ASCII"));
PatchExplainer explainer = new PatchExplainer(fakeCompressor, fakeDeltaGenerator);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile);
EntryExplanation expected =
new EntryExplanation(
path(ENTRY_A1_STORED),
Expand All @@ -320,7 +320,7 @@ public void testExplainPatch_UncompressedChangedToCompressed() throws Exception
FakeCompressor fakeCompressor =
new FakeCompressor(FakeDeltaGenerator.OUTPUT.getBytes("US-ASCII"));
PatchExplainer explainer = new PatchExplainer(fakeCompressor, fakeDeltaGenerator);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile);
EntryExplanation expected =
new EntryExplanation(
path(ENTRY_A1_LEVEL_6),
Expand Down Expand Up @@ -354,7 +354,7 @@ public void testExplainPatch_Unsuitable() throws Exception {
FakeCompressor fakeCompressor =
new FakeCompressor(FakeDeltaGenerator.OUTPUT.getBytes("US-ASCII"));
PatchExplainer explainer = new PatchExplainer(fakeCompressor, fakeDeltaGenerator);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile);
EntryExplanation expected =
new EntryExplanation(
path(ENTRY_A1_LEVEL_6),
Expand All @@ -373,7 +373,7 @@ public void testExplainPatch_NewFile() throws Exception {
FakeCompressor fakeCompressor =
new FakeCompressor(ENTRY_B_LEVEL_6.getCompressedBinaryContent());
PatchExplainer explainer = new PatchExplainer(fakeCompressor, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile, null);
List<EntryExplanation> explanations = explainer.explainPatch(oldFile, newFile);
EntryExplanation expected =
new EntryExplanation(
path(ENTRY_B_LEVEL_6),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
// Copyright 2016 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.google.archivepatcher.generator;

import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;

/**
* Limits the size of the delta-friendly old blob, which is an implicit limitation on the amount of
* temp space required to apply a patch.
*
* <p>This class implements the following algorithm:
*
* <ol>
* <li>Check the size of the old archive and subtract it from the maximum size, this is the number
* of bytes that can be used to uncompress entries in the delta-friendly old file.
* <li>Identify all of the {@link QualifiedRecommendation}s that have {@link
* Recommendation#uncompressOldEntry} set to <code>true</code>. These identify all the entries
* that would be uncompressed in the delta-friendly old file.
* <li>Sort those {@link QualifiedRecommendation}s in order of decreasing uncompressed size.
* <li>Iterate over the list in order. For each entry, calculate the difference between the
* uncompressed size and the compressed size; this is the number of bytes that would be
* consumed to transform the data from compressed to uncompressed in the delta-friendly old
* file. If the number of bytes that would be consumed is less than the number of bytes
* remaining before hitting the cap, retain it; else, discard it.
* <li>Return the resulting list of the retained entries. Note that the order of this list may not
* be the same as the input order (i.e., it has been sorted in order of decreasing compressed
* size).
* </ol>
*/
public class DeltaFriendlyOldBlobSizeLimiter implements RecommendationModifier {

/** The maximum size of the delta-friendly old blob. */
private final long maxSizeBytes;

private static final Comparator<QualifiedRecommendation> COMPARATOR =
new UncompressedOldEntrySizeComparator();

/**
* Create a new limiter that will restrict the total size of the delta-friendly old blob.
*
* @param maxSizeBytes the maximum size of the delta-friendly old blob
*/
public DeltaFriendlyOldBlobSizeLimiter(long maxSizeBytes) {
if (maxSizeBytes < 0) {
throw new IllegalArgumentException("maxSizeBytes must be non-negative: " + maxSizeBytes);
}
this.maxSizeBytes = maxSizeBytes;
}

@Override
public List<QualifiedRecommendation> getModifiedRecommendations(
File oldFile, File newFile, List<QualifiedRecommendation> originalRecommendations) {

List<QualifiedRecommendation> sorted = sortRecommendations(originalRecommendations);

List<QualifiedRecommendation> result = new ArrayList<>(sorted.size());
long bytesRemaining = maxSizeBytes - oldFile.length();
for (QualifiedRecommendation originalRecommendation : sorted) {
if (!originalRecommendation.getRecommendation().uncompressOldEntry) {
// Keep the original recommendation, no need to track size since it won't be uncompressed.
result.add(originalRecommendation);
} else {
long extraBytesConsumed =
originalRecommendation.getOldEntry().getUncompressedSize()
- originalRecommendation.getOldEntry().getCompressedSize();
if (bytesRemaining - extraBytesConsumed >= 0) {
// Keep the original recommendation, but also subtract from the remaining space.
result.add(originalRecommendation);
bytesRemaining -= extraBytesConsumed;
} else {
// Update the recommendation to prevent uncompressing this tuple.
result.add(
new QualifiedRecommendation(
originalRecommendation.getOldEntry(),
originalRecommendation.getNewEntry(),
Recommendation.UNCOMPRESS_NEITHER,
RecommendationReason.RESOURCE_CONSTRAINED));
}
}
}
return result;
}

private static List<QualifiedRecommendation> sortRecommendations(
List<QualifiedRecommendation> originalRecommendations) {
List<QualifiedRecommendation> sorted =
new ArrayList<QualifiedRecommendation>(originalRecommendations);
Collections.sort(sorted, COMPARATOR);
Collections.reverse(sorted);
return sorted;
}

/** Helper class implementing the sort order described in the class documentation. */
private static class UncompressedOldEntrySizeComparator
implements Comparator<QualifiedRecommendation> {
@Override
public int compare(QualifiedRecommendation qr1, QualifiedRecommendation qr2) {
return Long.compare(
qr1.getOldEntry().getUncompressedSize(), qr2.getOldEntry().getUncompressedSize());
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,33 +20,33 @@
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;

/**
* Generates file-by-file patches.
*/
public class FileByFileV1DeltaGenerator implements DeltaGenerator {

/** Optional modifier for planning and patch generation. */
private final RecommendationModifier recommendationModifier;

/**
* Constructs a new generator for File-by-File v1 patches, using the default configuration. This
* is equivalent to calling {@link #FileByFileV1DeltaGenerator(RecommendationModifier)} with a
* <code>null</code> {@link RecommendationModifier} argument.
*/
public FileByFileV1DeltaGenerator() {
this(null);
}
/** Optional modifiers for planning and patch generation. */
private final List<RecommendationModifier> recommendationModifiers;

/**
* Constructs a new generator for File-by-File v1 patches, using the specified configuration.
*
* @param recommendationModifier optionally, a {@link RecommendationModifier} to use for modifying
* the planning phase of patch generation. This can be used to, e.g., limit the total amount
* of recompression that a patch applier needs to do.
* @param recommendationModifiers optionally, {@link RecommendationModifier}s to use for modifying
* the planning phase of patch generation. These can be used to, e.g., limit the total amount
* of recompression that a patch applier needs to do. Modifiers are applied in the order they
* are specified.
*/
public FileByFileV1DeltaGenerator(RecommendationModifier recommendationModifier) {
this.recommendationModifier = recommendationModifier;
public FileByFileV1DeltaGenerator(RecommendationModifier... recommendationModifiers) {
if (recommendationModifiers != null) {
this.recommendationModifiers =
Collections.unmodifiableList(Arrays.asList(recommendationModifiers));
} else {
this.recommendationModifiers = Collections.emptyList();
}
}

/**
Expand All @@ -70,12 +70,14 @@ public void generateDelta(File oldFile, File newFile, OutputStream patchOut)
TempFileHolder deltaFile = new TempFileHolder();
FileOutputStream deltaFileOut = new FileOutputStream(deltaFile.file);
BufferedOutputStream bufferedDeltaOut = new BufferedOutputStream(deltaFileOut)) {
PreDiffExecutor executor =
PreDiffExecutor.Builder builder =
new PreDiffExecutor.Builder()
.readingOriginalFiles(oldFile, newFile)
.writingDeltaFriendlyFiles(deltaFriendlyOldFile.file, deltaFriendlyNewFile.file)
.withRecommendationModifier(recommendationModifier)
.build();
.writingDeltaFriendlyFiles(deltaFriendlyOldFile.file, deltaFriendlyNewFile.file);
for (RecommendationModifier modifier : recommendationModifiers) {
builder.withRecommendationModifier(modifier);
}
PreDiffExecutor executor = builder.build();
PreDiffPlan preDiffPlan = executor.prepareForDiffing();
DeltaGenerator deltaGenerator = getDeltaGenerator();
deltaGenerator.generateDelta(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
Expand All @@ -38,7 +39,8 @@ public static final class Builder {
private File originalNewFile;
private File deltaFriendlyOldFile;
private File deltaFriendlyNewFile;
private RecommendationModifier recommendationModifier;
private List<RecommendationModifier> recommendationModifiers =
new ArrayList<RecommendationModifier>();

/**
* Sets the original, read-only input files to the patch generation process. This has to be
Expand Down Expand Up @@ -75,14 +77,17 @@ public Builder writingDeltaFriendlyFiles(File deltaFriendlyOldFile, File deltaFr
}

/**
* Sets an optional {@link RecommendationModifier} to be used during the generation of the
* Appends an optional {@link RecommendationModifier} to be used during the generation of the
* {@link PreDiffPlan} and/or delta-friendly blobs.
*
* @param recommendationModifier the modifier to set
* @return this builder
*/
public Builder withRecommendationModifier(RecommendationModifier recommendationModifier) {
this.recommendationModifier = recommendationModifier;
if (recommendationModifier == null) {
throw new IllegalArgumentException("recommendationModifier cannot be null");
}
this.recommendationModifiers.add(recommendationModifier);
return this;
}

Expand All @@ -101,7 +106,7 @@ public PreDiffExecutor build() {
originalNewFile,
deltaFriendlyOldFile,
deltaFriendlyNewFile,
recommendationModifier);
recommendationModifiers);
}
}

Expand All @@ -123,21 +128,23 @@ public PreDiffExecutor build() {
*/
private final File deltaFriendlyNewFile;

/** Optional {@link RecommendationModifier} to be used for modifying the patch to be generated. */
private final RecommendationModifier recommendationModifier;
/**
* Optional {@link RecommendationModifier}s to be used for modifying the patch to be generated.
*/
private final List<RecommendationModifier> recommendationModifiers;

/** Constructs a new PreDiffExecutor to work with the specified configuration. */
private PreDiffExecutor(
File originalOldFile,
File originalNewFile,
File deltaFriendlyOldFile,
File deltaFriendlyNewFile,
RecommendationModifier recommendationModifier) {
List<RecommendationModifier> recommendationModifiers) {
this.originalOldFile = originalOldFile;
this.originalNewFile = originalNewFile;
this.deltaFriendlyOldFile = deltaFriendlyOldFile;
this.deltaFriendlyNewFile = deltaFriendlyNewFile;
this.recommendationModifier = recommendationModifier;
this.recommendationModifiers = recommendationModifiers;
}

/**
Expand Down Expand Up @@ -220,7 +227,7 @@ private PreDiffPlan generatePreDiffPlan() throws IOException {
originalNewFile,
originalNewArchiveZipEntriesByPath,
originalNewArchiveJreDeflateParametersByPath,
recommendationModifier);
recommendationModifiers.toArray(new RecommendationModifier[] {}));
return preDiffPlanner.generatePreDiffPlan();
}
}
Loading

0 comments on commit 0bd5dd5

Please sign in to comment.