diff --git a/core/build.gradle b/core/build.gradle index cccd38adc9..019cba6bad 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -18,6 +18,7 @@ plugins { id 'java' id 'maven-publish' id 'signing' + id 'io.freefair.aspectj.post-compile-weaving' version '6.4.1' } //publishing { @@ -42,6 +43,8 @@ dependencies { implementation 'com.googlecode.libphonenumber:libphonenumber:8.12.13' implementation 'com.google.flogger:flogger:0.6' implementation 'io.github.classgraph:classgraph:4.8.146' + implementation 'org.aspectj:aspectjrt:1.9.20' + implementation 'org.aspectj:aspectjweaver:1.9.20' testImplementation 'com.google.flogger:flogger-system-backend:0.6' testImplementation group: 'junit', name: 'junit', version: '4.13' testImplementation "com.google.truth:truth:1.0.1" diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/io/ValidationReportDeserializer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/io/ValidationReportDeserializer.java index 3197be8f81..d088be3285 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/io/ValidationReportDeserializer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/io/ValidationReportDeserializer.java @@ -23,16 +23,13 @@ import com.google.gson.JsonElement; import com.google.gson.JsonObject; import java.lang.reflect.Type; -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import org.mobilitydata.gtfsvalidator.model.NoticeReport; import org.mobilitydata.gtfsvalidator.model.ValidationReport; import org.mobilitydata.gtfsvalidator.notice.Notice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.ResolvedNotice; +import org.mobilitydata.gtfsvalidator.performance.MemoryUsage; /** * Used to (de)serialize a JSON validation report. This represents a validation report as a list of @@ -44,6 +41,7 @@ public class ValidationReportDeserializer implements JsonDeserializer memoryUsageRecords = null; if (rootObject.has(SUMMARY_MEMBER_NAME)) { JsonObject summaryObject = rootObject.getAsJsonObject(SUMMARY_MEMBER_NAME); if (summaryObject.has(VALIDATION_TIME_MEMBER_NAME)) { validationTimeSeconds = summaryObject.get(VALIDATION_TIME_MEMBER_NAME).getAsDouble(); } + if (summaryObject.has(MEMORY_USAGE_RECORDS_MEMBER_NAME)) { + JsonArray memoryUsageArray = summaryObject.getAsJsonArray(MEMORY_USAGE_RECORDS_MEMBER_NAME); + memoryUsageRecords = new ArrayList<>(); + for (JsonElement element : memoryUsageArray) { + MemoryUsage memoryUsage = Notice.GSON.fromJson(element, MemoryUsage.class); + memoryUsageRecords.add(memoryUsage); + } + } } JsonArray noticesArray = rootObject.getAsJsonArray(NOTICES_MEMBER_NAME); for (JsonElement childObject : noticesArray) { notices.add(Notice.GSON.fromJson(childObject, NoticeReport.class)); } - return new ValidationReport(notices, validationTimeSeconds); + return new ValidationReport(notices, validationTimeSeconds, memoryUsageRecords); } public static JsonObject serialize( diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/model/ValidationReport.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/model/ValidationReport.java index 5ea4c76a35..c1a3da670e 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/model/ValidationReport.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/model/ValidationReport.java @@ -23,8 +23,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.Collections; +import java.util.List; import java.util.Set; import org.mobilitydata.gtfsvalidator.io.ValidationReportDeserializer; +import org.mobilitydata.gtfsvalidator.performance.MemoryUsage; /** * Used to (de)serialize a {@code NoticeContainer}. This represents a validation report as a list of @@ -42,6 +44,7 @@ public class ValidationReport { .create(); private final Set notices; private final Double validationTimeSeconds; + private List memoryUsageRecords; /** * Public constructor needed for deserialization by {@code ValidationReportDeserializer}. Only @@ -50,7 +53,7 @@ public class ValidationReport { * @param noticeReports set of {@code NoticeReport}s */ public ValidationReport(Set noticeReports) { - this(noticeReports, null); + this(noticeReports, null, null); } /** @@ -60,9 +63,13 @@ public ValidationReport(Set noticeReports) { * @param noticeReports set of {@code NoticeReport}s * @param validationTimeSeconds the time taken to validate the GTFS dataset */ - public ValidationReport(Set noticeReports, Double validationTimeSeconds) { + public ValidationReport( + Set noticeReports, + Double validationTimeSeconds, + List memoryUsageRecords) { this.notices = Collections.unmodifiableSet(noticeReports); this.validationTimeSeconds = validationTimeSeconds; + this.memoryUsageRecords = memoryUsageRecords; } /** @@ -86,6 +93,9 @@ public Double getValidationTimeSeconds() { return validationTimeSeconds; } + public List getMemoryUsageRecords() { + return memoryUsageRecords; + } /** * Determines if two validation reports are equal regardless of the order of the fields in the set * of {@code NoticeReport}. diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryMonitor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryMonitor.java new file mode 100644 index 0000000000..cd05c2a509 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryMonitor.java @@ -0,0 +1,16 @@ +package org.mobilitydata.gtfsvalidator.performance; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotation to monitor memory usage of a method. The annotated method should return a {@link + * MemoryUsage} object. The key is used to group memory usage of different methods. + */ +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +public @interface MemoryMonitor { + String key() default ""; +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryMonitorAspect.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryMonitorAspect.java new file mode 100644 index 0000000000..3c2fdef403 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryMonitorAspect.java @@ -0,0 +1,39 @@ +package org.mobilitydata.gtfsvalidator.performance; + +import org.apache.commons.lang3.StringUtils; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.reflect.MethodSignature; + +/** Aspect to monitor memory usage of a method. */ +@Aspect +public class MemoryMonitorAspect { + + @Around("execution(@org.mobilitydata.gtfsvalidator.performance.MemoryMonitor * *(..))") + public Object monitorMemoryUsage(ProceedingJoinPoint joinPoint) throws Throwable { + String key = extractKey(joinPoint); + MemoryUsage before = MemoryUsageRegister.getInstance().getMemoryUsageSnapshot(key, null); + try { + Object result = joinPoint.proceed(); + return result; + } finally { + MemoryUsage after = MemoryUsageRegister.getInstance().getMemoryUsageSnapshot(key, before); + MemoryUsageRegister.getInstance().registerMemoryUsage(after); + } + } + + /** + * Extracts the key from the method signature or the annotation. + * + * @param joinPoint the join point + * @return the key either from the annotation or the method signature. + */ + private String extractKey(ProceedingJoinPoint joinPoint) { + var method = ((MethodSignature) joinPoint.getSignature()).getMethod(); + var memoryMonitor = method.getAnnotation(MemoryMonitor.class); + return memoryMonitor != null && StringUtils.isNotBlank(memoryMonitor.key()) + ? memoryMonitor.key() + : method.getDeclaringClass().getCanonicalName() + "." + method.getName(); + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryUsage.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryUsage.java new file mode 100644 index 0000000000..f81321afb3 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryUsage.java @@ -0,0 +1,176 @@ +package org.mobilitydata.gtfsvalidator.performance; + +import java.text.DecimalFormat; +import org.apache.commons.lang3.StringUtils; + +/** Represents memory usage information. */ +public class MemoryUsage { + private static final DecimalFormat TWO_DECIMAL_FORMAT = new DecimalFormat("0.00"); + + private String key; + private long totalMemory; + private long freeMemory; + private long maxMemory; + private Long diffMemory; + + public MemoryUsage() {} + + public MemoryUsage( + String key, long totalMemory, long freeMemory, long maxMemory, Long diffMemory) { + this.key = key; + this.totalMemory = totalMemory; + this.freeMemory = freeMemory; + this.maxMemory = maxMemory; + this.diffMemory = diffMemory; + } + + /** + * Converts bytes to human-readable memory. + * + * @param bytes + * @return human-readable memory, e.g., "1.23 GiB" + */ + public static String convertToHumanReadableMemory(Long bytes) { + if (bytes == null) { + return "N/A"; + } + long size = Math.abs(bytes); + if (size < 1024) { + return bytes + " bytes"; + } + if (size < 1048576) { + return TWO_DECIMAL_FORMAT.format(Math.copySign(size / 1024.0, bytes)) + " KiB"; + } + if (size < 1073741824) { + return TWO_DECIMAL_FORMAT.format(Math.copySign(size / 1048576.0, bytes)) + " MiB"; + } + if (size < 1099511627776L) { + return TWO_DECIMAL_FORMAT.format(Math.copySign(size / 1073741824.0, bytes)) + " GiB"; + } + return TWO_DECIMAL_FORMAT.format(Math.copySign(size / 1099511627776L, bytes)) + " TiB"; + } + + /** + * The memory used is computed as the difference between the total memory and the free memory. + * + * @return the memory used. + */ + public long usedMemory() { + return totalMemory - freeMemory; + } + + /** + * Returns a human-readable string representation of the memory usage. + * + * @return a human-readable string representation of the memory usage. + */ + public String humanReadablePrint() { + StringBuffer result = new StringBuffer(); + result.append("Memory usage registered"); + if (StringUtils.isNotBlank(key)) { + result.append(" for key: ").append(key); + } else { + result.append(":"); + } + result.append(" Max: ").append(convertToHumanReadableMemory(maxMemory)); + result.append(" Total: ").append(convertToHumanReadableMemory(totalMemory)); + result.append(" Free: ").append(convertToHumanReadableMemory(freeMemory)); + result.append(" Used: ").append(convertToHumanReadableMemory(usedMemory())); + result.append(" Diff: ").append(convertToHumanReadableMemory(diffMemory)); + return result.toString(); + } + + public String getKey() { + return key; + } + + public void setKey(String key) { + this.key = key; + } + + public long getTotalMemory() { + return totalMemory; + } + + public void setTotalMemory(long totalMemory) { + this.totalMemory = totalMemory; + } + + public long getFreeMemory() { + return freeMemory; + } + + public void setFreeMemory(long freeMemory) { + this.freeMemory = freeMemory; + } + + public long getMaxMemory() { + return maxMemory; + } + + public void setMaxMemory(long maxMemory) { + this.maxMemory = maxMemory; + } + + public Long getDiffMemory() { + return diffMemory; + } + + public void setDiffMemory(Long diffMemory) { + this.diffMemory = diffMemory; + } + + @Override + public String toString() { + return "MemoryUsage{" + + "key=" + + key + + ", " + + "totalMemory=" + + totalMemory + + ", " + + "freeMemory=" + + freeMemory + + ", " + + "maxMemory=" + + maxMemory + + ", " + + "diffMemory=" + + diffMemory + + "}"; + } + + @Override + public boolean equals(Object o) { + if (o == this) { + return true; + } + if (o instanceof MemoryUsage) { + MemoryUsage that = (MemoryUsage) o; + return this.key.equals(that.getKey()) + && this.totalMemory == that.getTotalMemory() + && this.freeMemory == that.getFreeMemory() + && this.maxMemory == that.getMaxMemory() + && (this.diffMemory == null + ? that.getDiffMemory() == null + : this.getDiffMemory().equals(that.getDiffMemory())); + } + return false; + } + + @Override + public int hashCode() { + int h$ = 1; + h$ *= 1000003; + h$ ^= key.hashCode(); + h$ *= 1000003; + h$ ^= (int) ((totalMemory >>> 32) ^ totalMemory); + h$ *= 1000003; + h$ ^= (int) ((freeMemory >>> 32) ^ freeMemory); + h$ *= 1000003; + h$ ^= (int) ((maxMemory >>> 32) ^ maxMemory); + h$ *= 1000003; + h$ ^= (diffMemory == null) ? 0 : diffMemory.hashCode(); + return h$; + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryUsageRegister.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryUsageRegister.java new file mode 100644 index 0000000000..21dcb2d658 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/performance/MemoryUsageRegister.java @@ -0,0 +1,90 @@ +package org.mobilitydata.gtfsvalidator.performance; + +import com.google.common.flogger.FluentLogger; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** Register for memory usage snapshots. */ +public class MemoryUsageRegister { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private static MemoryUsageRegister instance = new MemoryUsageRegister(); + private final Runtime runtime; + private List registry = new ArrayList<>(); + + private MemoryUsageRegister() { + runtime = Runtime.getRuntime(); + } + + /** @return the singleton instance of the memory usage register. */ + public static MemoryUsageRegister getInstance() { + return instance; + } + + /** + * Returns the memory usage registry. + * + * @return the memory usage registry unmodifiable list. + */ + public List getRegistry() { + return Collections.unmodifiableList(registry); + } + + /** + * Returns a memory usage snapshot. + * + * @param key + * @param previous + * @return + */ + public MemoryUsage getMemoryUsageSnapshot(String key, MemoryUsage previous) { + Long memoryDiff = null; + if (previous != null) { + memoryDiff = runtime.freeMemory() - previous.getFreeMemory(); + } + return new MemoryUsage( + key, runtime.totalMemory(), runtime.freeMemory(), runtime.maxMemory(), memoryDiff); + } + + /** + * Registers a memory usage snapshot. + * + * @param key + * @return + */ + public MemoryUsage registerMemoryUsage(String key) { + MemoryUsage memoryUsage = getMemoryUsageSnapshot(key, null); + registerMemoryUsage(memoryUsage); + return memoryUsage; + } + + /** + * Registers a memory usage snapshot. + * + * @param key + * @param previous previous memory usage snapshot used to compute the memory difference between + * two snapshots. + * @return + */ + public MemoryUsage registerMemoryUsage(String key, MemoryUsage previous) { + MemoryUsage memoryUsage = getMemoryUsageSnapshot(key, previous); + registerMemoryUsage(memoryUsage); + return memoryUsage; + } + + /** + * Registers a memory usage snapshot. + * + * @param memoryUsage + */ + public void registerMemoryUsage(MemoryUsage memoryUsage) { + registry.add(memoryUsage); + logger.atInfo().log(memoryUsage.humanReadablePrint()); + } + + /** Clears the memory usage registry. */ + public void clearRegistry() { + registry.clear(); + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java index 21dc38e98b..0fcaa738da 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java @@ -36,6 +36,8 @@ import org.mobilitydata.gtfsvalidator.notice.RuntimeExceptionInLoaderError; import org.mobilitydata.gtfsvalidator.notice.ThreadExecutionError; import org.mobilitydata.gtfsvalidator.notice.UnknownFileNotice; +import org.mobilitydata.gtfsvalidator.performance.MemoryMonitor; +import org.mobilitydata.gtfsvalidator.performance.MemoryUsageRegister; import org.mobilitydata.gtfsvalidator.validator.FileValidator; import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; import org.mobilitydata.gtfsvalidator.validator.ValidatorUtil; @@ -95,6 +97,7 @@ public List> getMultiFileValidatorsWithParsingErr } @SuppressWarnings("unchecked") + @MemoryMonitor() public GtfsFeedContainer loadAndValidate( GtfsInput gtfsInput, ValidatorProvider validatorProvider, NoticeContainer noticeContainer) throws InterruptedException { @@ -146,47 +149,85 @@ public GtfsFeedContainer loadAndValidate( tableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer)); } try { - for (Future futureContainer : exec.invokeAll(loaderCallables)) { - try { - TableAndNoticeContainers containers = futureContainer.get(); - tableContainers.add(containers.tableContainer); - noticeContainer.addAll(containers.noticeContainer); - } catch (ExecutionException e) { - // All runtime exceptions should be caught above. - // ExecutionException is not expected to happen. - addThreadExecutionError(e, noticeContainer); - } - } + var beforeLoading = + MemoryUsageRegister.getInstance() + .getMemoryUsageSnapshot("GtfsFeedLoader.loadTables", null); + loadTables(noticeContainer, exec, loaderCallables, tableContainers); + MemoryUsageRegister.getInstance() + .registerMemoryUsage("GtfsFeedLoader.loadTables", beforeLoading); + GtfsFeedContainer feed = new GtfsFeedContainer(tableContainers); - List> validatorCallables = new ArrayList<>(); - // Validators with parser-error dependencies will not be returned here, but instead added to - // the skippedValidators list. - for (FileValidator validator : - validatorProvider.createMultiFileValidators( - feed, multiFileValidatorsWithParsingErrors::add)) { - validatorCallables.add( - () -> { - NoticeContainer validatorNotices = new NoticeContainer(); - ValidatorUtil.safeValidate( - validator::validate, validator.getClass(), validatorNotices); - return validatorNotices; - }); - } - for (Future futureContainer : exec.invokeAll(validatorCallables)) { - try { - noticeContainer.addAll(futureContainer.get()); - } catch (ExecutionException e) { - // All runtime exceptions should be caught above. - // ExecutionException is not expected to happen. - addThreadExecutionError(e, noticeContainer); - } - } + var beforeMultiFileValidators = + MemoryUsageRegister.getInstance() + .getMemoryUsageSnapshot("GtfsFeedLoader.executeMultiFileValidators", null); + executeMultiFileValidators(validatorProvider, noticeContainer, feed, exec); + MemoryUsageRegister.getInstance() + .registerMemoryUsage( + "GtfsFeedLoader.executeMultiFileValidators", beforeMultiFileValidators); + return feed; } finally { exec.shutdown(); } } + private static void loadTables( + NoticeContainer noticeContainer, + ExecutorService exec, + List> loaderCallables, + ArrayList> tableContainers) + throws InterruptedException { + for (Future futureContainer : exec.invokeAll(loaderCallables)) { + try { + TableAndNoticeContainers containers = futureContainer.get(); + tableContainers.add(containers.tableContainer); + noticeContainer.addAll(containers.noticeContainer); + } catch (ExecutionException e) { + // All runtime exceptions should be caught above. + // ExecutionException is not expected to happen. + addThreadExecutionError(e, noticeContainer); + } + } + } + + private void executeMultiFileValidators( + ValidatorProvider validatorProvider, + NoticeContainer noticeContainer, + GtfsFeedContainer feed, + ExecutorService exec) + throws InterruptedException { + List> validatorCallables = new ArrayList<>(); + // Validators with parser-error dependencies will not be returned here, but instead added to + // the skippedValidators list. + for (FileValidator validator : + validatorProvider.createMultiFileValidators( + feed, multiFileValidatorsWithParsingErrors::add)) { + validatorCallables.add( + () -> { + NoticeContainer validatorNotices = new NoticeContainer(); + ValidatorUtil.safeValidate(validator::validate, validator.getClass(), validatorNotices); + return validatorNotices; + }); + } + collectMultiFileValidationNotices(noticeContainer, exec, validatorCallables); + } + + private static void collectMultiFileValidationNotices( + NoticeContainer noticeContainer, + ExecutorService exec, + List> validatorCallables) + throws InterruptedException { + for (Future futureContainer : exec.invokeAll(validatorCallables)) { + try { + noticeContainer.addAll(futureContainer.get()); + } catch (ExecutionException e) { + // All runtime exceptions should be caught above. + // ExecutionException is not expected to happen. + addThreadExecutionError(e, noticeContainer); + } + } + } + /** Adds a ThreadExecutionError to the notice container. */ private static void addThreadExecutionError( ExecutionException e, NoticeContainer noticeContainer) { diff --git a/docs/ACCEPTANCE_TESTS.md b/docs/ACCEPTANCE_TESTS.md index b17d036462..263b697116 100644 --- a/docs/ACCEPTANCE_TESTS.md +++ b/docs/ACCEPTANCE_TESTS.md @@ -104,6 +104,33 @@ We follow this process: +## Performance metrics within the acceptance reports + +There are two main metrics added to the acceptance report comment at the PR level, _Validation Time_ and _Memory Consumption_. +The performance metrics are **not a blocker** as performance might vary due to external factors including GitHub infrastructure performance. +However, large jumps in performance values should be investigated before approving a PR. + +### Validation Time +The validation time consists in general metrics like average, median, standard deviation, minimums and maximums. +This metrics can be affected by addition of new validators than introduce a penalty in processing time. + +### Memory Consumption +There are two main patterns on how to take a memory usage snapshot: + +- MemoryMonitor annotation: This annotation persists the memory usage in the target method. As a limitation, for methods that have concurrent thread executions, the annotation persists in multiple snapshots. This cannot be very clear when analyzing memory usage. +- MemoryUsageRegister: using the registry directly give you more flexibility than the annotation and can be used in cases where MemoryMonitor produces multiple entries on concurrent executed methods. + +The memory consumption section contains three tables. +- The first, list the first 25 datasets that the difference increased memory comparing with the main branch. +- The second, list the first 25 datasets that the difference decreased memory comparing with the main branch. +- The third, list(not always visible) the first 25 datasets that were not available for comparison as the main branch didn't contain the memory usage information. + +Memory usage is collected in critical points and persists in the JSON report. The added snapshot points are: +- _GtfsFeedLoader.loadTables_: This is taken after the validator loads all files. +- _GtfsFeedLoader.executeMultiFileValidators_: This is taken after the validator executed all multi-file validators +- _org.mobilitydata.gtfsvalidator.table.GtfsFeedLoader.loadAndValidate_: This is taken for the complete load and validation method. +- _ValidationRunner.run_: This is taken for the complete run of the validator, excluding report generation + ## Instructions to run the pipeline 1. Provide code changes by creating a new PR on the [GitHub repository](https://github.com/MobilityData/gtfs-validator); diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/JsonReportSummary.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/JsonReportSummary.java index 4480cc544f..1f75f1d57b 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/JsonReportSummary.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/JsonReportSummary.java @@ -6,6 +6,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import org.mobilitydata.gtfsvalidator.performance.MemoryUsage; import org.mobilitydata.gtfsvalidator.report.model.AgencyMetadata; import org.mobilitydata.gtfsvalidator.report.model.FeatureMetadata; import org.mobilitydata.gtfsvalidator.report.model.FeedMetadata; @@ -35,6 +36,7 @@ public class JsonReportSummary { private List agencies; private Set files; private Double validationTimeSeconds; + public List memoryUsageRecords; @SerializedName("counts") private JsonReportCounts jsonReportCounts; @@ -68,6 +70,7 @@ public JsonReportSummary( if (feedMetadata.feedInfo != null) { this.feedInfo = new JsonReportFeedInfo(feedMetadata.feedInfo); this.validationTimeSeconds = feedMetadata.validationTimeSeconds; + this.memoryUsageRecords = feedMetadata.memoryUsageRecords; } else { logger.atSevere().log( "No feed info for feed " diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java index 38f9d78f8b..6c8918d0e3 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java @@ -8,6 +8,8 @@ import java.time.format.DateTimeFormatter; import java.util.*; import java.util.function.Function; +import org.mobilitydata.gtfsvalidator.performance.MemoryUsage; +import org.mobilitydata.gtfsvalidator.performance.MemoryUsageRegister; import org.mobilitydata.gtfsvalidator.table.*; import org.mobilitydata.gtfsvalidator.util.CalendarUtil; import org.mobilitydata.gtfsvalidator.util.ServicePeriod; @@ -54,6 +56,7 @@ public class FeedMetadata { public double validationTimeSeconds; + public List memoryUsageRecords; // List of features that only require checking the presence of one record in the file. private final List> FILE_BASED_FEATURES = List.of( @@ -116,6 +119,7 @@ public static FeedMetadata from(GtfsFeedContainer feedContainer, ImmutableSet + */ +public class BoundedPriorityQueue extends PriorityQueue { + private final int maxCapacity; + + public BoundedPriorityQueue(int maxCapacity) { + super(); + if (maxCapacity <= 0) { + throw new IllegalArgumentException("Max capacity must be greater than zero"); + } + this.maxCapacity = maxCapacity; + } + + public BoundedPriorityQueue(int maxCapacity, int initialCapacity, Comparator comparator) { + super(initialCapacity, comparator); + if (maxCapacity <= 0) { + throw new IllegalArgumentException("Max capacity must be greater than zero"); + } + this.maxCapacity = maxCapacity; + } + + @Override + public boolean offer(E e) { + if (size() >= maxCapacity) { + E head = peek(); + if (head != null && compare(e, head) > 0) { + poll(); + } else { + return false; + } + } + return super.offer(e); + } + + @SuppressWarnings("unchecked") + private int compare(E a, E b) { + if (comparator() != null) { + return comparator().compare(a, b); + } + return ((Comparable) a).compareTo(b); + } + + public int getMaxCapacity() { + return maxCapacity; + } +} diff --git a/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/DatasetMemoryUsage.java b/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/DatasetMemoryUsage.java new file mode 100644 index 0000000000..01accf7864 --- /dev/null +++ b/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/DatasetMemoryUsage.java @@ -0,0 +1,91 @@ +package org.mobilitydata.gtfsvalidator.outputcomparator.io; + +import com.google.common.flogger.FluentLogger; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.mobilitydata.gtfsvalidator.performance.MemoryUsage; + +/** + * Represents memory usage information for a dataset. This class contains the information associated + * with the memory usage of a dataset when running the validation process. + */ +public class DatasetMemoryUsage { + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + private String datasetId; + private List referenceMemoryUsage; + private List latestMemoryUsage; + private Map referenceUsedMemoryByKey = Collections.unmodifiableMap(new HashMap<>()); + private Map latestUsedMemoryByKey = Collections.unmodifiableMap(new HashMap<>()); + + public DatasetMemoryUsage( + String datasetId, + List referenceMemoryUsage, + List latestMemoryUsage) { + this.datasetId = datasetId; + this.referenceMemoryUsage = referenceMemoryUsage; + this.latestMemoryUsage = latestMemoryUsage; + if (referenceMemoryUsage != null) { + this.referenceUsedMemoryByKey = + referenceMemoryUsage.stream() + .collect( + Collectors.toUnmodifiableMap( + MemoryUsage::getKey, + MemoryUsage::usedMemory, + (existing, replacement) -> { + logger.atWarning().log( + "Duplicate key found in referenceMemoryUsage: " + existing); + return existing; + })); + } + if (latestMemoryUsage != null) { + this.latestUsedMemoryByKey = + latestMemoryUsage.stream() + .collect( + Collectors.toUnmodifiableMap( + MemoryUsage::getKey, + MemoryUsage::usedMemory, + (existing, replacement) -> { + logger.atWarning().log( + "Duplicate key found in latestMemoryUsage: " + existing); + return existing; + })); + } + } + + public String getDatasetId() { + return datasetId; + } + + public void setDatasetId(String datasetId) { + this.datasetId = datasetId; + } + + public List getReferenceMemoryUsage() { + return referenceMemoryUsage; + } + + public void setReferenceMemoryUsage(List referenceMemoryUsage) { + this.referenceMemoryUsage = referenceMemoryUsage; + } + + public List getLatestMemoryUsage() { + return latestMemoryUsage; + } + + public void setLatestMemoryUsage(List latestMemoryUsage) { + this.latestMemoryUsage = latestMemoryUsage; + } + + public Map getReferenceUsedMemoryByKey() { + return referenceUsedMemoryByKey; + } + + public Map getLatestUsedMemoryByKey() { + return latestUsedMemoryByKey; + } +} diff --git a/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/LatestReportUsedMemoryComparator.java b/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/LatestReportUsedMemoryComparator.java new file mode 100644 index 0000000000..89ea4c699a --- /dev/null +++ b/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/LatestReportUsedMemoryComparator.java @@ -0,0 +1,43 @@ +package org.mobilitydata.gtfsvalidator.outputcomparator.io; + +import java.util.Comparator; +import org.mobilitydata.gtfsvalidator.performance.MemoryUsage; + +/** + * Comparator to compare two {@link DatasetMemoryUsage} objects based on used memory of the two + * objects, {@link DatasetMemoryUsage#getLatestMemoryUsage}. + */ +public class LatestReportUsedMemoryComparator implements Comparator { + + @Override + public int compare(DatasetMemoryUsage o1, DatasetMemoryUsage o2) { + if (o1 == o2) { + return 0; + } + if (o1 == null || o2 == null) { + return o1 == null ? -1 : 1; + } + if (o1.getLatestMemoryUsage() == null && o2.getLatestMemoryUsage() == null) { + return 0; + } + if (o1.getLatestMemoryUsage() == null || o2.getLatestMemoryUsage() == null) { + return o1.getLatestMemoryUsage() == null ? -1 : 1; + } + long o1MaxMemory = + o1.getLatestMemoryUsage().stream() + .max(Comparator.comparingLong(MemoryUsage::usedMemory)) + .get() + .usedMemory(); + long o2MaxMemory = + o2.getLatestMemoryUsage().stream() + .max(Comparator.comparingLong(MemoryUsage::usedMemory)) + .get() + .usedMemory(); + return Long.compare(o1MaxMemory, o2MaxMemory); + } + + @Override + public Comparator reversed() { + return Comparator.super.reversed(); + } +} diff --git a/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/UsedMemoryIncreasedComparator.java b/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/UsedMemoryIncreasedComparator.java new file mode 100644 index 0000000000..9b37b6bd8e --- /dev/null +++ b/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/UsedMemoryIncreasedComparator.java @@ -0,0 +1,57 @@ +package org.mobilitydata.gtfsvalidator.outputcomparator.io; + +import java.util.Comparator; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * Comparator to compare two {@link DatasetMemoryUsage} objects based on the difference between the + * used memory of the two objects. The difference is calculated by comparing the used memory of the + * two objects for each key present in both objects. If a key is present in one object but not in + * the other, the key it is ignored. This comparator is used to sort DatasetMemoryUsage by the + * minimum difference between the used memory of the two. This means the order is by the dataset + * validation that increased the memory. + */ +public class UsedMemoryIncreasedComparator implements Comparator { + + @Override + public int compare(DatasetMemoryUsage o1, DatasetMemoryUsage o2) { + if (o1 == o2) { + return 0; + } + if (o1 == null || o2 == null) { + return o1 == null ? -1 : 1; + } + if (o1.getReferenceMemoryUsage() == null + && o1.getLatestMemoryUsage() == null + && o2.getReferenceMemoryUsage() == null + && o2.getLatestMemoryUsage() == null) { + return 0; + } + if (o1.getReferenceMemoryUsage() == null || o2.getReferenceMemoryUsage() == null) { + return o1.getReferenceMemoryUsage() == null ? -1 : 1; + } + if (o1.getLatestMemoryUsage() == null || o2.getLatestMemoryUsage() == null) { + return o1.getLatestMemoryUsage() == null ? -1 : 1; + } + long o1MinDiff = + getMinimumDifferenceByKey(o1.getReferenceUsedMemoryByKey(), o1.getLatestUsedMemoryByKey()); + long o2MinDiff = + getMinimumDifferenceByKey(o2.getReferenceUsedMemoryByKey(), o2.getLatestUsedMemoryByKey()); + return Long.compare(o1MinDiff, o2MinDiff); + } + + private long getMinimumDifferenceByKey( + Map referenceMemoryUsage, Map latestMemoryUsage) { + Set keys = new HashSet<>(); + keys.addAll(latestMemoryUsage.keySet()); + keys.addAll(referenceMemoryUsage.keySet()); + return keys.stream() + .filter(key -> latestMemoryUsage.containsKey(key) && referenceMemoryUsage.containsKey(key)) + .filter(key -> latestMemoryUsage.get(key) - referenceMemoryUsage.get(key) != 0) + .mapToLong(key -> referenceMemoryUsage.get(key) - latestMemoryUsage.get(key)) + .min() + .orElse(Long.MAX_VALUE); + } +} diff --git a/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollector.java b/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollector.java index eadd1861ee..db7468d00f 100644 --- a/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollector.java +++ b/output-comparator/src/main/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollector.java @@ -1,17 +1,34 @@ package org.mobilitydata.gtfsvalidator.outputcomparator.io; import java.util.*; +import java.util.concurrent.atomic.AtomicBoolean; import org.mobilitydata.gtfsvalidator.model.ValidationReport; import org.mobilitydata.gtfsvalidator.outputcomparator.model.report.ValidationPerformance; +import org.mobilitydata.gtfsvalidator.performance.MemoryUsage; public class ValidationPerformanceCollector { + public static final int MEMORY_USAGE_COMPARE_MAX = 25; private final Map referenceTimes; private final Map latestTimes; + private final BoundedPriorityQueue datasetsDecreasedMemoryUsage; + private final BoundedPriorityQueue datasetsIncreasedMemoryUsage; + private final List datasetsMemoryUsageNoReference; public ValidationPerformanceCollector() { this.referenceTimes = new HashMap<>(); this.latestTimes = new HashMap<>(); + this.datasetsDecreasedMemoryUsage = + new BoundedPriorityQueue<>( + MEMORY_USAGE_COMPARE_MAX, + MEMORY_USAGE_COMPARE_MAX, + (new UsedMemoryIncreasedComparator().reversed())); + this.datasetsIncreasedMemoryUsage = + new BoundedPriorityQueue<>( + MEMORY_USAGE_COMPARE_MAX, + MEMORY_USAGE_COMPARE_MAX, + new UsedMemoryIncreasedComparator()); + this.datasetsMemoryUsageNoReference = new ArrayList<>(); } public void addReferenceTime(String sourceId, Double time) { @@ -69,6 +86,21 @@ private String formatMetrics(String metric, String datasetId, Double reference, "| %s | %s | %.2f | %.2f | %s |\n", metric, datasetId, reference, latest, diff); } + private static String getMemoryDiff(Long reference, Long latest) { + String diff; + if (reference == null || latest == null) { + diff = "N/A"; + } else { + long difference = latest - reference; + if (difference == 0) { + return "-"; + } + String arrow = difference > 0 ? "⬆️+" : "⬇️"; + diff = String.format("%s%s", arrow, MemoryUsage.convertToHumanReadableMemory(difference)); + } + return diff; + } + public String generateLogString() { StringBuilder b = new StringBuilder(); b.append("### ⏱️ Performance Assessment\n") @@ -176,11 +208,107 @@ public String generateLogString() { .append(String.join(", ", warnings)) .append("\n\n"); } + b.append("\n\n"); + if (datasetsIncreasedMemoryUsage.size() > 0 + || datasetsDecreasedMemoryUsage.size() > 0 + || datasetsMemoryUsageNoReference.size() > 0) { + b.append("
\n"); + b.append("📜 Memory Consumption\n"); + if (datasetsIncreasedMemoryUsage.size() > 0) { + List increasedMemoryUsages = + getDatasetMemoryUsages(datasetsIncreasedMemoryUsage); + addMemoryUsageReport(increasedMemoryUsages, "memory has increased", b, true); + } + if (datasetsDecreasedMemoryUsage.size() > 0) { + List decreasedMemoryUsages = + getDatasetMemoryUsages(datasetsDecreasedMemoryUsage); + addMemoryUsageReport(decreasedMemoryUsages, "memory has decreased", b, true); + } + if (datasetsMemoryUsageNoReference.size() > 0) { + // Sorting from the highest to the lowest memory usage + datasetsMemoryUsageNoReference.sort((new LatestReportUsedMemoryComparator()).reversed()); + addMemoryUsageReport( + datasetsMemoryUsageNoReference.subList( + 0, Math.min(datasetsMemoryUsageNoReference.size(), MEMORY_USAGE_COMPARE_MAX)), + "no reference available", + b, + false); + } + b.append("
\n"); + } return b.toString(); } + private List getDatasetMemoryUsages( + BoundedPriorityQueue datasetsMemoryUsage) { + List increasedMemoryUsages = new ArrayList<>(datasetsMemoryUsage); + increasedMemoryUsages.sort(datasetsMemoryUsage.comparator()); + return increasedMemoryUsages; + } + + private void addMemoryUsageReport( + List memoryUsages, + String order, + StringBuilder b, + boolean includeDifference) { + b.append(String.format("

List of %s datasets(%s).

", MEMORY_USAGE_COMPARE_MAX, order)) + .append("\n\n") + .append( + "| Dataset ID | Snapshot Key(Used Memory) | Reference | Latest |"); + if (includeDifference) { + b.append(" Difference |"); + } + b.append("\n"); + b.append( + "|-----------------------------|-------------------|----------------|----------------|"); + if (includeDifference) { + b.append("----------------|"); + } + b.append("\n"); + memoryUsages.stream() + .forEachOrdered( + datasetMemoryUsage -> { + generateMemoryLogByKey(datasetMemoryUsage, b, includeDifference); + }); + } + + private static void generateMemoryLogByKey( + DatasetMemoryUsage datasetMemoryUsage, StringBuilder b, boolean includeDifference) { + AtomicBoolean isFirst = new AtomicBoolean(true); + Set keys = new HashSet<>(); + keys.addAll(datasetMemoryUsage.getReferenceUsedMemoryByKey().keySet()); + keys.addAll(datasetMemoryUsage.getLatestUsedMemoryByKey().keySet()); + keys.stream() + .forEach( + key -> { + var reference = datasetMemoryUsage.getReferenceUsedMemoryByKey().get(key); + var latest = datasetMemoryUsage.getLatestUsedMemoryByKey().get(key); + if (isFirst.get()) { + b.append(String.format("| %s | | | |", datasetMemoryUsage.getDatasetId())); + if (includeDifference) { + b.append(" |"); + } + b.append("\n"); + isFirst.set(false); + } + String usedMemoryDiff = getMemoryDiff(reference, latest); + b.append( + String.format( + "| | %s | %s | %s |", + key, + reference != null + ? MemoryUsage.convertToHumanReadableMemory(reference) + : "N/A", + latest != null ? MemoryUsage.convertToHumanReadableMemory(latest) : "N/A")); + if (includeDifference) { + b.append(String.format(" %s |", usedMemoryDiff)); + } + b.append("\n"); + }); + } + public void compareValidationReports( String sourceId, ValidationReport referenceReport, ValidationReport latestReport) { if (referenceReport.getValidationTimeSeconds() != null) { @@ -189,6 +317,26 @@ public void compareValidationReports( if (latestReport.getValidationTimeSeconds() != null) { addLatestTime(sourceId, latestReport.getValidationTimeSeconds()); } + + compareValidationReportMemoryUsage(sourceId, referenceReport, latestReport); + } + + private void compareValidationReportMemoryUsage( + String sourceId, ValidationReport referenceReport, ValidationReport latestReport) { + DatasetMemoryUsage datasetMemoryUsage = + new DatasetMemoryUsage( + sourceId, + referenceReport.getMemoryUsageRecords(), + latestReport.getMemoryUsageRecords()); + if (referenceReport.getMemoryUsageRecords() != null + && referenceReport.getMemoryUsageRecords().size() > 0 + && latestReport.getMemoryUsageRecords() != null + && latestReport.getMemoryUsageRecords().size() > 0) { + datasetsIncreasedMemoryUsage.offer(datasetMemoryUsage); + datasetsDecreasedMemoryUsage.offer(datasetMemoryUsage); + } else { + datasetsMemoryUsageNoReference.add(datasetMemoryUsage); + } } public List toReport() { diff --git a/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/cli/ValidationReportComparatorTest.java b/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/cli/ValidationReportComparatorTest.java index 18e8d66e56..6cf2ac3a7d 100644 --- a/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/cli/ValidationReportComparatorTest.java +++ b/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/cli/ValidationReportComparatorTest.java @@ -141,7 +141,13 @@ public void addedErrorNotice_summaryString() throws Exception { + "\n" + "| Time Metric | Dataset ID | Reference (s) | Latest (s) | Difference (s) |\n" + "|-----------------------------|-------------------|----------------|----------------|----------------|\n" - + "\n\n\n"); + + "\n\n" + + "
\n" + + "📜 Memory Consumption\n" + + "

List of 25 datasets(no reference available).

\n\n" + + "| Dataset ID | Snapshot Key(Used Memory) | Reference | Latest |\n" + + "|-----------------------------|-------------------|----------------|----------------|\n" + + "
\n\n"); } @Test diff --git a/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/MemoryUsageUsedMemoryComparatorTest.java b/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/MemoryUsageUsedMemoryComparatorTest.java new file mode 100644 index 0000000000..b09e55387d --- /dev/null +++ b/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/MemoryUsageUsedMemoryComparatorTest.java @@ -0,0 +1,60 @@ +package org.mobilitydata.gtfsvalidator.outputcomparator.io; + +import static org.junit.Assert.assertEquals; + +import java.util.*; +import org.junit.Before; +import org.junit.Test; +import org.mobilitydata.gtfsvalidator.performance.MemoryUsage; + +public class MemoryUsageUsedMemoryComparatorTest { + + private UsedMemoryIncreasedComparator comparator; + + @Before + public void setUp() { + comparator = new UsedMemoryIncreasedComparator(); + } + + @Test + public void testCompare_equalMemoryUsage() { + List referenceMemoryUsage = getMemoryUsage(100L); + List latestMemoryUsage = getMemoryUsage(100L); + DatasetMemoryUsage o1 = + new DatasetMemoryUsage("dataset1", referenceMemoryUsage, latestMemoryUsage); + DatasetMemoryUsage o2 = + new DatasetMemoryUsage("dataset1", referenceMemoryUsage, latestMemoryUsage); + assertEquals(0, comparator.compare(o1, o2)); + } + + @Test + public void testCompare_firstHasMoreMemoryDifference() { + List referenceMemoryUsage = getMemoryUsage(100L); + List latestMemoryUsage = getMemoryUsage(50L); + DatasetMemoryUsage o1 = + new DatasetMemoryUsage("dataset1", referenceMemoryUsage, latestMemoryUsage); + DatasetMemoryUsage o2 = + new DatasetMemoryUsage("dataset1", referenceMemoryUsage, getMemoryUsage(100L)); + assertEquals(-1, comparator.compare(o1, o2)); + } + + @Test + public void testCompare_firstHasLessMemoryDifference() { + List referenceMemoryUsage = getMemoryUsage(100L); + List latestMemoryUsage = getMemoryUsage(50L); + DatasetMemoryUsage o1 = + new DatasetMemoryUsage("dataset1", referenceMemoryUsage, latestMemoryUsage); + DatasetMemoryUsage o2 = + new DatasetMemoryUsage("dataset1", referenceMemoryUsage, getMemoryUsage(10L)); + assertEquals(1, comparator.compare(o1, o2)); + } + + private static List getMemoryUsage(long freeMemory) { + MemoryUsage[] referenceMemoryUsage = + new MemoryUsage[] { + new MemoryUsage("key1", 100L, freeMemory, 100L, 100L), + new MemoryUsage("key2", 100L, freeMemory, 100L, 100L), + }; + return Arrays.asList(referenceMemoryUsage); + } +} diff --git a/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollectorTest.java b/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollectorTest.java index 962331fc32..82c204b20c 100644 --- a/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollectorTest.java +++ b/output-comparator/src/test/java/org/mobilitydata/gtfsvalidator/outputcomparator/io/ValidationPerformanceCollectorTest.java @@ -2,7 +2,11 @@ import static com.google.common.truth.Truth.assertThat; +import java.util.Arrays; +import java.util.Collections; import org.junit.Test; +import org.mobilitydata.gtfsvalidator.model.ValidationReport; +import org.mobilitydata.gtfsvalidator.performance.MemoryUsage; public class ValidationPerformanceCollectorTest { @@ -19,6 +23,49 @@ public void generateLogString_test() { collector.addReferenceTime("feed-id-b", 20.0); collector.addLatestTime("feed-id-b", 22.0); + // Adding some sample data + long baseMemory = 1000000; + // Memory usage latest null + collector.compareValidationReports( + "feed-id-m1", + new ValidationReport( + Collections.EMPTY_SET, + null, + Arrays.asList( + new MemoryUsage("key1", baseMemory, baseMemory + baseMemory * 10, 200, 50L), + new MemoryUsage("key2", baseMemory, baseMemory, 200, 50L))), + new ValidationReport(Collections.EMPTY_SET, 16.0, Collections.EMPTY_LIST)); + // Memory usage increased as there is less free memory + collector.compareValidationReports( + "feed-id-m2", + new ValidationReport( + Collections.EMPTY_SET, + null, + Arrays.asList( + new MemoryUsage("key1", baseMemory, baseMemory, 200, 50L), + new MemoryUsage("key2", baseMemory, baseMemory, 200, 50L))), + new ValidationReport( + Collections.EMPTY_SET, + null, + Arrays.asList( + new MemoryUsage("key1", baseMemory, baseMemory - baseMemory / 2, 200, null), + new MemoryUsage("key2", baseMemory, baseMemory - baseMemory / 2, 200, null)))); + + // // Memory usage decreased as there is more free memory + collector.compareValidationReports( + "feed-id-m3", + new ValidationReport( + Collections.EMPTY_SET, + null, + Arrays.asList( + new MemoryUsage("key3", baseMemory, baseMemory + 100, 200, null), + new MemoryUsage("key4", baseMemory, baseMemory + 100, 200, null))), + new ValidationReport( + Collections.EMPTY_SET, + null, + Arrays.asList( + new MemoryUsage("key3", baseMemory, baseMemory * 2, 200, null), + new MemoryUsage("key4", baseMemory, baseMemory * 2, 200, null)))); // Generating the log string String logString = collector.generateLogString(); String expectedLogString = @@ -37,7 +84,43 @@ public void generateLogString_test() { + "| Maximum in Reference Reports | feed-id-b | 20.00 | 22.00 | ⬆️+2.00 |\n" + "| Minimum in Latest Reports | feed-id-a | 14.00 | 18.00 | ⬆\uFE0F+4.00 |\n" + "| Maximum in Latest Reports | feed-id-b | 20.00 | 22.00 | ⬆️+2.00 |\n" - + "\n\n"; + + "#### ⚠️ Warnings\n\n" + + "The following dataset IDs are missing validation times either in reference or latest:\n" + + "feed-id-m1\n\n" + + "\n\n" + + "
\n" + + "📜 Memory Consumption\n" + + "

List of " + + ValidationPerformanceCollector.MEMORY_USAGE_COMPARE_MAX + + " datasets(memory has increased).

\n\n" + + "| Dataset ID | Snapshot Key(Used Memory) | Reference | Latest | Difference |\n" + + "|-----------------------------|-------------------|----------------|----------------|----------------|\n" + + "| feed-id-m2 | | | | |\n" + + "| | key1 | 0 bytes | 488.28 KiB | ⬆\uFE0F+488.28 KiB |\n" + + "| | key2 | 0 bytes | 488.28 KiB | ⬆\uFE0F+488.28 KiB |\n" + + "| feed-id-m3 | | | | |\n" + + "| | key3 | -100 bytes | -976.56 KiB | ⬇\uFE0F-976.46 KiB |\n" + + "| | key4 | -100 bytes | -976.56 KiB | ⬇\uFE0F-976.46 KiB |\n" + + "

List of " + + ValidationPerformanceCollector.MEMORY_USAGE_COMPARE_MAX + + " datasets(memory has decreased).

\n\n" + + "| Dataset ID | Snapshot Key(Used Memory) | Reference | Latest | Difference |\n" + + "|-----------------------------|-------------------|----------------|----------------|----------------|\n" + + "| feed-id-m3 | | | | |\n" + + "| | key3 | -100 bytes | -976.56 KiB | ⬇️-976.46 KiB |\n" + + "| | key4 | -100 bytes | -976.56 KiB | ⬇️-976.46 KiB |\n" + + "| feed-id-m2 | | | | |\n" + + "| | key1 | 0 bytes | 488.28 KiB | ⬆️+488.28 KiB |\n" + + "| | key2 | 0 bytes | 488.28 KiB | ⬆️+488.28 KiB |\n" + + "

List of " + + ValidationPerformanceCollector.MEMORY_USAGE_COMPARE_MAX + + " datasets(no reference available).

\n\n" + + "| Dataset ID | Snapshot Key(Used Memory) | Reference | Latest |\n" + + "|-----------------------------|-------------------|----------------|----------------|\n" + + "| feed-id-m1 | | | |\n" + + "| | key1 | -9.54 MiB | N/A |\n" + + "| | key2 | 0 bytes | N/A |\n" + + "
\n"; // Assert that the generated log string matches the expected log string assertThat(logString).isEqualTo(expectedLogString); }