Skip to content

Commit

Permalink
Merge branch 'master' into feat/validate-polygons
Browse files Browse the repository at this point in the history
  • Loading branch information
cka-y authored Oct 9, 2024
2 parents e91e246 + fa91ddc commit e8e473d
Show file tree
Hide file tree
Showing 20 changed files with 1,018 additions and 44 deletions.
3 changes: 3 additions & 0 deletions core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ plugins {
id 'java'
id 'maven-publish'
id 'signing'
id 'io.freefair.aspectj.post-compile-weaving' version '6.4.1'
}

//publishing {
Expand All @@ -42,6 +43,8 @@ dependencies {
implementation 'com.googlecode.libphonenumber:libphonenumber:8.12.13'
implementation 'com.google.flogger:flogger:0.6'
implementation 'io.github.classgraph:classgraph:4.8.146'
implementation 'org.aspectj:aspectjrt:1.9.20'
implementation 'org.aspectj:aspectjweaver:1.9.20'
testImplementation 'com.google.flogger:flogger-system-backend:0.6'
testImplementation group: 'junit', name: 'junit', version: '4.13'
testImplementation "com.google.truth:truth:1.0.1"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,13 @@
import com.google.gson.JsonElement;
import com.google.gson.JsonObject;
import java.lang.reflect.Type;
import java.util.Collection;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import org.mobilitydata.gtfsvalidator.model.NoticeReport;
import org.mobilitydata.gtfsvalidator.model.ValidationReport;
import org.mobilitydata.gtfsvalidator.notice.Notice;
import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
import org.mobilitydata.gtfsvalidator.notice.ResolvedNotice;
import org.mobilitydata.gtfsvalidator.performance.MemoryUsage;

/**
* Used to (de)serialize a JSON validation report. This represents a validation report as a list of
Expand All @@ -44,6 +41,7 @@ public class ValidationReportDeserializer implements JsonDeserializer<Validation
private static final String NOTICES_MEMBER_NAME = "notices";
private static final String SUMMARY_MEMBER_NAME = "summary";
private static final String VALIDATION_TIME_MEMBER_NAME = "validationTimeSeconds";
private static final String MEMORY_USAGE_RECORDS_MEMBER_NAME = "memoryUsageRecords";

@Override
public ValidationReport deserialize(
Expand All @@ -53,17 +51,26 @@ public ValidationReport deserialize(
// Note that the json file contains the summary in addition to the notices, but it is ignored
// since currently the report comparison is only on the notices and the validation time.
Double validationTimeSeconds = null;
List<MemoryUsage> memoryUsageRecords = null;
if (rootObject.has(SUMMARY_MEMBER_NAME)) {
JsonObject summaryObject = rootObject.getAsJsonObject(SUMMARY_MEMBER_NAME);
if (summaryObject.has(VALIDATION_TIME_MEMBER_NAME)) {
validationTimeSeconds = summaryObject.get(VALIDATION_TIME_MEMBER_NAME).getAsDouble();
}
if (summaryObject.has(MEMORY_USAGE_RECORDS_MEMBER_NAME)) {
JsonArray memoryUsageArray = summaryObject.getAsJsonArray(MEMORY_USAGE_RECORDS_MEMBER_NAME);
memoryUsageRecords = new ArrayList<>();
for (JsonElement element : memoryUsageArray) {
MemoryUsage memoryUsage = Notice.GSON.fromJson(element, MemoryUsage.class);
memoryUsageRecords.add(memoryUsage);
}
}
}
JsonArray noticesArray = rootObject.getAsJsonArray(NOTICES_MEMBER_NAME);
for (JsonElement childObject : noticesArray) {
notices.add(Notice.GSON.fromJson(childObject, NoticeReport.class));
}
return new ValidationReport(notices, validationTimeSeconds);
return new ValidationReport(notices, validationTimeSeconds, memoryUsageRecords);
}

public static <T extends Notice> JsonObject serialize(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,10 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import org.mobilitydata.gtfsvalidator.io.ValidationReportDeserializer;
import org.mobilitydata.gtfsvalidator.performance.MemoryUsage;

/**
* Used to (de)serialize a {@code NoticeContainer}. This represents a validation report as a list of
Expand All @@ -42,6 +44,7 @@ public class ValidationReport {
.create();
private final Set<NoticeReport> notices;
private final Double validationTimeSeconds;
private List<MemoryUsage> memoryUsageRecords;

/**
* Public constructor needed for deserialization by {@code ValidationReportDeserializer}. Only
Expand All @@ -50,7 +53,7 @@ public class ValidationReport {
* @param noticeReports set of {@code NoticeReport}s
*/
public ValidationReport(Set<NoticeReport> noticeReports) {
this(noticeReports, null);
this(noticeReports, null, null);
}

/**
Expand All @@ -60,9 +63,13 @@ public ValidationReport(Set<NoticeReport> noticeReports) {
* @param noticeReports set of {@code NoticeReport}s
* @param validationTimeSeconds the time taken to validate the GTFS dataset
*/
public ValidationReport(Set<NoticeReport> noticeReports, Double validationTimeSeconds) {
public ValidationReport(
Set<NoticeReport> noticeReports,
Double validationTimeSeconds,
List<MemoryUsage> memoryUsageRecords) {
this.notices = Collections.unmodifiableSet(noticeReports);
this.validationTimeSeconds = validationTimeSeconds;
this.memoryUsageRecords = memoryUsageRecords;
}

/**
Expand All @@ -86,6 +93,9 @@ public Double getValidationTimeSeconds() {
return validationTimeSeconds;
}

public List<MemoryUsage> getMemoryUsageRecords() {
return memoryUsageRecords;
}
/**
* Determines if two validation reports are equal regardless of the order of the fields in the set
* of {@code NoticeReport}.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package org.mobilitydata.gtfsvalidator.performance;

import java.lang.annotation.ElementType;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;

/**
* Annotation to monitor memory usage of a method. The annotated method should return a {@link
* MemoryUsage} object. The key is used to group memory usage of different methods.
*/
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface MemoryMonitor {
String key() default "";
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package org.mobilitydata.gtfsvalidator.performance;

import org.apache.commons.lang3.StringUtils;
import org.aspectj.lang.ProceedingJoinPoint;
import org.aspectj.lang.annotation.Around;
import org.aspectj.lang.annotation.Aspect;
import org.aspectj.lang.reflect.MethodSignature;

/** Aspect to monitor memory usage of a method. */
@Aspect
public class MemoryMonitorAspect {

@Around("execution(@org.mobilitydata.gtfsvalidator.performance.MemoryMonitor * *(..))")
public Object monitorMemoryUsage(ProceedingJoinPoint joinPoint) throws Throwable {
String key = extractKey(joinPoint);
MemoryUsage before = MemoryUsageRegister.getInstance().getMemoryUsageSnapshot(key, null);
try {
Object result = joinPoint.proceed();
return result;
} finally {
MemoryUsage after = MemoryUsageRegister.getInstance().getMemoryUsageSnapshot(key, before);
MemoryUsageRegister.getInstance().registerMemoryUsage(after);
}
}

/**
* Extracts the key from the method signature or the annotation.
*
* @param joinPoint the join point
* @return the key either from the annotation or the method signature.
*/
private String extractKey(ProceedingJoinPoint joinPoint) {
var method = ((MethodSignature) joinPoint.getSignature()).getMethod();
var memoryMonitor = method.getAnnotation(MemoryMonitor.class);
return memoryMonitor != null && StringUtils.isNotBlank(memoryMonitor.key())
? memoryMonitor.key()
: method.getDeclaringClass().getCanonicalName() + "." + method.getName();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
package org.mobilitydata.gtfsvalidator.performance;

import java.text.DecimalFormat;
import org.apache.commons.lang3.StringUtils;

/** Represents memory usage information. */
public class MemoryUsage {
private static final DecimalFormat TWO_DECIMAL_FORMAT = new DecimalFormat("0.00");

private String key;
private long totalMemory;
private long freeMemory;
private long maxMemory;
private Long diffMemory;

public MemoryUsage() {}

public MemoryUsage(
String key, long totalMemory, long freeMemory, long maxMemory, Long diffMemory) {
this.key = key;
this.totalMemory = totalMemory;
this.freeMemory = freeMemory;
this.maxMemory = maxMemory;
this.diffMemory = diffMemory;
}

/**
* Converts bytes to human-readable memory.
*
* @param bytes
* @return human-readable memory, e.g., "1.23 GiB"
*/
public static String convertToHumanReadableMemory(Long bytes) {
if (bytes == null) {
return "N/A";
}
long size = Math.abs(bytes);
if (size < 1024) {
return bytes + " bytes";
}
if (size < 1048576) {
return TWO_DECIMAL_FORMAT.format(Math.copySign(size / 1024.0, bytes)) + " KiB";
}
if (size < 1073741824) {
return TWO_DECIMAL_FORMAT.format(Math.copySign(size / 1048576.0, bytes)) + " MiB";
}
if (size < 1099511627776L) {
return TWO_DECIMAL_FORMAT.format(Math.copySign(size / 1073741824.0, bytes)) + " GiB";
}
return TWO_DECIMAL_FORMAT.format(Math.copySign(size / 1099511627776L, bytes)) + " TiB";
}

/**
* The memory used is computed as the difference between the total memory and the free memory.
*
* @return the memory used.
*/
public long usedMemory() {
return totalMemory - freeMemory;
}

/**
* Returns a human-readable string representation of the memory usage.
*
* @return a human-readable string representation of the memory usage.
*/
public String humanReadablePrint() {
StringBuffer result = new StringBuffer();
result.append("Memory usage registered");
if (StringUtils.isNotBlank(key)) {
result.append(" for key: ").append(key);
} else {
result.append(":");
}
result.append(" Max: ").append(convertToHumanReadableMemory(maxMemory));
result.append(" Total: ").append(convertToHumanReadableMemory(totalMemory));
result.append(" Free: ").append(convertToHumanReadableMemory(freeMemory));
result.append(" Used: ").append(convertToHumanReadableMemory(usedMemory()));
result.append(" Diff: ").append(convertToHumanReadableMemory(diffMemory));
return result.toString();
}

public String getKey() {
return key;
}

public void setKey(String key) {
this.key = key;
}

public long getTotalMemory() {
return totalMemory;
}

public void setTotalMemory(long totalMemory) {
this.totalMemory = totalMemory;
}

public long getFreeMemory() {
return freeMemory;
}

public void setFreeMemory(long freeMemory) {
this.freeMemory = freeMemory;
}

public long getMaxMemory() {
return maxMemory;
}

public void setMaxMemory(long maxMemory) {
this.maxMemory = maxMemory;
}

public Long getDiffMemory() {
return diffMemory;
}

public void setDiffMemory(Long diffMemory) {
this.diffMemory = diffMemory;
}

@Override
public String toString() {
return "MemoryUsage{"
+ "key="
+ key
+ ", "
+ "totalMemory="
+ totalMemory
+ ", "
+ "freeMemory="
+ freeMemory
+ ", "
+ "maxMemory="
+ maxMemory
+ ", "
+ "diffMemory="
+ diffMemory
+ "}";
}

@Override
public boolean equals(Object o) {
if (o == this) {
return true;
}
if (o instanceof MemoryUsage) {
MemoryUsage that = (MemoryUsage) o;
return this.key.equals(that.getKey())
&& this.totalMemory == that.getTotalMemory()
&& this.freeMemory == that.getFreeMemory()
&& this.maxMemory == that.getMaxMemory()
&& (this.diffMemory == null
? that.getDiffMemory() == null
: this.getDiffMemory().equals(that.getDiffMemory()));
}
return false;
}

@Override
public int hashCode() {
int h$ = 1;
h$ *= 1000003;
h$ ^= key.hashCode();
h$ *= 1000003;
h$ ^= (int) ((totalMemory >>> 32) ^ totalMemory);
h$ *= 1000003;
h$ ^= (int) ((freeMemory >>> 32) ^ freeMemory);
h$ *= 1000003;
h$ ^= (int) ((maxMemory >>> 32) ^ maxMemory);
h$ *= 1000003;
h$ ^= (diffMemory == null) ? 0 : diffMemory.hashCode();
return h$;
}
}
Loading

0 comments on commit e8e473d

Please sign in to comment.