From 9fe59797010353d909b0fa17926069f446ee049e Mon Sep 17 00:00:00 2001 From: Jingsi Lu Date: Wed, 25 Sep 2024 15:56:11 -0400 Subject: [PATCH] feat: 1534 service window in summary report (#1837) added service window in HTML report --- .../report/model/FeedMetadata.java | 168 +++++++++++++++++- main/src/main/resources/report.html | 7 +- .../report/model/FeedMetadataTest.java | 84 ++++++++- 3 files changed, 249 insertions(+), 10 deletions(-) diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java index 46bbe985c3..4edc8d7322 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java @@ -2,11 +2,15 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableSortedSet; +import com.google.common.flogger.FluentLogger; import com.vladsch.flexmark.util.misc.Pair; import java.time.LocalDate; +import java.time.format.DateTimeFormatter; import java.util.*; import java.util.function.Function; import org.mobilitydata.gtfsvalidator.table.*; +import org.mobilitydata.gtfsvalidator.util.CalendarUtil; +import org.mobilitydata.gtfsvalidator.util.ServicePeriod; public class FeedMetadata { /* @@ -20,6 +24,9 @@ public class FeedMetadata { public static final String FEED_INFO_FEED_LANGUAGE = "Feed Language"; public static final String FEED_INFO_FEED_START_DATE = "Feed Start Date"; public static final String FEED_INFO_FEED_END_DATE = "Feed End Date"; + public static final String FEED_INFO_SERVICE_WINDOW = "Service Window"; + + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); /* * Use these strings as keys in the counts map. Also used to specify the info that will appear in @@ -84,9 +91,22 @@ public static FeedMetadata from(GtfsFeedContainer feedContainer, ImmutableSet) feedContainer.getTableForFilename(GtfsFeedInfo.FILENAME).get()); } + feedMetadata.loadAgencyData( (GtfsTableContainer) feedContainer.getTableForFilename(GtfsAgency.FILENAME).get()); + + if (feedContainer.getTableForFilename(GtfsTrip.FILENAME).isPresent() + && (feedContainer.getTableForFilename(GtfsCalendar.FILENAME).isPresent() + || feedContainer.getTableForFilename(GtfsCalendarDate.FILENAME).isPresent())) { + feedMetadata.loadServiceWindow( + (GtfsTableContainer) feedContainer.getTableForFilename(GtfsTrip.FILENAME).get(), + (GtfsTableContainer) + feedContainer.getTableForFilename(GtfsCalendar.FILENAME).get(), + (GtfsTableContainer) + feedContainer.getTableForFilename(GtfsCalendarDate.FILENAME).get()); + } + feedMetadata.loadSpecFeatures(feedContainer); return feedMetadata; } @@ -368,16 +388,148 @@ private void loadFeedInfo(GtfsTableContainer feedTable) { feedInfo.put( FEED_INFO_FEED_LANGUAGE, info == null ? "N/A" : info.feedLang().getDisplayLanguage()); if (feedTable.hasColumn(GtfsFeedInfo.FEED_START_DATE_FIELD_NAME)) { - LocalDate localDate = info.feedStartDate().getLocalDate(); - String displayDate = - localDate.equals(GtfsFeedInfo.DEFAULT_FEED_START_DATE) ? "N/A" : localDate.toString(); - feedInfo.put(FEED_INFO_FEED_START_DATE, info == null ? "N/A" : displayDate); + if (info != null) { + LocalDate localDate = info.feedStartDate().getLocalDate(); + feedInfo.put(FEED_INFO_FEED_START_DATE, checkLocalDate(localDate)); + } } if (feedTable.hasColumn(GtfsFeedInfo.FEED_END_DATE_FIELD_NAME)) { - LocalDate localDate = info.feedEndDate().getLocalDate(); - String displayDate = - localDate.equals(GtfsFeedInfo.DEFAULT_FEED_END_DATE) ? "N/A" : localDate.toString(); - feedInfo.put(FEED_INFO_FEED_END_DATE, info == null ? "N/A" : displayDate); + if (info != null) { + LocalDate localDate = info.feedEndDate().getLocalDate(); + feedInfo.put(FEED_INFO_FEED_END_DATE, checkLocalDate(localDate)); + } + } + } + + private String checkLocalDate(LocalDate localDate) { + String displayDate; + if (localDate.toString().equals(LocalDate.EPOCH.toString())) { + displayDate = "N/A"; + } else { + displayDate = localDate.toString(); + } + return displayDate; + } + + /** + * Loads the service date range by determining the earliest start date and the latest end date for + * all services referenced with a trip\_id in `trips.txt`. It handles three cases: 1. When only + * `calendars.txt` is used. 2. When only `calendar\_dates.txt` is used. 3. When both + * `calendars.txt` and `calendar\_dates.txt` are used. + * + * @param tripContainer the container for `trips.txt` data + * @param calendarTable the container for `calendars.txt` data + * @param calendarDateTable the container for `calendar\_dates.txt` data + */ + public void loadServiceWindow( + GtfsTableContainer tripContainer, + GtfsTableContainer calendarTable, + GtfsTableContainer calendarDateTable) { + List trips = tripContainer.getEntities(); + + LocalDate earliestStartDate = null; + LocalDate latestEndDate = null; + try { + if ((calendarDateTable == null) && (calendarTable != null)) { + // When only calendars.txt is used + List calendars = calendarTable.getEntities(); + for (GtfsTrip trip : trips) { + String serviceId = trip.serviceId(); + for (GtfsCalendar calendar : calendars) { + if (calendar.serviceId().equals(serviceId)) { + LocalDate startDate = calendar.startDate().getLocalDate(); + LocalDate endDate = calendar.endDate().getLocalDate(); + if (startDate != null || endDate != null) { + if (startDate.toString().equals(LocalDate.EPOCH.toString()) + || endDate.toString().equals(LocalDate.EPOCH.toString())) { + continue; + } + if (earliestStartDate == null || startDate.isBefore(earliestStartDate)) { + earliestStartDate = startDate; + } + if (latestEndDate == null || endDate.isAfter(latestEndDate)) { + latestEndDate = endDate; + } + } + } + } + } + } else if ((calendarDateTable != null) && (calendarTable == null)) { + // When only calendar_dates.txt is used + List calendarDates = calendarDateTable.getEntities(); + for (GtfsTrip trip : trips) { + String serviceId = trip.serviceId(); + for (GtfsCalendarDate calendarDate : calendarDates) { + if (calendarDate.serviceId().equals(serviceId)) { + LocalDate date = calendarDate.date().getLocalDate(); + if (date != null && !date.toString().equals(LocalDate.EPOCH.toString())) { + if (earliestStartDate == null || date.isBefore(earliestStartDate)) { + earliestStartDate = date; + } + if (latestEndDate == null || date.isAfter(latestEndDate)) { + latestEndDate = date; + } + } + } + } + } + } else if ((calendarTable != null) && (calendarDateTable != null)) { + // When both calendars.txt and calendar_dates.txt are used + Map servicePeriods = + CalendarUtil.buildServicePeriodMap( + (GtfsCalendarTableContainer) calendarTable, + (GtfsCalendarDateTableContainer) calendarDateTable); + List removedDates = new ArrayList<>(); + for (GtfsTrip trip : trips) { + String serviceId = trip.serviceId(); + ServicePeriod servicePeriod = servicePeriods.get(serviceId); + LocalDate startDate = servicePeriod.getServiceStart(); + LocalDate endDate = servicePeriod.getServiceEnd(); + if (startDate != null && endDate != null) { + if (startDate.toString().equals(LocalDate.EPOCH.toString()) + || endDate.toString().equals(LocalDate.EPOCH.toString())) { + continue; + } + if (earliestStartDate == null || startDate.isBefore(earliestStartDate)) { + earliestStartDate = startDate; + } + if (latestEndDate == null || endDate.isAfter(latestEndDate)) { + latestEndDate = endDate; + } + } + removedDates.addAll(servicePeriod.getRemovedDays()); + } + + for (LocalDate date : removedDates) { + if (date.isEqual(earliestStartDate)) { + earliestStartDate = date.plusDays(1); + } + if (date.isEqual(latestEndDate)) { + latestEndDate = date.minusDays(1); + } + } + } + } catch (Exception e) { + logger.atSevere().withCause(e).log("Error while loading Service Window"); + } finally { + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("MMMM d, yyyy"); + if ((earliestStartDate == null) && (latestEndDate == null)) { + feedInfo.put(FEED_INFO_SERVICE_WINDOW, "N/A"); + } else if (earliestStartDate == null && latestEndDate != null) { + feedInfo.put(FEED_INFO_SERVICE_WINDOW, latestEndDate.format(formatter)); + } else if (latestEndDate == null && earliestStartDate != null) { + if (earliestStartDate.isAfter(latestEndDate)) { + feedInfo.put(FEED_INFO_SERVICE_WINDOW, "N/A"); + } else { + feedInfo.put(FEED_INFO_SERVICE_WINDOW, earliestStartDate.format(formatter)); + } + } else { + StringBuilder serviceWindow = new StringBuilder(); + serviceWindow.append(earliestStartDate); + serviceWindow.append(" to "); + serviceWindow.append(latestEndDate); + feedInfo.put(FEED_INFO_SERVICE_WINDOW, serviceWindow.toString()); + } } } diff --git a/main/src/main/resources/report.html b/main/src/main/resources/report.html index fa06e86da2..565f5fa5df 100644 --- a/main/src/main/resources/report.html +++ b/main/src/main/resources/report.html @@ -271,7 +271,12 @@

Feed Info

- + + + (?) + The range of service dates covered by the feed, based on trips with an associated service_id in calendar.txt and/or calendar_dates.txt + + diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java index cdebe4c4f6..df9844ff86 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java @@ -1,6 +1,7 @@ package org.mobilitydata.gtfsvalidator.report.model; import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertEquals; import com.google.common.collect.ImmutableList; import java.io.BufferedWriter; @@ -8,6 +9,7 @@ import java.io.FileWriter; import java.io.IOException; import java.time.LocalDate; +import java.util.List; import org.junit.Before; import org.junit.Rule; import org.junit.Test; @@ -17,6 +19,7 @@ import org.mobilitydata.gtfsvalidator.input.GtfsInput; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.table.*; +import org.mobilitydata.gtfsvalidator.type.GtfsDate; import org.mobilitydata.gtfsvalidator.validator.*; public class FeedMetadataTest { @@ -30,6 +33,12 @@ public class FeedMetadataTest { .build(); ValidatorLoader validatorLoader; File rootDir; + NoticeContainer noticeContainer = new NoticeContainer(); + + private GtfsTableContainer tripContainer; + private GtfsTableContainer calendarTable; + private GtfsTableContainer calendarDateTable; + private FeedMetadata feedMetadata = new FeedMetadata(); private void createDataFile(String filename, String content) throws IOException { File dataFile = tmpDir.newFile("data/" + filename); @@ -50,12 +59,85 @@ public void setup() throws IOException, ValidatorLoaderException { ValidatorLoader.createForClasses(ClassGraphDiscovery.discoverValidatorsInDefaultPackage()); } + public static GtfsTrip createTrip(int csvRowNumber, String serviceId) { + return new GtfsTrip.Builder().setCsvRowNumber(csvRowNumber).setServiceId(serviceId).build(); + } + + public static GtfsCalendar createCalendar( + int csvRowNumber, String serviceId, GtfsDate startDate, GtfsDate endDate) { + return new GtfsCalendar.Builder() + .setCsvRowNumber(csvRowNumber) + .setServiceId(serviceId) + .setStartDate(startDate) + .setEndDate(endDate) + .build(); + } + + public static GtfsCalendarDate createCalendarDate( + int csvRowNumber, + String serviceId, + GtfsDate date, + GtfsCalendarDateExceptionType exceptionType) { + return new GtfsCalendarDate.Builder() + .setCsvRowNumber(csvRowNumber) + .setServiceId(serviceId) + .setDate(date) + .setExceptionType(exceptionType) + .build(); + } + + @Test + public void testLoadServiceWindow() { + GtfsTrip trip1 = createTrip(1, "JUN24-MVS-SUB-Weekday-01"); + GtfsTrip trip2 = createTrip(2, "JUN24-MVS-SUB-Weekday-02"); + // when(tripContainer.getEntities()).thenReturn(List.of(trip1, trip2)); + tripContainer = GtfsTripTableContainer.forEntities(List.of(trip1, trip2), noticeContainer); + GtfsCalendar calendar1 = + createCalendar( + 1, + "JUN24-MVS-SUB-Weekday-01", + GtfsDate.fromLocalDate(LocalDate.of(2024, 1, 1)), + GtfsDate.fromLocalDate(LocalDate.of(2024, 12, 20))); + GtfsCalendar calendar2 = + createCalendar( + 2, + "JUN24-MVS-SUB-Weekday-02", + GtfsDate.fromLocalDate(LocalDate.of(2024, 6, 1)), + GtfsDate.fromLocalDate(LocalDate.of(2024, 12, 31))); + // when(calendarTable.getEntities()).thenReturn(List.of(calendar1, calendar2)); + calendarTable = + GtfsCalendarTableContainer.forEntities(List.of(calendar1, calendar2), noticeContainer); + GtfsCalendarDate calendarDate1 = + createCalendarDate( + 1, + "JUN24-MVS-SUB-Weekday-01", + GtfsDate.fromLocalDate(LocalDate.of(2024, 1, 1)), + GtfsCalendarDateExceptionType.SERVICE_REMOVED); + GtfsCalendarDate calendarDate2 = + createCalendarDate( + 2, + "JUN24-MVS-SUB-Weekday-02", + GtfsDate.fromLocalDate(LocalDate.of(2024, 6, 1)), + GtfsCalendarDateExceptionType.SERVICE_ADDED); + // when(calendarDateTable.getEntities()).thenReturn(List.of(calendarDate1, calendarDate2)); + calendarDateTable = + GtfsCalendarDateTableContainer.forEntities( + List.of(calendarDate1, calendarDate2), noticeContainer); + + // Call the method + feedMetadata.loadServiceWindow(tripContainer, calendarTable, calendarDateTable); + + // Verify the result + String expectedServiceWindow = "2024-01-02 to 2024-12-31"; + assertEquals( + expectedServiceWindow, feedMetadata.feedInfo.get(FeedMetadata.FEED_INFO_SERVICE_WINDOW)); + } + private void validateSpecFeature( String specFeature, Boolean expectedValue, ImmutableList>> tableDescriptors) throws IOException, InterruptedException { - NoticeContainer noticeContainer = new NoticeContainer(); feedLoaderMock = new GtfsFeedLoader(tableDescriptors); try (GtfsInput gtfsInput = GtfsInput.createFromPath(rootDir.toPath(), noticeContainer)) { GtfsFeedContainer feedContainer =