diff --git a/.gitignore b/.gitignore index e4b3a4c077..ee0a26f565 100644 --- a/.gitignore +++ b/.gitignore @@ -104,5 +104,6 @@ app/pkg/bin/ processor/notices/bin/ processor/notices/tests/bin/ web/service/bin/ +/web/service/execution_result.json RULES.md \ No newline at end of file diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java index 82d637755d..621fccaa58 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/notice/schema/NoticeSchemaGenerator.java @@ -32,6 +32,7 @@ import java.util.Optional; import java.util.TreeMap; import java.util.logging.Level; +import org.mobilitydata.gtfsvalidator.annotation.GtfsJson; import org.mobilitydata.gtfsvalidator.annotation.GtfsTable; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidationNotice.SectionRef; @@ -125,7 +126,12 @@ public static NoticeDocComments loadComments(Class noticeClass) { private static ReferencesSchema generateReferences(GtfsValidationNotice noticeAnnotation) { ReferencesSchema schema = new ReferencesSchema(); Arrays.stream(noticeAnnotation.files().value()) - .map(NoticeSchemaGenerator::getFileIdForTableClass) + .map( + // Both Table and Json annotations specify a file name, collect them all. + fileClass -> { + Optional fileId = getFileIdForTableClass(fileClass); + return fileId.or(() -> getFileIdForJsonClass(fileClass)); + }) .flatMap(Optional::stream) .forEach(schema::addFileReference); Arrays.stream(noticeAnnotation.bestPractices().value()) @@ -146,6 +152,11 @@ private static Optional getFileIdForTableClass(Class getFileIdForJsonClass(Class entityClass) { + GtfsJson annotation = entityClass.getAnnotation(GtfsJson.class); + return Optional.ofNullable(annotation).map(GtfsJson::value); + } + private static UrlReference convertUrlRef(UrlRef ref) { return new UrlReference(ref.label(), ref.url()); } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/CsvFileLoader.java similarity index 70% rename from core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java rename to core/src/main/java/org/mobilitydata/gtfsvalidator/table/CsvFileLoader.java index 3765bf22cb..e00baebcc8 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/CsvFileLoader.java @@ -7,48 +7,44 @@ import com.univocity.parsers.csv.CsvParserSettings; import java.io.InputStream; import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Optional; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import javax.annotation.Nullable; import org.mobilitydata.gtfsvalidator.notice.CsvParsingFailedNotice; import org.mobilitydata.gtfsvalidator.notice.EmptyFileNotice; -import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedFileNotice; -import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFileNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.parsing.CsvFile; import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; import org.mobilitydata.gtfsvalidator.parsing.CsvRow; import org.mobilitydata.gtfsvalidator.parsing.FieldCache; import org.mobilitydata.gtfsvalidator.parsing.RowParser; -import org.mobilitydata.gtfsvalidator.validator.FileValidator; import org.mobilitydata.gtfsvalidator.validator.SingleEntityValidator; import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; import org.mobilitydata.gtfsvalidator.validator.ValidatorUtil; -public final class AnyTableLoader { +/** This class loads csv files specifically. */ +public final class CsvFileLoader extends TableLoader { - private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private static final List> singleFileValidatorsWithParsingErrors = - new ArrayList<>(); + private CsvFileLoader() {} + // Create the singleton and add a method to obtain it + private static final CsvFileLoader INSTANCE = new CsvFileLoader(); - private static final List> - singleEntityValidatorsWithParsingErrors = new ArrayList<>(); - - public List> getValidatorsWithParsingErrors() { - return Collections.unmodifiableList(singleFileValidatorsWithParsingErrors); + @Nonnull + public static CsvFileLoader getInstance() { + return INSTANCE; } - public List> getSingleEntityValidatorsWithParsingErrors() { - return Collections.unmodifiableList(singleEntityValidatorsWithParsingErrors); - } + private final FluentLogger logger = FluentLogger.forEnclosingClass(); - public static GtfsTableContainer load( - GtfsTableDescriptor tableDescriptor, + @Override + public GtfsEntityContainer load( + GtfsFileDescriptor fileDescriptor, ValidatorProvider validatorProvider, InputStream csvInputStream, NoticeContainer noticeContainer) { + GtfsTableDescriptor tableDescriptor = (GtfsTableDescriptor) fileDescriptor; final String gtfsFilename = tableDescriptor.gtfsFilename(); CsvFile csvFile; @@ -61,13 +57,11 @@ public static GtfsTableContainer load( csvFile = new CsvFile(csvInputStream, gtfsFilename, settings); } catch (TextParsingException e) { noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e)); - return tableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.INVALID_HEADERS); + return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS); } if (csvFile.isEmpty()) { noticeContainer.addValidationNotice(new EmptyFileNotice(gtfsFilename)); - return tableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.EMPTY_FILE); + return tableDescriptor.createContainerForInvalidStatus(TableStatus.EMPTY_FILE); } final CsvHeader header = csvFile.getHeader(); final ImmutableList columnDescriptors = tableDescriptor.getColumns(); @@ -75,8 +69,7 @@ public static GtfsTableContainer load( validateHeaders(validatorProvider, gtfsFilename, header, columnDescriptors); noticeContainer.addAll(headerNotices); if (headerNotices.hasValidationErrors()) { - return tableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.INVALID_HEADERS); + return tableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS); } final int nColumns = columnDescriptors.size(); final ImmutableMap fieldLoadersMap = tableDescriptor.getFieldLoaders(); @@ -99,8 +92,8 @@ public static GtfsTableContainer load( final List entities = new ArrayList<>(); boolean hasUnparsableRows = false; final List> singleEntityValidators = - validatorProvider.createSingleEntityValidators( - tableDescriptor.getEntityClass(), singleEntityValidatorsWithParsingErrors::add); + createSingleEntityValidators(tableDescriptor.getEntityClass(), validatorProvider); + try { for (CsvRow row : csvFile) { if (row.getRowNumber() % 200000 == 0) { @@ -133,26 +126,23 @@ public static GtfsTableContainer load( } } catch (TextParsingException e) { noticeContainer.addValidationNotice(new CsvParsingFailedNotice(gtfsFilename, e)); - return tableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.UNPARSABLE_ROWS); + return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); } finally { logFieldCacheStats(gtfsFilename, fieldCaches, columnDescriptors); } if (hasUnparsableRows) { logger.atSevere().log("Failed to parse some rows in %s", gtfsFilename); - return tableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.UNPARSABLE_ROWS); + return tableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); } GtfsTableContainer table = tableDescriptor.createContainerForHeaderAndEntities(header, entities, noticeContainer); + ValidatorUtil.invokeSingleFileValidators( - validatorProvider.createSingleFileValidators( - table, singleFileValidatorsWithParsingErrors::add), - noticeContainer); + createSingleFileValidators(table, validatorProvider), noticeContainer); return table; } - private static NoticeContainer validateHeaders( + private NoticeContainer validateHeaders( ValidatorProvider validatorProvider, String gtfsFilename, CsvHeader header, @@ -178,7 +168,7 @@ private static NoticeContainer validateHeaders( return headerNotices; } - private static void logFieldCacheStats( + private void logFieldCacheStats( String gtfsFilename, FieldCache[] fieldCaches, ImmutableList columnDescriptors) { @@ -196,25 +186,4 @@ private static void logFieldCacheStats( } } } - - public static GtfsTableContainer loadMissingFile( - GtfsTableDescriptor tableDescriptor, - ValidatorProvider validatorProvider, - NoticeContainer noticeContainer) { - String gtfsFilename = tableDescriptor.gtfsFilename(); - GtfsTableContainer table = - tableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.MISSING_FILE); - if (tableDescriptor.isRecommended()) { - noticeContainer.addValidationNotice(new MissingRecommendedFileNotice(gtfsFilename)); - } - if (tableDescriptor.isRequired()) { - noticeContainer.addValidationNotice(new MissingRequiredFileNotice(gtfsFilename)); - } - ValidatorUtil.invokeSingleFileValidators( - validatorProvider.createSingleFileValidators( - table, singleFileValidatorsWithParsingErrors::add), - noticeContainer); - return table; - } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java new file mode 100644 index 0000000000..fcaef33a9e --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsEntityContainer.java @@ -0,0 +1,58 @@ +package org.mobilitydata.gtfsvalidator.table; + +import java.util.List; +import java.util.Optional; + +/** + * This class is the parent of containers holding table (csv) entities and containers holding JSON + * entities + * + * @param The entity for this container (e.g. GtfsCalendarDate or GtfsGeojsonFeature ) + * @param The descriptor for the file for the container (e.g. GtfsCalendarDateTableDescriptor or + * GtfsGeojsonFileDescriptor) + */ +public abstract class GtfsEntityContainer { + + private final D descriptor; + private final TableStatus tableStatus; + + public GtfsEntityContainer(D descriptor, TableStatus tableStatus) { + this.tableStatus = tableStatus; + this.descriptor = descriptor; + } + + public TableStatus getTableStatus() { + return tableStatus; + } + + public D getDescriptor() { + return descriptor; + } + + public abstract Class getEntityClass(); + + public int entityCount() { + return getEntities().size(); + } + + public abstract List getEntities(); + + public abstract String gtfsFilename(); + + public abstract Optional byTranslationKey(String recordId, String recordSubId); + + public boolean isMissingFile() { + return tableStatus == TableStatus.MISSING_FILE; + } + + public boolean isParsedSuccessfully() { + switch (tableStatus) { + case PARSABLE_HEADERS_AND_ROWS: + return true; + case MISSING_FILE: + return !descriptor.isRequired(); + default: + return false; + } + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java index 9b96d3d68d..967cfaedd8 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java @@ -18,20 +18,19 @@ import com.google.common.base.Ascii; import java.util.*; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; /** * Container for a whole parsed GTFS feed with all its tables. * - *

The tables are kept as {@code GtfsTableContainer} instances. + *

The tables are kept as {@link GtfsEntityContainer} instances. */ public class GtfsFeedContainer { - private final Map> tables = new HashMap<>(); - private final Map, GtfsTableContainer> tablesByClass = + private final Map> tables = new HashMap<>(); + private final Map, GtfsEntityContainer> tablesByClass = new HashMap<>(); - public GtfsFeedContainer(List> tableContainerList) { - for (GtfsTableContainer table : tableContainerList) { + public GtfsFeedContainer(List> tableContainerList) { + for (GtfsEntityContainer table : tableContainerList) { tables.put(table.gtfsFilename(), table); tablesByClass.put(table.getClass(), table); } @@ -49,11 +48,12 @@ public GtfsFeedContainer(List> tableContainerList) { * @param filename file name, including ".txt" extension * @return GTFS table or empty if the table is not supported by schema */ - public Optional> getTableForFilename(String filename) { - return Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null)); + public > Optional getTableForFilename(String filename) { + return (Optional) + Optional.ofNullable(tables.getOrDefault(Ascii.toLowerCase(filename), null)); } - public > T getTable(Class clazz) { + public > T getTable(Class clazz) { return (T) tablesByClass.get(clazz); } @@ -65,7 +65,7 @@ public > T getTable(Class clazz) { * @return true if all files were successfully parsed, false otherwise */ public boolean isParsedSuccessfully() { - for (GtfsTableContainer table : tables.values()) { + for (GtfsEntityContainer table : tables.values()) { if (!table.isParsedSuccessfully()) { return false; } @@ -73,13 +73,13 @@ public boolean isParsedSuccessfully() { return true; } - public Collection> getTables() { + public Collection> getTables() { return tables.values(); } public String tableTotalsText() { List totalList = new ArrayList<>(); - for (GtfsTableContainer table : tables.values()) { + for (GtfsEntityContainer table : tables.values()) { if (table.getTableStatus() == TableStatus.MISSING_FILE && !table.getDescriptor().isRequired()) { continue; diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java index 1edb051184..21dc38e98b 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedLoader.java @@ -19,6 +19,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.flogger.FluentLogger; import java.io.InputStream; +import java.lang.reflect.Modifier; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -46,7 +47,7 @@ */ public class GtfsFeedLoader { private static final FluentLogger logger = FluentLogger.forEnclosingClass(); - private final HashMap> tableDescriptors = new HashMap<>(); + private final HashMap> tableDescriptors = new HashMap<>(); private int numThreads = 1; /** @@ -57,11 +58,15 @@ public class GtfsFeedLoader { new ArrayList<>(); public GtfsFeedLoader( - ImmutableList>> tableDescriptorClasses) { - for (Class> clazz : tableDescriptorClasses) { - GtfsTableDescriptor descriptor; + ImmutableList>> tableDescriptorClasses) { + for (Class> clazz : tableDescriptorClasses) { + GtfsFileDescriptor descriptor; try { - descriptor = clazz.asSubclass(GtfsTableDescriptor.class).getConstructor().newInstance(); + // Skipping abstract classes. Example: GtfsTableDescriptor. + if (Modifier.isAbstract(clazz.getModifiers())) { + continue; + } + descriptor = clazz.asSubclass(GtfsFileDescriptor.class).getConstructor().newInstance(); } catch (ReflectiveOperationException e) { logger.atSevere().withCause(e).log( "Possible bug in GTFS annotation processor: expected a constructor without parameters" @@ -73,7 +78,7 @@ public GtfsFeedLoader( } } - public Collection> getTableDescriptors() { + public Collection> getTableDescriptors() { return Collections.unmodifiableCollection(tableDescriptors.values()); } @@ -100,18 +105,20 @@ public GtfsFeedContainer loadAndValidate( Map> remainingDescriptors = (Map>) tableDescriptors.clone(); for (String filename : gtfsInput.getFilenames()) { - GtfsTableDescriptor tableDescriptor = remainingDescriptors.remove(filename.toLowerCase()); + GtfsFileDescriptor tableDescriptor = remainingDescriptors.remove(filename.toLowerCase()); if (tableDescriptor == null) { noticeContainer.addValidationNotice(new UnknownFileNotice(filename)); } else { loaderCallables.add( () -> { NoticeContainer loaderNotices = new NoticeContainer(); - GtfsTableContainer tableContainer; + GtfsEntityContainer tableContainer; + // The descriptor knows what loader to use to load the file + TableLoader tableLoader = tableDescriptor.getTableLoader(); try (InputStream inputStream = gtfsInput.getFile(filename)) { try { tableContainer = - AnyTableLoader.load( + tableLoader.load( tableDescriptor, validatorProvider, inputStream, loaderNotices); } catch (RuntimeException e) { // This handler should prevent ExecutionException for @@ -121,8 +128,9 @@ public GtfsFeedContainer loadAndValidate( loaderNotices.addSystemError(new RuntimeExceptionInLoaderError(filename, e)); // Since the file was not loaded successfully, we treat // it as missing for continuing validation. + tableContainer = - AnyTableLoader.loadMissingFile( + tableLoader.loadMissingFile( tableDescriptor, validatorProvider, loaderNotices); } } @@ -130,11 +138,12 @@ public GtfsFeedContainer loadAndValidate( }); } } - ArrayList> tableContainers = new ArrayList<>(); + ArrayList> tableContainers = new ArrayList<>(); tableContainers.ensureCapacity(tableDescriptors.size()); - for (GtfsTableDescriptor tableDescriptor : remainingDescriptors.values()) { + for (GtfsFileDescriptor tableDescriptor : remainingDescriptors.values()) { + TableLoader tableLoader = tableDescriptor.getTableLoader(); tableContainers.add( - AnyTableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer)); + tableLoader.loadMissingFile(tableDescriptor, validatorProvider, noticeContainer)); } try { for (Future futureContainer : exec.invokeAll(loaderCallables)) { @@ -186,11 +195,11 @@ private static void addThreadExecutionError( } static class TableAndNoticeContainers { - final GtfsTableContainer tableContainer; + final GtfsEntityContainer tableContainer; final NoticeContainer noticeContainer; public TableAndNoticeContainers( - GtfsTableContainer tableContainer, NoticeContainer noticeContainer) { + GtfsEntityContainer tableContainer, NoticeContainer noticeContainer) { this.tableContainer = tableContainer; this.noticeContainer = noticeContainer; } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java new file mode 100644 index 0000000000..3a9193e503 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFileDescriptor.java @@ -0,0 +1,45 @@ +package org.mobilitydata.gtfsvalidator.table; + +import javax.annotation.Nonnull; + +/** + * This class provides some info about the different files within a GTFS dataset. Its children + * relate to either a csv table or a geojson file. + * + * @param The entity that will be extracted from the file. For example, GtfsCalendarDate or + * GtfsGeojsonFeature + */ +public abstract class GtfsFileDescriptor { + + public abstract C createContainerForInvalidStatus( + TableStatus tableStatus); + + // True if the specified file is required in a feed. + private boolean required; + + private TableStatus tableStatus; + + public abstract boolean isRecommended(); + + public abstract Class getEntityClass(); + + public abstract String gtfsFilename(); + + public boolean isRequired() { + return this.required; + } + + public void setRequired(boolean required) { + this.required = required; + } + + /** + * Get the looder for the file described by this file descriptor. + * + * @return the appropriate file loader. + */ + @Nonnull + public TableLoader getTableLoader() { + return CsvFileLoader.getInstance(); + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java index 010da6a1da..3723b2b08e 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java @@ -23,37 +23,23 @@ import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; /** - * Container for {@code GtfsEntity} instances for the whole GTFS table, e.g., stops.txt. + * Container for {@code GtfsEntity} instances coming from a CSV file. e.g., stops.txt. * *

Its subclasses are generated by annotation processor based on GTFS schema annotations. - * Instances of the subclasses are created by subclasses of {@code GtfsTableLoader} which are also - * generated by the processor. * * @param subclass of {@code GtfsEntity} + * @param subclass of {@code GtfsTableDescriptor} */ -public abstract class GtfsTableContainer { - - private final GtfsTableDescriptor descriptor; - - private final TableStatus tableStatus; +public abstract class GtfsTableContainer + extends GtfsEntityContainer { private final CsvHeader header; - public GtfsTableContainer( - GtfsTableDescriptor descriptor, TableStatus tableStatus, CsvHeader header) { - this.descriptor = descriptor; - this.tableStatus = tableStatus; + public GtfsTableContainer(D descriptor, TableStatus tableStatus, CsvHeader header) { + super(descriptor, tableStatus); this.header = header; } - public GtfsTableDescriptor getDescriptor() { - return descriptor; - } - - public TableStatus getTableStatus() { - return tableStatus; - } - public CsvHeader getHeader() { return header; } @@ -94,74 +80,4 @@ public boolean hasColumn(String columnName) { * @return entity with the given translation record id, if any */ public abstract Optional byTranslationKey(String recordId, String recordSubId); - - /** - * Tells if the file is missing. - * - * @return true if the file is missing, false otherwise - */ - public boolean isMissingFile() { - return tableStatus == TableStatus.MISSING_FILE; - } - - /** - * Tells if the file was successfully parsed. - * - *

If all files in the feed were successfully parsed, then file validators may be executed. - * - *

A successfully parsed file must meet the following conditions: - * - *

    - *
  • the file was successfully parsed as CSV; - *
  • all headers are valid, required headers are present; - *
  • all rows are successfully parsed; - *
  • if the file is required, it is present in the feed. - *
- * - * @return true if file was successfully parsed, false otherwise - */ - public boolean isParsedSuccessfully() { - switch (tableStatus) { - case PARSABLE_HEADERS_AND_ROWS: - return true; - case MISSING_FILE: - return !descriptor.isRequired(); - default: - return false; - } - } - - /** - * Status of loading this table. This is includes parsing of the CSV file and validation of the - * single file, but does not include any cross-file validations. - */ - public enum TableStatus { - /** The file is completely empty, i.e. it has no rows and even no headers. */ - EMPTY_FILE, - - /** The file is missing in the GTFS feed. */ - MISSING_FILE, - - /** The file was parsed successfully. It has headers and 0, 1 or many rows. */ - PARSABLE_HEADERS_AND_ROWS, - - /** - * The file has invalid headers, e.g., they failed to parse or some required headers are - * missing. The other rows were not scanned. - * - *

Note that unknown headers are not considered invalid. - */ - INVALID_HEADERS, - - /** - * Some of the rows failed to parse, e.g., they have missing required fields or invalid field - * values. - * - *

However, the headers are valid. - * - *

This does not include cross-file or cross-row validation. This also does not include - * single-entity validation. - */ - UNPARSABLE_ROWS, - } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java index 01f3b773db..b7a6bf7600 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java @@ -7,35 +7,18 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; -public abstract class GtfsTableDescriptor { +public abstract class GtfsTableDescriptor extends GtfsFileDescriptor { - // True if the specified file is required in a feed. - private boolean required; - - public abstract GtfsTableContainer createContainerForInvalidStatus( - GtfsTableContainer.TableStatus tableStatus); + @Override + public abstract GtfsTableContainer createContainerForInvalidStatus(TableStatus tableStatus); public abstract GtfsTableContainer createContainerForHeaderAndEntities( CsvHeader header, List entities, NoticeContainer noticeContainer); public abstract GtfsEntityBuilder createEntityBuilder(); - public abstract Class getEntityClass(); - - public abstract String gtfsFilename(); - public abstract ImmutableMap getFieldLoaders(); - public abstract boolean isRecommended(); - - public boolean isRequired() { - return this.required; - } - - public void setRequired(boolean required) { - this.required = required; - } - public abstract Optional maxCharsPerColumn(); public abstract ImmutableList getColumns(); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java new file mode 100644 index 0000000000..c02f54f46e --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableLoader.java @@ -0,0 +1,80 @@ +package org.mobilitydata.gtfsvalidator.table; + +import java.io.InputStream; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import org.mobilitydata.gtfsvalidator.notice.MissingRecommendedFileNotice; +import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFileNotice; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; +import org.mobilitydata.gtfsvalidator.validator.FileValidator; +import org.mobilitydata.gtfsvalidator.validator.SingleEntityValidator; +import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; +import org.mobilitydata.gtfsvalidator.validator.ValidatorUtil; + +/** Parent class for the different file loaders. */ +public abstract class TableLoader { + + private static final List> + singleEntityValidatorsWithParsingErrors = new ArrayList<>(); + + private static final List> singleFileValidatorsWithParsingErrors = + new ArrayList<>(); + + public static List> getValidatorsWithParsingErrors() { + return Collections.unmodifiableList(singleFileValidatorsWithParsingErrors); + } + + public static List> + getSingleEntityValidatorsWithParsingErrors() { + return Collections.unmodifiableList(singleEntityValidatorsWithParsingErrors); + } + + /** + * Load the file + * + * @param fileDescriptor Description of the file + * @param validatorProvider Will provide validators to run on the file. + * @param csvInputStream Stream to load from + * @param noticeContainer Where to put the notices if errors occur during the loading. + * @return A container for the loaded entities + */ + abstract GtfsEntityContainer load( + GtfsFileDescriptor fileDescriptor, + ValidatorProvider validatorProvider, + InputStream csvInputStream, + NoticeContainer noticeContainer); + + protected List> createSingleEntityValidators( + Class entityClass, ValidatorProvider validatorProvider) { + return validatorProvider.createSingleEntityValidators( + entityClass, singleEntityValidatorsWithParsingErrors::add); + } + + protected + List createSingleFileValidators( + GtfsEntityContainer table, ValidatorProvider validatorProvider) { + + return validatorProvider.createSingleFileValidators( + table, singleFileValidatorsWithParsingErrors::add); + } + + public GtfsEntityContainer loadMissingFile( + GtfsFileDescriptor tableDescriptor, + ValidatorProvider validatorProvider, + NoticeContainer noticeContainer) { + String gtfsFilename = tableDescriptor.gtfsFilename(); + GtfsEntityContainer table = + tableDescriptor.createContainerForInvalidStatus(TableStatus.MISSING_FILE); + if (tableDescriptor.isRecommended()) { + noticeContainer.addValidationNotice(new MissingRecommendedFileNotice(gtfsFilename)); + } + if (tableDescriptor.isRequired()) { + noticeContainer.addValidationNotice(new MissingRequiredFileNotice(gtfsFilename)); + } + ValidatorUtil.invokeSingleFileValidators( + createSingleFileValidators(table, validatorProvider), noticeContainer); + + return table; + } +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableStatus.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableStatus.java new file mode 100644 index 0000000000..0d88aaaf44 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/table/TableStatus.java @@ -0,0 +1,35 @@ +package org.mobilitydata.gtfsvalidator.table; + +/** + * Status of loading this table. This includes parsing of the CSV file and validation of the single + * file, but does not include any cross-file validations. + */ +public enum TableStatus { + /** The file is completely empty, i.e. it has no rows and even no headers. */ + EMPTY_FILE, + + /** The file is missing in the GTFS feed. */ + MISSING_FILE, + + /** The file was parsed successfully. It has headers and 0, 1 or many rows. */ + PARSABLE_HEADERS_AND_ROWS, + + /** + * The file has invalid headers, e.g., they failed to parse or some required headers are missing. + * The other rows were not scanned. + * + *

Note that unknown headers are not considered invalid. + */ + INVALID_HEADERS, + + /** + * Some of the rows failed to parse, e.g., they have missing required fields or invalid field + * values. + * + *

However, the headers are valid. + * + *

This does not include cross-file or cross-row validation. This also does not include + * single-entity validation. + */ + UNPARSABLE_ROWS, +} diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java index b51f788eb6..fbd5dfa80e 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/testing/LoadingHelper.java @@ -26,7 +26,7 @@ import org.mobilitydata.gtfsvalidator.input.DateForValidation; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; -import org.mobilitydata.gtfsvalidator.table.AnyTableLoader; +import org.mobilitydata.gtfsvalidator.table.CsvFileLoader; import org.mobilitydata.gtfsvalidator.table.GtfsEntity; import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; @@ -58,7 +58,7 @@ public void setValidatorLoader(ValidatorLoader validatorLoader) { this.validatorLoader = validatorLoader; } - public > Y load( + public > Y load( GtfsTableDescriptor tableDescriptor, String... lines) throws ValidatorLoaderException { String content = Arrays.stream(lines).collect(Collectors.joining("\n")); InputStream in = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); @@ -70,6 +70,6 @@ public > Y load( .setDateForValidation(new DateForValidation(dateForValidation)) .build(); ValidatorProvider provider = new DefaultValidatorProvider(context, validatorLoader); - return (Y) AnyTableLoader.load(tableDescriptor, provider, in, noticeContainer); + return (Y) CsvFileLoader.getInstance().load(tableDescriptor, provider, in, noticeContainer); } } diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ClassGraphDiscovery.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ClassGraphDiscovery.java index f435c3e079..4d0073377c 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ClassGraphDiscovery.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ClassGraphDiscovery.java @@ -7,7 +7,7 @@ import java.util.List; import org.mobilitydata.gtfsvalidator.annotation.GtfsValidator; import org.mobilitydata.gtfsvalidator.notice.Notice; -import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; +import org.mobilitydata.gtfsvalidator.table.GtfsFileDescriptor; /** Discovers GTFS table descriptor and validator classes in the given Java packages. */ public class ClassGraphDiscovery { @@ -23,8 +23,8 @@ private ClassGraphDiscovery() {} /** Discovers GtfsTableDescriptor subclasses in the default table package. */ @SuppressWarnings("unchecked") - public static ImmutableList>> discoverTables() { - ImmutableList.Builder>> tableDescriptors = + public static ImmutableList>> discoverTables() { + ImmutableList.Builder>> tableDescriptors = ImmutableList.builder(); try (ScanResult scanResult = new ClassGraph() @@ -32,8 +32,8 @@ public static ImmutableList>> discoverTab .enableAnnotationInfo() .acceptPackages(DEFAULT_TABLE_PACKAGE) .scan()) { - for (ClassInfo classInfo : scanResult.getSubclasses(GtfsTableDescriptor.class)) { - tableDescriptors.add((Class>) classInfo.loadClass()); + for (ClassInfo classInfo : scanResult.getSubclasses(GtfsFileDescriptor.class)) { + tableDescriptors.add((Class>) classInfo.loadClass()); } } return tableDescriptors.build(); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java index 835e494d3b..c5a1578670 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProvider.java @@ -22,8 +22,10 @@ import java.util.List; import java.util.function.Consumer; import org.mobilitydata.gtfsvalidator.table.GtfsEntity; +import org.mobilitydata.gtfsvalidator.table.GtfsEntityContainer; import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; import org.mobilitydata.gtfsvalidator.validator.ValidatorLoader.ValidatorWithDependencyStatus; /** Default implementation of {@link ValidatorProvider}. */ @@ -35,7 +37,8 @@ public class DefaultValidatorProvider implements ValidatorProvider { private final TableHeaderValidator tableHeaderValidator; private final ListMultimap, Class>> singleEntityValidators; - private final ListMultimap>, Class> + private final ListMultimap< + Class>, Class> singleFileValidators; private final List> multiFileValidators; @@ -103,12 +106,13 @@ public List> createSingleEntityV @Override @SuppressWarnings("unchecked") - public List createSingleFileValidators( - GtfsTableContainer table, - Consumer> validatorsWithParsingErrors) { + public + List createSingleFileValidators( + GtfsEntityContainer table, + Consumer> validatorsWithParsingErrors) { List validators = new ArrayList<>(); for (Class validatorClass : - singleFileValidators.get((Class>) table.getClass())) { + singleFileValidators.get((Class>) table.getClass())) { try { ValidatorWithDependencyStatus validatorWithStatus = ValidatorLoader.createSingleFileValidator(validatorClass, table, validationContext); diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoader.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoader.java index edd3ffb0a8..bb4e6b3c8a 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoader.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoader.java @@ -30,8 +30,8 @@ import javax.inject.Inject; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.table.GtfsEntity; +import org.mobilitydata.gtfsvalidator.table.GtfsEntityContainer; import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; /** * A {@code ValidatorLoader} object locates all validators registered with {@code @GtfsValidator} @@ -43,7 +43,8 @@ public class ValidatorLoader { private final ListMultimap, Class>> singleEntityValidators = ArrayListMultimap.create(); - private final ListMultimap>, Class> + private final ListMultimap< + Class>, Class> singleFileValidators = ArrayListMultimap.create(); private final List> multiFileValidators = new ArrayList<>(); @@ -75,7 +76,7 @@ private ValidatorLoader() {} } /** Loaded single-file validator classes keyed by table container class. */ - public ListMultimap>, Class> + public ListMultimap>, Class> getSingleFileValidators() { return singleFileValidators; } @@ -113,14 +114,14 @@ private void addFileValidator(Class validatorClass) // Indicates that the full GtfsFeedContainer needs to be injected. boolean injectFeedContainer = false; // Find out which GTFS tables need to be injected. - List>> injectedTables = new ArrayList<>(); + List>> injectedTables = new ArrayList<>(); for (Class parameterType : constructor.getParameterTypes()) { if (GtfsFeedContainer.class.isAssignableFrom(parameterType)) { injectFeedContainer = true; continue; } - if (GtfsTableContainer.class.isAssignableFrom(parameterType)) { - injectedTables.add((Class>) parameterType); + if (GtfsEntityContainer.class.isAssignableFrom(parameterType)) { + injectedTables.add((Class>) parameterType); } } @@ -201,7 +202,7 @@ ValidatorWithDependencyStatus createValidatorWithContext( public static ValidatorWithDependencyStatus createSingleFileValidator( Class clazz, - GtfsTableContainer table, + GtfsEntityContainer table, ValidationContext validationContext) throws ReflectiveOperationException, ValidatorLoaderException { return (ValidatorWithDependencyStatus) @@ -222,7 +223,7 @@ public static ValidatorWithDependencyStatus createM */ private static class DependencyResolver { private final ValidationContext context; - @Nullable private final GtfsTableContainer tableContainer; + @Nullable private final GtfsEntityContainer tableContainer; @Nullable private final GtfsFeedContainer feedContainer; /** This will be set to true if a resolved dependency was not parsed successfully. */ @@ -230,7 +231,7 @@ private static class DependencyResolver { public DependencyResolver( ValidationContext context, - @Nullable GtfsTableContainer tableContainer, + @Nullable GtfsEntityContainer tableContainer, @Nullable GtfsFeedContainer feedContainer) { this.context = context; this.tableContainer = tableContainer; @@ -257,9 +258,9 @@ public Object resolveDependency(Class parameterClass) { } return tableContainer; } - if (feedContainer != null && GtfsTableContainer.class.isAssignableFrom(parameterClass)) { - GtfsTableContainer container = - feedContainer.getTable((Class>) parameterClass); + if (feedContainer != null && GtfsEntityContainer.class.isAssignableFrom(parameterClass)) { + GtfsEntityContainer container = + feedContainer.getTable((Class>) parameterClass); if (container != null && !container.isParsedSuccessfully()) { dependenciesHaveErrors = true; } @@ -305,7 +306,8 @@ public String listValidators() { if (!singleFileValidators.isEmpty()) { builder.append("Single-file validators\n"); for (Map.Entry< - Class>, Collection>> + Class>, + Collection>> entry : singleFileValidators.asMap().entrySet()) { builder.append("\t").append(entry.getKey().getSimpleName()).append(": "); for (Class validatorClass : entry.getValue()) { diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java index 2c243a8dc9..2a9d754eca 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/validator/ValidatorProvider.java @@ -19,8 +19,9 @@ import java.util.List; import java.util.function.Consumer; import org.mobilitydata.gtfsvalidator.table.GtfsEntity; +import org.mobilitydata.gtfsvalidator.table.GtfsEntityContainer; import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; /** * Provider of all kinds of validators for fields, entities and files. @@ -57,9 +58,10 @@ List> createSingleEntityValidato * @param table GTFS table to validate * @param type of the GTFS entity */ - List createSingleFileValidators( - GtfsTableContainer table, - Consumer> validatorsWithParsingErrors); + + List createSingleFileValidators( + GtfsEntityContainer table, + Consumer> validatorsWithParsingErrors); /** * Creates a list of cross-table validators. Any validator that has a dependency with parse errors diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/CsvTableLoaderTest.java similarity index 83% rename from core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java rename to core/src/test/java/org/mobilitydata/gtfsvalidator/table/CsvTableLoaderTest.java index 6c6da42f6e..34cf2e1423 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/AnyTableLoaderTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/CsvTableLoaderTest.java @@ -30,7 +30,7 @@ import org.mockito.junit.MockitoRule; import org.mockito.quality.Strictness; -public class AnyTableLoaderTest { +public class CsvTableLoaderTest { @Rule public MockitoRule rule = MockitoJUnit.rule().strictness(Strictness.STRICT_STUBS); @Mock private GtfsTableContainer mockContainer; @@ -46,12 +46,12 @@ public void setup() { public void invalidInputStream() { var testTableDescriptor = mock(GtfsTableDescriptor.class); when(testTableDescriptor.gtfsFilename()).thenReturn("_not_a_valid_file_"); - when(testTableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.INVALID_HEADERS)) + when(testTableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS)) .thenReturn(mockContainer); var loadedContainer = - AnyTableLoader.load(testTableDescriptor, validatorProvider, null, loaderNotices); + CsvFileLoader.getInstance() + .load(testTableDescriptor, validatorProvider, null, loaderNotices); assertThat(validationNoticeTypes(loaderNotices)).containsExactly(CsvParsingFailedNotice.class); assertThat(loadedContainer).isEqualTo(mockContainer); @@ -61,13 +61,13 @@ public void invalidInputStream() { public void emptyInputStream() { var testTableDescriptor = mock(GtfsTableDescriptor.class); when(testTableDescriptor.gtfsFilename()).thenReturn("filename"); - when(testTableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.EMPTY_FILE)) + when(testTableDescriptor.createContainerForInvalidStatus(TableStatus.EMPTY_FILE)) .thenReturn(mockContainer); InputStream csvInputStream = toInputStream(""); var loadedContainer = - AnyTableLoader.load(testTableDescriptor, validatorProvider, csvInputStream, loaderNotices); + CsvFileLoader.getInstance() + .load(testTableDescriptor, validatorProvider, csvInputStream, loaderNotices); assertThat(loaderNotices.getValidationNotices()) .containsExactly(new EmptyFileNotice("filename")); @@ -79,8 +79,7 @@ public void invalidHeaders() { var testTableDescriptor = mock(GtfsTableDescriptor.class); when(testTableDescriptor.gtfsFilename()).thenReturn("filename"); when(testTableDescriptor.getColumns()).thenReturn(ImmutableList.of()); - when(testTableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.INVALID_HEADERS)) + when(testTableDescriptor.createContainerForInvalidStatus(TableStatus.INVALID_HEADERS)) .thenReturn(mockContainer); InputStream csvInputStream = toInputStream("A file with no headers"); ValidationNotice headerValidationNotice = new EmptyColumnNameNotice("stops.txt", 0); @@ -100,7 +99,8 @@ public void validate( when(validatorProvider.getTableHeaderValidator()).thenReturn(tableHeaderValidator); var loadedContainer = - AnyTableLoader.load(testTableDescriptor, validatorProvider, csvInputStream, loaderNotices); + CsvFileLoader.getInstance() + .load(testTableDescriptor, validatorProvider, csvInputStream, loaderNotices); assertThat(loaderNotices.getValidationNotices()).containsExactly(headerValidationNotice); assertThat(loadedContainer).isEqualTo(mockContainer); @@ -109,14 +109,14 @@ public void validate( @Test public void invalidRowLengthNotice() { var testTableDescriptor = spy(new GtfsTestTableDescriptor()); - when(testTableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.UNPARSABLE_ROWS)) + when(testTableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS)) .thenReturn(mockContainer); when(validatorProvider.getTableHeaderValidator()).thenReturn(mock(TableHeaderValidator.class)); InputStream inputStream = toInputStream("id,code\n" + "s1\n"); var loadedContainer = - AnyTableLoader.load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); + CsvFileLoader.getInstance() + .load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); assertThat(loaderNotices.getValidationNotices()) .containsExactly(new InvalidRowLengthNotice("filename.txt", 2, 1, 2)); @@ -136,10 +136,10 @@ public void parsableTableRows() { InputStream inputStream = toInputStream("id,stop_lat,_no_name_\n" + "s1, 23.00, no_value\n"); var loadedContainer = - AnyTableLoader.load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); + CsvFileLoader.getInstance() + .load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); - assertThat(loadedContainer.getTableStatus()) - .isEqualTo(GtfsTableContainer.TableStatus.PARSABLE_HEADERS_AND_ROWS); + assertThat(loadedContainer.getTableStatus()).isEqualTo(TableStatus.PARSABLE_HEADERS_AND_ROWS); verify(validator, times(1)).validate(any()); } @@ -165,15 +165,15 @@ public void missingRequiredField() { .setIsMixedCase(false) .setIsCached(false) .build())); - when(testTableDescriptor.createContainerForInvalidStatus( - GtfsTableContainer.TableStatus.UNPARSABLE_ROWS)) + when(testTableDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS)) .thenReturn(mockContainer); when(validatorProvider.getTableHeaderValidator()).thenReturn(mock(TableHeaderValidator.class)); when(validatorProvider.getFieldValidator()).thenReturn(mock(GtfsFieldValidator.class)); InputStream inputStream = toInputStream("id,code\n" + "s1,\n"); var loadedContainer = - AnyTableLoader.load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); + CsvFileLoader.getInstance() + .load(testTableDescriptor, validatorProvider, inputStream, loaderNotices); assertThat(loaderNotices.getValidationNotices()) .contains(new MissingRequiredFieldNotice("filename.txt", 2, "code")); diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainerTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainerTest.java index 21735bac2b..15073b5d4f 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainerTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainerTest.java @@ -20,7 +20,6 @@ import com.google.common.collect.ImmutableList; import org.junit.Test; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestTableContainer; public class GtfsFeedContainerTest { diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer.java index e84ab8041a..94940cced1 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer.java @@ -23,8 +23,10 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; +import org.mobilitydata.gtfsvalidator.table.TableStatus; -public class GtfsTestTableContainer extends GtfsTableContainer { +public class GtfsTestTableContainer + extends GtfsTableContainer { private static final ImmutableList KEY_COLUMN_NAMES = ImmutableList.of(GtfsTestEntity.ID_FIELD_NAME); diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer2.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer2.java index 962b5bb6e7..662adc7319 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer2.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer2.java @@ -23,9 +23,11 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; +import org.mobilitydata.gtfsvalidator.table.TableStatus; // We need a second test table class to test multi file validators. -public class GtfsTestTableContainer2 extends GtfsTableContainer { +public class GtfsTestTableContainer2 + extends GtfsTableContainer { private static final ImmutableList KEY_COLUMN_NAMES = ImmutableList.of(GtfsTestEntity.ID_FIELD_NAME); diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java index 58dcdae890..2da7a405a9 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java @@ -13,8 +13,7 @@ public class GtfsTestTableDescriptor extends GtfsTableDescriptor { @Override - public GtfsTableContainer createContainerForInvalidStatus( - GtfsTableContainer.TableStatus tableStatus) { + public GtfsTableContainer createContainerForInvalidStatus(TableStatus tableStatus) { return new GtfsTestTableContainer(tableStatus); } diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor2.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor2.java index c8442428a5..a3896d7237 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor2.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor2.java @@ -14,8 +14,7 @@ // We need a second test table descriptor to test multi file contaioners public class GtfsTestTableDescriptor2 extends GtfsTableDescriptor { @Override - public GtfsTableContainer createContainerForInvalidStatus( - GtfsTableContainer.TableStatus tableStatus) { + public GtfsTableContainer createContainerForInvalidStatus(TableStatus tableStatus) { return new GtfsTestTableContainer2(tableStatus); } diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProviderTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProviderTest.java index 90d10fa10f..823234582e 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProviderTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/DefaultValidatorProviderTest.java @@ -11,7 +11,7 @@ import org.junit.runners.JUnit4; import org.mobilitydata.gtfsvalidator.TestUtils; import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; +import org.mobilitydata.gtfsvalidator.table.TableStatus; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestEntity; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestEntityValidator; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestMultiFileValidator; diff --git a/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoaderTest.java b/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoaderTest.java index c169d29ac3..eadc3983d0 100644 --- a/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoaderTest.java +++ b/core/src/test/java/org/mobilitydata/gtfsvalidator/validator/ValidatorLoaderTest.java @@ -24,7 +24,7 @@ import org.mobilitydata.gtfsvalidator.input.CountryCode; import org.mobilitydata.gtfsvalidator.input.DateForValidation; import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; +import org.mobilitydata.gtfsvalidator.table.TableStatus; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestEntityValidator; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestSingleFileValidator; import org.mobilitydata.gtfsvalidator.testgtfs.GtfsTestTableContainer; diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeatureMetadata.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeatureMetadata.java index 47721e8f76..dc190746f2 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeatureMetadata.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeatureMetadata.java @@ -16,6 +16,10 @@ public String getFeatureName() { return featureName; } + public String getFeatureGroup() { + return featureGroup; + } + public String getDocUrl() { String formattedFeatureName = featureName.toLowerCase().replace(' ', '-'); String formattedFeatureGroup = featureGroup.toLowerCase().replace(' ', '_'); diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java index 6f58e93bb3..38f9d78f8b 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadata.java @@ -93,23 +93,25 @@ public static FeedMetadata from(GtfsFeedContainer feedContainer, ImmutableSet) - feedContainer.getTableForFilename(GtfsFeedInfo.FILENAME).get()); + Optional> + feedInfoTableOptional = feedContainer.getTableForFilename(GtfsFeedInfo.FILENAME); + feedMetadata.loadFeedInfo(feedInfoTableOptional.get()); + } + if (feedContainer.getTableForFilename(GtfsAgency.FILENAME).isPresent()) { + Optional> agencyTableOptional = + feedContainer.getTableForFilename(GtfsAgency.FILENAME); + feedMetadata.loadAgencyData(agencyTableOptional.get()); } - - feedMetadata.loadAgencyData( - (GtfsTableContainer) - feedContainer.getTableForFilename(GtfsAgency.FILENAME).get()); if (feedContainer.getTableForFilename(GtfsTrip.FILENAME).isPresent() && (feedContainer.getTableForFilename(GtfsCalendar.FILENAME).isPresent() || feedContainer.getTableForFilename(GtfsCalendarDate.FILENAME).isPresent())) { feedMetadata.loadServiceWindow( - (GtfsTableContainer) feedContainer.getTableForFilename(GtfsTrip.FILENAME).get(), - (GtfsTableContainer) + (GtfsTableContainer) + feedContainer.getTableForFilename(GtfsTrip.FILENAME).get(), + (GtfsTableContainer) feedContainer.getTableForFilename(GtfsCalendar.FILENAME).get(), - (GtfsTableContainer) + (GtfsTableContainer) feedContainer.getTableForFilename(GtfsCalendarDate.FILENAME).get()); } @@ -131,7 +133,7 @@ private void setCounts(GtfsFeedContainer feedContainer) { setCount(COUNTS_BLOCKS, feedContainer, GtfsTrip.FILENAME, GtfsTrip.class, GtfsTrip::blockId); } - private , E extends GtfsEntity> void setCount( + private void setCount( String countName, GtfsFeedContainer feedContainer, String fileName, @@ -141,13 +143,11 @@ private , E extends GtfsEntity> void setCount( var table = feedContainer.getTableForFilename(fileName); this.counts.put( countName, - table - .map(gtfsTableContainer -> loadUniqueCount(gtfsTableContainer, clazz, idExtractor)) - .orElse(0)); + table.map(gtfsContainer -> loadUniqueCount(gtfsContainer, clazz, idExtractor)).orElse(0)); } private int loadUniqueCount( - GtfsTableContainer table, Class clazz, Function idExtractor) { + GtfsEntityContainer table, Class clazz, Function idExtractor) { // Iterate through entities and count unique IDs Set uniqueIds = new HashSet<>(); for (GtfsEntity entity : table.getEntities()) { @@ -202,31 +202,33 @@ private boolean hasAtLeastOneTripWithAllFields(GtfsFeedContainer feedContainer) .map(GtfsStopTimeTableContainer::byTripIdMap) .map( byTripIdMap -> - byTripIdMap.keySet().stream() + byTripIdMap.asMap().values().stream() .anyMatch( - tripId -> { + gtfsStopTimes -> { boolean hasTripId = false, hasLocationId = false, hasStopId = false, hasArrivalTime = false, hasDepartureTime = false; - for (GtfsStopTime stopTime : byTripIdMap.get(tripId)) { + for (GtfsStopTime stopTime : gtfsStopTimes) { hasTripId |= stopTime.hasTripId(); hasLocationId |= stopTime.hasLocationId(); hasStopId |= stopTime.hasStopId(); hasArrivalTime |= stopTime.hasArrivalTime(); hasDepartureTime |= stopTime.hasDepartureTime(); + // Early return if all fields are found for this trip if (hasTripId && hasLocationId && hasStopId && hasArrivalTime && hasDepartureTime) { - return true; // Early return if all fields are found for this trip + return true; } } - return false; // Continue checking other trips + // Continue checking other trips + return false; })) .orElse(false); } @@ -238,8 +240,7 @@ private void loadZoneBasedDemandResponsiveTransitFeature(GtfsFeedContainer feedC } private boolean hasAtLeastOneTripWithOnlyLocationId(GtfsFeedContainer feedContainer) { - Optional> optionalStopTimeTable = - feedContainer.getTableForFilename(GtfsStopTime.FILENAME); + var optionalStopTimeTable = feedContainer.getTableForFilename(GtfsStopTime.FILENAME); if (optionalStopTimeTable.isPresent()) { for (GtfsEntity entity : optionalStopTimeTable.get().getEntities()) { if (entity instanceof GtfsStopTime) { @@ -393,13 +394,15 @@ private void loadRouteColorsFeature(GtfsFeedContainer feedContainer) { List.of((Function) GtfsRoute::hasRouteTextColor))); } - private void loadAgencyData(GtfsTableContainer agencyTable) { + private void loadAgencyData( + GtfsEntityContainer agencyTable) { for (GtfsAgency agency : agencyTable.getEntities()) { agencies.add(AgencyMetadata.from(agency)); } } - private void loadFeedInfo(GtfsTableContainer feedTable) { + private void loadFeedInfo( + GtfsTableContainer feedTable) { var info = feedTable.getEntities().isEmpty() ? null : feedTable.getEntities().get(0); feedInfo.put(FEED_INFO_PUBLISHER_NAME, info == null ? "N/A" : info.feedPublisherName()); @@ -442,9 +445,9 @@ private String checkLocalDate(LocalDate localDate) { * @param calendarDateTable the container for `calendar\_dates.txt` data */ public void loadServiceWindow( - GtfsTableContainer tripContainer, - GtfsTableContainer calendarTable, - GtfsTableContainer calendarDateTable) { + GtfsTableContainer tripContainer, + GtfsTableContainer calendarTable, + GtfsTableContainer calendarDateTable) { List trips = tripContainer.getEntities(); LocalDate earliestStartDate = null; @@ -582,7 +585,7 @@ private boolean hasAtLeastOneRecordForFields( public ArrayList foundFiles() { var foundFiles = new ArrayList(); for (var table : tableMetaData.values()) { - if (table.getTableStatus() != GtfsTableContainer.TableStatus.MISSING_FILE) { + if (table.getTableStatus() != TableStatus.MISSING_FILE) { foundFiles.add(table.getFilename()); } } @@ -600,4 +603,13 @@ public void setFilenames(ImmutableSortedSet filenames) { public ImmutableSortedSet getFilenames() { return filenames; } + + public Boolean hasFlexFeatures() { + return specFeatures.keySet().stream() + .anyMatch( + feature -> + feature.getFeatureGroup() != null + && feature.getFeatureGroup().equals("Flexible Services") + && specFeatures.get(feature)); + } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/TableMetadata.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/TableMetadata.java index 5cddcccb5f..cd34b9400b 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/TableMetadata.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/report/model/TableMetadata.java @@ -1,21 +1,21 @@ package org.mobilitydata.gtfsvalidator.report.model; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; +import org.mobilitydata.gtfsvalidator.table.GtfsEntityContainer; +import org.mobilitydata.gtfsvalidator.table.TableStatus; public class TableMetadata { private final String filename; - private final GtfsTableContainer.TableStatus tableStatus; + private final TableStatus tableStatus; private final int entityCount; - public TableMetadata( - String filename, GtfsTableContainer.TableStatus tableStatus, int entityCount) { + public TableMetadata(String filename, TableStatus tableStatus, int entityCount) { this.filename = filename; this.tableStatus = tableStatus; this.entityCount = entityCount; } - public static TableMetadata from(GtfsTableContainer table) { + public static TableMetadata from(GtfsEntityContainer table) { return new TableMetadata(table.gtfsFilename(), table.getTableStatus(), table.entityCount()); } @@ -23,7 +23,7 @@ public String getFilename() { return filename; } - public GtfsTableContainer.TableStatus getTableStatus() { + public TableStatus getTableStatus() { return tableStatus; } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java index 1127a25533..214e7d7526 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/runner/ValidationRunner.java @@ -40,9 +40,9 @@ import org.mobilitydata.gtfsvalidator.report.JsonReport; import org.mobilitydata.gtfsvalidator.report.JsonReportGenerator; import org.mobilitydata.gtfsvalidator.report.model.FeedMetadata; -import org.mobilitydata.gtfsvalidator.table.AnyTableLoader; import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; import org.mobilitydata.gtfsvalidator.table.GtfsFeedLoader; +import org.mobilitydata.gtfsvalidator.table.TableLoader; import org.mobilitydata.gtfsvalidator.util.VersionInfo; import org.mobilitydata.gtfsvalidator.util.VersionResolver; import org.mobilitydata.gtfsvalidator.validator.*; @@ -90,7 +90,6 @@ public Status run(ValidationRunnerConfig config) { return Status.EXCEPTION; } GtfsFeedLoader feedLoader = new GtfsFeedLoader(ClassGraphDiscovery.discoverTables()); - AnyTableLoader anyTableLoader = new AnyTableLoader(); logger.atInfo().log("validation config:\n%s", config); logger.atInfo().log("validators:\n%s", validatorLoader.listValidators()); @@ -138,7 +137,7 @@ public Status run(ValidationRunnerConfig config) { // Output exportReport(feedMetadata, noticeContainer, config, versionInfo); - printSummary(feedMetadata, feedContainer, feedLoader, anyTableLoader); + printSummary(feedMetadata, feedContainer, feedLoader); return Status.SUCCESS; } @@ -149,19 +148,16 @@ public Status run(ValidationRunnerConfig config) { * @param feedContainer the {@code GtfsFeedContainer} */ public static void printSummary( - FeedMetadata feedMetadata, - GtfsFeedContainer feedContainer, - GtfsFeedLoader loader, - AnyTableLoader anyTableLoader) { + FeedMetadata feedMetadata, GtfsFeedContainer feedContainer, GtfsFeedLoader loader) { final long endNanos = System.nanoTime(); List> multiFileValidatorsWithParsingErrors = loader.getMultiFileValidatorsWithParsingErrors(); List> singleFileValidatorsWithParsingErrors = - anyTableLoader.getValidatorsWithParsingErrors(); + TableLoader.getValidatorsWithParsingErrors(); // In theory single entity validators do not depend on files so there should not be any of these // with parsing errors List> singleEntityValidatorsWithParsingErrors = - anyTableLoader.getSingleEntityValidatorsWithParsingErrors(); + TableLoader.getSingleEntityValidatorsWithParsingErrors(); if (!singleFileValidatorsWithParsingErrors.isEmpty() || !singleEntityValidatorsWithParsingErrors.isEmpty() || !multiFileValidatorsWithParsingErrors.isEmpty()) { diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoader.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoader.java new file mode 100644 index 0000000000..87357ba0f8 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoader.java @@ -0,0 +1,101 @@ +package org.mobilitydata.gtfsvalidator.table; + +import com.google.common.flogger.FluentLogger; +import com.google.gson.JsonArray; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParseException; +import com.google.gson.JsonParser; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; +import org.mobilitydata.gtfsvalidator.notice.IOError; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; +import org.mobilitydata.gtfsvalidator.validator.ValidatorProvider; + +/** + * This class knows how to load a geojson file. Typical geojson file: { "type": "FeatureCollection", + * "features": [ { "id": "area_548", "type": "Feature", "geometry": { "type": "Polygon", + * "coordinates": [ [ [ -122.4112929, 48.0834848 ], ... ] ] }, "properties": { "stop_name": "Some + * name", "stop_desc": "Some description" } }, ... ] } + */ +public class GeojsonFileLoader extends TableLoader { + private static final FluentLogger logger = FluentLogger.forEnclosingClass(); + + @Override + public GtfsEntityContainer load( + GtfsFileDescriptor fileDescriptor, + ValidatorProvider validatorProvider, + InputStream inputStream, + NoticeContainer noticeContainer) { + GtfsGeojsonFileDescriptor geojsonFileDescriptor = (GtfsGeojsonFileDescriptor) fileDescriptor; + try { + List entities = extractFeaturesFromStream(inputStream, noticeContainer); + return geojsonFileDescriptor.createContainerForEntities(entities, noticeContainer); + } catch (JsonParseException jpex) { + // TODO: Add a notice for malformed locations.geojson + logger.atSevere().withCause(jpex).log("Malformed JSON in locations.geojson"); + return geojsonFileDescriptor.createContainerForEntities(new ArrayList<>(), noticeContainer); + } catch (IOException ioex) { + noticeContainer.addSystemError(new IOError(ioex)); + return fileDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); + } catch (Exception ex) { + logger.atSevere().withCause(ex).log("Error while loading locations.geojson"); + return fileDescriptor.createContainerForInvalidStatus(TableStatus.UNPARSABLE_ROWS); + } + } + + public List extractFeaturesFromStream( + InputStream inputStream, NoticeContainer noticeContainer) throws IOException { + List features = new ArrayList<>(); + try (InputStreamReader reader = new InputStreamReader(inputStream)) { + JsonObject jsonObject = JsonParser.parseReader(reader).getAsJsonObject(); + JsonArray featuresArray = jsonObject.getAsJsonArray("features"); + for (JsonElement feature : featuresArray) { + GtfsGeojsonFeature gtfsGeojsonFeature = extractFeature(feature, noticeContainer); + if (gtfsGeojsonFeature != null) { + features.add(gtfsGeojsonFeature); + } + } + } + return features; + } + + public GtfsGeojsonFeature extractFeature(JsonElement feature, NoticeContainer noticeContainer) { + GtfsGeojsonFeature gtfsGeojsonFeature = null; + if (feature.isJsonObject()) { + JsonObject featureObject = feature.getAsJsonObject(); + if (featureObject.has("properties")) { + JsonObject properties = featureObject.getAsJsonObject("properties"); + // Add stop_name and stop_desc + } else { + // Add a notice because properties is required + } + if (featureObject.has("id")) { + gtfsGeojsonFeature = new GtfsGeojsonFeature(); + gtfsGeojsonFeature.setFeatureId(featureObject.get("id").getAsString()); + } else { + // Add a notice because id is required + } + + if (featureObject.has("geometry")) { + JsonObject geometry = featureObject.getAsJsonObject("geometry"); + if (geometry.has("type")) { + String type = geometry.get("type").getAsString(); + if (type.equals("Polygon")) { + // Extract the polygon + } else if (type.equals("Multipolygon")) { + // extract the multipolygon + } + } else { + // Add a notice because type is required + } + } else { + // Add a notice because geometry is required + } + } + return gtfsGeojsonFeature; + } +} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java new file mode 100644 index 0000000000..76c5a33bd5 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeature.java @@ -0,0 +1,34 @@ +package org.mobilitydata.gtfsvalidator.table; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + +/** This class contains the information from one feature in the geojson file. */ +public final class GtfsGeojsonFeature implements GtfsEntity { + public static final String FILENAME = "locations.geojson"; + + public static final String FEATURE_ID_FIELD_NAME = "feature_id"; + + private String featureId; // The id of a feature in the GeoJSON file. + + public GtfsGeojsonFeature() {} + + // TODO: Change the interface hierarchy so we dont need this. It's not relevant for geojson + @Override + public int csvRowNumber() { + return 0; + } + + @Nonnull + public String featureId() { + return featureId; + } + + public boolean hasFeatureId() { + return featureId != null; + } + + public void setFeatureId(@Nullable String featureId) { + this.featureId = featureId; + } +} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureSchema.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureSchema.java new file mode 100644 index 0000000000..0a644ae7d2 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeatureSchema.java @@ -0,0 +1,13 @@ +package org.mobilitydata.gtfsvalidator.table; + +import org.mobilitydata.gtfsvalidator.annotation.GtfsJson; + +/** + * This class contains the information from one feature in the geojson file. Note that currently no + * class is autogenerated from this schema, contrarily to csv based entities. + */ +@GtfsJson("locations.geojson") +public interface GtfsGeojsonFeatureSchema extends GtfsEntity { + + String featureId(); +} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java new file mode 100644 index 0000000000..04c48d4909 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFeaturesContainer.java @@ -0,0 +1,89 @@ +/* + * Copyright 2024 MobilityData + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.mobilitydata.gtfsvalidator.table; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; + +/** + * Container for geojson features. Contrarily to the csv containers, this class is not auto + * generated since we have only one such class. + */ +public class GtfsGeojsonFeaturesContainer + extends GtfsEntityContainer { + + private final Map byLocationIdMap = new HashMap<>(); + + private final List entities; + + public GtfsGeojsonFeaturesContainer( + GtfsGeojsonFileDescriptor descriptor, + List entities, + NoticeContainer noticeContainer) { + super(descriptor, TableStatus.PARSABLE_HEADERS_AND_ROWS); + this.entities = entities; + setupIndices(noticeContainer); + } + + public GtfsGeojsonFeaturesContainer( + GtfsGeojsonFileDescriptor descriptor, TableStatus tableStatus) { + super(descriptor, tableStatus); + this.entities = new ArrayList<>(); + } + + @Override + public Class getEntityClass() { + return GtfsGeojsonFeature.class; + } + + @Override + public List getEntities() { + return entities; + } + + @Override + public String gtfsFilename() { + return "locations.geojson"; + } + + @Override + public Optional byTranslationKey(String recordId, String recordSubId) { + return Optional.empty(); + } + + private void setupIndices(NoticeContainer noticeContainer) { + for (GtfsGeojsonFeature newEntity : entities) { + if (!newEntity.hasFeatureId()) { + continue; + } + GtfsGeojsonFeature oldEntity = byLocationIdMap.getOrDefault(newEntity.featureId(), null); + if (oldEntity == null) { + byLocationIdMap.put(newEntity.featureId(), newEntity); + } + // TODO: Removed that code until the notice is supported. + // else { + // noticeContainer.addValidationNotice( + // new JsonDuplicateKeyNotice( + // gtfsFilename(), GtfsGeojsonFeature.FEATURE_ID_FIELD_NAME, + // newEntity.featureId())); + // } + } + } +} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java new file mode 100644 index 0000000000..ce96f087ca --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsGeojsonFileDescriptor.java @@ -0,0 +1,46 @@ +package org.mobilitydata.gtfsvalidator.table; + +import java.util.List; +import javax.annotation.Nonnull; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; + +/** + * File descriptor for geojson file. Contrarily to the csv file descriptor, this class is not auto + * generated since we have only one such class. + */ +public class GtfsGeojsonFileDescriptor extends GtfsFileDescriptor { + + public GtfsGeojsonFileDescriptor() { + setRequired(false); + } + + public GtfsGeojsonFeaturesContainer createContainerForEntities( + List entities, NoticeContainer noticeContainer) { + return new GtfsGeojsonFeaturesContainer(this, entities, noticeContainer); + } + + @Override + public GtfsGeojsonFeaturesContainer createContainerForInvalidStatus(TableStatus tableStatus) { + return new GtfsGeojsonFeaturesContainer(this, tableStatus); + } + + @Override + public boolean isRecommended() { + return false; + } + + @Override + public Class getEntityClass() { + return GtfsGeojsonFeature.class; + } + + @Override + public String gtfsFilename() { + return "locations.geojson"; + } + + @Nonnull + public TableLoader getTableLoader() { + return new GeojsonFileLoader(); + } +} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationGroupStopsSchema.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationGroupStopsSchema.java new file mode 100644 index 0000000000..b8e661a684 --- /dev/null +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsLocationGroupStopsSchema.java @@ -0,0 +1,23 @@ +package org.mobilitydata.gtfsvalidator.table; + +import org.mobilitydata.gtfsvalidator.annotation.FieldType; +import org.mobilitydata.gtfsvalidator.annotation.FieldTypeEnum; +import org.mobilitydata.gtfsvalidator.annotation.GtfsTable; +import org.mobilitydata.gtfsvalidator.annotation.Index; +import org.mobilitydata.gtfsvalidator.annotation.Required; + +@GtfsTable("location_group_stops.txt") +public interface GtfsLocationGroupStopsSchema extends GtfsEntity { + + @FieldType(FieldTypeEnum.ID) + // TODO: Put back the foreign key annotation when ready to publish the notice + // @ForeignKey(table = "location_groups.txt", field = "location_group_id") + @Index + @Required + String locationGroupId(); + + @FieldType(FieldTypeEnum.ID) + // @ForeignKey(table = "stops.txt", field = "stop_id") + @Required + String stopId(); +} diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/BookingRulesEntityValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/BookingRulesEntityValidator.java index b2f4f2261e..66d254a8aa 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/BookingRulesEntityValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/BookingRulesEntityValidator.java @@ -12,6 +12,7 @@ import org.mobilitydata.gtfsvalidator.table.GtfsBookingRules; import org.mobilitydata.gtfsvalidator.table.GtfsBookingRulesSchema; import org.mobilitydata.gtfsvalidator.table.GtfsBookingType; +import org.mobilitydata.gtfsvalidator.type.GtfsTime; @GtfsValidator public class BookingRulesEntityValidator extends SingleEntityValidator { @@ -39,25 +40,46 @@ public void validate(GtfsBookingRules entity, NoticeContainer noticeContainer) { validatePriorNoticeDurationMin(entity, noticeContainer); validatePriorNoticeStartDay(entity, noticeContainer); validatePriorNoticeDayRange(entity, noticeContainer); + validateMissingPriorDayBookingFields(entity, noticeContainer); + validatePriorNoticeStartTime(entity, noticeContainer); } private static void validatePriorNoticeDurationMin( GtfsBookingRules entity, NoticeContainer noticeContainer) { + // Check if prior_notice_duration_min is set for booking_type SAMEDAY + if (entity.bookingType() == GtfsBookingType.SAMEDAY && !entity.hasPriorNoticeDurationMin()) { + noticeContainer.addValidationNotice(new MissingPriorNoticeDurationMinNotice(entity)); + } + // Check if prior_notice_duration_min is less than prior_notice_duration_max if (entity.hasPriorNoticeDurationMin() && entity.hasPriorNoticeDurationMax()) { if (entity.priorNoticeDurationMax() < entity.priorNoticeDurationMin()) { - noticeContainer.addValidationNotice( - new InvalidPriorNoticeDurationMinNotice( - entity, entity.priorNoticeDurationMin(), entity.priorNoticeDurationMax())); + noticeContainer.addValidationNotice(new InvalidPriorNoticeDurationMinNotice(entity)); } } } + private static void validateMissingPriorDayBookingFields( + GtfsBookingRules entity, NoticeContainer noticeContainer) { + if (entity.bookingType() == GtfsBookingType.PRIORDAY + && (!entity.hasPriorNoticeLastDay() || !entity.hasPriorNoticeLastTime())) { + noticeContainer.addValidationNotice(new MissingPriorDayBookingFieldValueNotice(entity)); + } + } + + private static void validatePriorNoticeStartTime( + GtfsBookingRules entity, NoticeContainer noticeContainer) { + if (entity.hasPriorNoticeStartTime() && !entity.hasPriorNoticeStartDay()) { + noticeContainer.addValidationNotice(new ForbiddenPriorNoticeStartTimeNotice(entity)); + } + if (!entity.hasPriorNoticeStartTime() && entity.hasPriorNoticeStartDay()) { + noticeContainer.addValidationNotice(new MissingPriorNoticeStartTimeNotice(entity)); + } + } + private static void validatePriorNoticeStartDay( GtfsBookingRules entity, NoticeContainer noticeContainer) { if (entity.hasPriorNoticeDurationMax() && entity.hasPriorNoticeStartDay()) { - noticeContainer.addValidationNotice( - new ForbiddenPriorNoticeStartDayNotice( - entity, entity.priorNoticeStartDay(), entity.priorNoticeDurationMax())); + noticeContainer.addValidationNotice(new ForbiddenPriorNoticeStartDayNotice(entity)); } } @@ -229,12 +251,30 @@ static class InvalidPriorNoticeDurationMinNotice extends ValidationNotice { /** The value of the `prior_notice_duration_max` field. */ private final int priorNoticeDurationMax; - InvalidPriorNoticeDurationMinNotice( - GtfsBookingRules bookingRule, int priorNoticeDurationMin, int priorNoticeDurationMax) { + InvalidPriorNoticeDurationMinNotice(GtfsBookingRules bookingRule) { + this.csvRowNumber = bookingRule.csvRowNumber(); + this.bookingRuleId = bookingRule.bookingRuleId(); + this.priorNoticeDurationMin = bookingRule.priorNoticeDurationMin(); + this.priorNoticeDurationMax = bookingRule.priorNoticeDurationMax(); + } + } + + /** + * `prior_notice_duration_min` value is required for same day `booking_type` in booking_rules.txt. + */ + @GtfsValidationNotice( + severity = SeverityLevel.ERROR, + files = @FileRefs(GtfsBookingRulesSchema.class)) + static class MissingPriorNoticeDurationMinNotice extends ValidationNotice { + /** The row number of the faulty record. */ + private final int csvRowNumber; + + /** The `booking_rules.booking_rule_id` of the faulty record. */ + private final String bookingRuleId; + + MissingPriorNoticeDurationMinNotice(GtfsBookingRules bookingRule) { this.csvRowNumber = bookingRule.csvRowNumber(); this.bookingRuleId = bookingRule.bookingRuleId(); - this.priorNoticeDurationMin = priorNoticeDurationMin; - this.priorNoticeDurationMax = priorNoticeDurationMax; } } @@ -255,12 +295,11 @@ static class ForbiddenPriorNoticeStartDayNotice extends ValidationNotice { /** The value of the `prior_notice_duration_max` field. */ private final int priorNoticeDurationMax; - ForbiddenPriorNoticeStartDayNotice( - GtfsBookingRules bookingRule, int priorNoticeStartDay, int priorNoticeDurationMax) { + ForbiddenPriorNoticeStartDayNotice(GtfsBookingRules bookingRule) { this.csvRowNumber = bookingRule.csvRowNumber(); this.bookingRuleId = bookingRule.bookingRuleId(); - this.priorNoticeStartDay = priorNoticeStartDay; - this.priorNoticeDurationMax = priorNoticeDurationMax; + this.priorNoticeStartDay = bookingRule.priorNoticeStartDay(); + this.priorNoticeDurationMax = bookingRule.priorNoticeDurationMax(); } } @@ -293,4 +332,72 @@ static class PriorNoticeLastDayAfterStartDayNotice extends ValidationNotice { this.priorNoticeStartDay = bookingRule.priorNoticeStartDay(); } } + + /** + * `prior_notice_last_day` and `prior_notice_last_time` values are required for prior day + * `booking_type` in booking_rules.txt. + */ + @GtfsValidationNotice( + severity = SeverityLevel.ERROR, + files = @FileRefs(GtfsBookingRulesSchema.class)) + static class MissingPriorDayBookingFieldValueNotice extends ValidationNotice { + /** The row number of the faulty record. */ + private final int csvRowNumber; + + /** The `booking_rules.booking_rule_id` of the faulty record. */ + private final String bookingRuleId; + + MissingPriorDayBookingFieldValueNotice(GtfsBookingRules bookingRule) { + this.csvRowNumber = bookingRule.csvRowNumber(); + this.bookingRuleId = bookingRule.bookingRuleId(); + } + } + + /** + * `prior_notice_start_time` value is forbidden when `prior_notice_start_day` value is not set in + * booking_rules.txt. + */ + @GtfsValidationNotice( + severity = SeverityLevel.ERROR, + files = @FileRefs(GtfsBookingRulesSchema.class)) + static class ForbiddenPriorNoticeStartTimeNotice extends ValidationNotice { + /** The row number of the faulty record. */ + private final int csvRowNumber; + + /** The `booking_rules.booking_rule_id` of the faulty record. */ + private final String bookingRuleId; + + /** The value of the `prior_notice_start_time` field. */ + private final GtfsTime priorNoticeStartTime; + + ForbiddenPriorNoticeStartTimeNotice(GtfsBookingRules bookingRule) { + this.csvRowNumber = bookingRule.csvRowNumber(); + this.bookingRuleId = bookingRule.bookingRuleId(); + this.priorNoticeStartTime = bookingRule.priorNoticeStartTime(); + } + } + + /** + * `prior_notice_start_time` value is required when `prior_notice_start_day` value is set in + * booking_rules.txt. + */ + @GtfsValidationNotice( + severity = SeverityLevel.ERROR, + files = @FileRefs(GtfsBookingRulesSchema.class)) + static class MissingPriorNoticeStartTimeNotice extends ValidationNotice { + /** The row number of the faulty record. */ + private final int csvRowNumber; + + /** The `booking_rules.booking_rule_id` of the faulty record. */ + private final String bookingRuleId; + + /** The value of the `prior_notice_start_day` field. */ + private final int priorNoticeStartDay; + + MissingPriorNoticeStartTimeNotice(GtfsBookingRules bookingRule) { + this.csvRowNumber = bookingRule.csvRowNumber(); + this.bookingRuleId = bookingRule.bookingRuleId(); + this.priorNoticeStartDay = bookingRule.priorNoticeStartDay(); + } + } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidator.java index 72538e1473..75f80db03d 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidator.java @@ -51,14 +51,16 @@ public void validate(GtfsStopTime stopTime, NoticeContainer noticeContainer) { noticeContainer.addValidationNotice( new MissingRequiredFieldNotice( GtfsStopTime.FILENAME, stopTime.csvRowNumber(), GtfsStopTime.STOP_ID_FIELD_NAME)); - } else if (presenceCount > 1) { - // More than one geography ID is present, but only one is allowed - noticeContainer.addValidationNotice( - new ForbiddenGeographyIdNotice( - stopTime.csvRowNumber(), - stopTime.stopId(), - stopTime.locationGroupId(), - stopTime.locationId())); } + // TODO: Put this back once we are ready to publish this notice. + // else if (presenceCount > 1) { + // // More than one geography ID is present, but only one is allowed + // noticeContainer.addValidationNotice( + // new ForbiddenGeographyIdNotice( + // stopTime.csvRowNumber(), + // stopTime.stopId(), + // stopTime.locationGroupId(), + // stopTime.locationId())); + // } } } diff --git a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TranslationFieldAndReferenceValidator.java b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TranslationFieldAndReferenceValidator.java index 45dcc2aabd..2073577482 100644 --- a/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TranslationFieldAndReferenceValidator.java +++ b/main/src/main/java/org/mobilitydata/gtfsvalidator/validator/TranslationFieldAndReferenceValidator.java @@ -27,11 +27,7 @@ import org.mobilitydata.gtfsvalidator.notice.MissingRequiredFieldNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; -import org.mobilitydata.gtfsvalidator.table.GtfsFeedContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTranslation; -import org.mobilitydata.gtfsvalidator.table.GtfsTranslationSchema; -import org.mobilitydata.gtfsvalidator.table.GtfsTranslationTableContainer; +import org.mobilitydata.gtfsvalidator.table.*; /** * Validates that translations are provided in accordance with GTFS Specification. @@ -125,12 +121,17 @@ private void validateTranslation(GtfsTranslation translation, NoticeContainer no translation, GtfsTranslation.RECORD_SUB_ID_FIELD_NAME, translation.recordSubId())); } } - Optional> parentTable = + Optional> parentTable = feedContainer.getTableForFilename(translation.tableName() + ".txt"); if (parentTable.isEmpty() || parentTable.get().isMissingFile()) { noticeContainer.addValidationNotice(new TranslationUnknownTableNameNotice(translation)); } else if (!translation.hasFieldValue()) { - validateReferenceIntegrity(translation, parentTable.get(), noticeContainer); + if (parentTable.isPresent() && parentTable.get() instanceof GtfsTableContainer) { + validateReferenceIntegrity( + translation, (GtfsTableContainer) parentTable.get(), noticeContainer); + } else { + // TODO check for JSON Tables here + } } } @@ -140,7 +141,7 @@ private void validateTranslation(GtfsTranslation translation, NoticeContainer no */ private void validateReferenceIntegrity( GtfsTranslation translation, - GtfsTableContainer parentTable, + GtfsTableContainer parentTable, NoticeContainer noticeContainer) { ImmutableList keyColumnNames = parentTable.getKeyColumnNames(); if (isMissingOrUnexpectedField( diff --git a/main/src/main/resources/report.html b/main/src/main/resources/report.html index 48430b4da4..1db8f42bed 100644 --- a/main/src/main/resources/report.html +++ b/main/src/main/resources/report.html @@ -221,6 +221,14 @@ align-content: center; flex-wrap: wrap; } + + .warning-display { + padding: 10px; + border: solid 2px orange; + border-radius: 11px; + width: fit-content; + font-weight: bold; + } @@ -239,6 +247,11 @@

GTFS Schedule Validation Report

Use this report alongside our documentation.

+

+ ⚠ This feed contains GTFS Flex features. Please note that GTFS Flex validation support is still in development. + You can manually review all the validation rules for Flex data here. +

+

A new version of the Canonical GTFS Schedule validator is available! Please update to get the latest/best validation results.

diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java index 61ec9b616b..254b2bc3f5 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/report/model/FeedMetadataTest.java @@ -35,9 +35,9 @@ public class FeedMetadataTest { File rootDir; NoticeContainer noticeContainer = new NoticeContainer(); - private GtfsTableContainer tripContainer; - private GtfsTableContainer calendarTable; - private GtfsTableContainer calendarDateTable; + private GtfsTableContainer tripContainer; + private GtfsTableContainer calendarTable; + private GtfsTableContainer calendarDateTable; private FeedMetadata feedMetadata = new FeedMetadata(); private void createDataFile(String filename, String content) throws IOException { @@ -136,7 +136,7 @@ public void testLoadServiceWindow() { private void validateSpecFeature( String specFeature, Boolean expectedValue, - ImmutableList>> tableDescriptors) + ImmutableList>> tableDescriptors) throws IOException, InterruptedException { feedLoaderMock = new GtfsFeedLoader(tableDescriptors); try (GtfsInput gtfsInput = GtfsInput.createFromPath(rootDir.toPath(), noticeContainer)) { diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoaderTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoaderTest.java new file mode 100644 index 0000000000..17d81eb2e9 --- /dev/null +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/table/GeojsonFileLoaderTest.java @@ -0,0 +1,115 @@ +/* + * Copyright 2024 MobilityData + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.mobilitydata.gtfsvalidator.table; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.JUnit4; +import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; + +/** Runs GeojsonFileLoader on test json data. */ +@RunWith(JUnit4.class) +public class GeojsonFileLoaderTest { + + static String validGeojsonData; + + @BeforeClass + public static void setUpBeforeClass() { + // To make the json text clearer, use single quotes and replace them by double quotes before + // using + validGeojsonData = + String.join( + "\n", + "{", + " 'type': 'FeatureCollection',", + " 'features': [", + " {", + " 'id': 'id1',", + " 'type': 'Feature',", + " 'geometry': {", + " 'type': 'Point',", + " 'coordinates': [", + " [102.0, 0.0],", + " [103.0, 1.0],", + " [104.0, 0.0],", + " [105.0, 1.0]", + " ]", + " },", + " 'properties': {}", + " },", + " {", + " 'type': 'Feature',", + " 'id': 'id2',", + " 'geometry': {", + " 'type': 'Polygon',", + " 'coordinates': [", + " [", + " [100.0, 0.0],", + " [101.0, 0.0],", + " [101.0, 1.0],", + " [100.0, 1.0],", + " [100.0, 0.0]", + " ]", + " ]", + " },", + " 'properties': {}", + " }", + " ]", + "}"); + + validGeojsonData = validGeojsonData.replace("'", "\""); + } + + @Test + public void testGtfsGeojsonFileLoader() /*throws ValidatorLoaderException*/ { + + var container = createLoader(validGeojsonData); + var geojsonContainer = (GtfsGeojsonFeaturesContainer) container; + assertNotNull(container); + assertEquals( + "Test geojson file is not parsable", + container.getTableStatus(), + TableStatus.PARSABLE_HEADERS_AND_ROWS); + assertEquals(2, container.entityCount()); + assertEquals("id1", geojsonContainer.getEntities().get(0).featureId()); + assertEquals("id2", geojsonContainer.getEntities().get(1).featureId()); + } + + @Test + public void testBrokenJson() { + var container = createLoader("This is a broken json"); + assertEquals( + "Parsing the Geojson file should fail, returning an empty list of entities", + 0, + container.entityCount()); + } + + private GtfsEntityContainer createLoader(String jsonData) { + GeojsonFileLoader loader = new GeojsonFileLoader(); + var fileDescriptor = new GtfsGeojsonFileDescriptor(); + NoticeContainer noticeContainer = new NoticeContainer(); + InputStream inputStream = new ByteArrayInputStream(jsonData.getBytes(StandardCharsets.UTF_8)); + return loader.load(fileDescriptor, null, inputStream, noticeContainer); + } +} diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/BookingRulesEntityValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/BookingRulesEntityValidatorTest.java index 4aa5dce5c1..017cbe8768 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/BookingRulesEntityValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/BookingRulesEntityValidatorTest.java @@ -62,6 +62,7 @@ public void scheduledBookingShouldNotGenerateNotice() { GtfsBookingRules bookingRule = new GtfsBookingRules.Builder() .setCsvRowNumber(1) + .setPriorNoticeDurationMin(1) .setBookingRuleId("rule-3") .setBookingType(GtfsBookingType.SAMEDAY) .build(); @@ -76,7 +77,7 @@ public void realTimeBookingWithMultipleForbiddenFieldsShouldGenerateNotice() { .setCsvRowNumber(1) .setBookingRuleId("rule-4") .setBookingType(REALTIME) - .setPriorNoticeDurationMax(60) // Forbidden field + .setPriorNoticeStartDay(5) // Forbidden field .setPriorNoticeStartTime(GtfsTime.fromSecondsSinceMidnight(2)) // Forbidden field .setPriorNoticeServiceId("service-1") // Forbidden field .build(); @@ -86,7 +87,7 @@ public void realTimeBookingWithMultipleForbiddenFieldsShouldGenerateNotice() { new ForbiddenRealTimeBookingFieldValueNotice( bookingRule, List.of( - GtfsBookingRules.PRIOR_NOTICE_DURATION_MAX_FIELD_NAME, + GtfsBookingRules.PRIOR_NOTICE_START_DAY_FIELD_NAME, GtfsBookingRules.PRIOR_NOTICE_START_TIME_FIELD_NAME, GtfsBookingRules.PRIOR_NOTICE_SERVICE_ID_FIELD_NAME))); } @@ -99,7 +100,7 @@ public void sameDayBookingWithForbiddenFieldsShouldGenerateNotice() { .setBookingRuleId("rule-5") .setBookingType(GtfsBookingType.SAMEDAY) .setPriorNoticeLastDay(2) // Forbidden field - .setPriorNoticeStartTime(GtfsTime.fromSecondsSinceMidnight(5000)) + .setPriorNoticeDurationMin(1) .build(); assertThat(generateNotices(bookingRule)) @@ -113,6 +114,7 @@ public void sameDayBookingWithoutForbiddenFieldsShouldNotGenerateNotice() { GtfsBookingRules bookingRule = new GtfsBookingRules.Builder() .setCsvRowNumber(1) + .setPriorNoticeDurationMin(1) .setBookingRuleId("rule-6") .setBookingType(GtfsBookingType.SAMEDAY) .build(); @@ -127,6 +129,8 @@ public void priorDayBookingWithForbiddenFieldsShouldGenerateNotice() { .setCsvRowNumber(1) .setBookingRuleId("rule-7") .setBookingType(GtfsBookingType.PRIORDAY) + .setPriorNoticeLastDay(1) + .setPriorNoticeLastTime(GtfsTime.fromSecondsSinceMidnight(5000)) .setPriorNoticeDurationMin(30) // Forbidden field .setPriorNoticeDurationMax(60) // Forbidden field .build(); @@ -153,8 +157,7 @@ public void invalidPriorNoticeDurationMinShouldGenerateNotice() { assertThat(generateNotices(bookingRule)) .containsExactly( - new BookingRulesEntityValidator.InvalidPriorNoticeDurationMinNotice( - bookingRule, 60, 30)); + new BookingRulesEntityValidator.InvalidPriorNoticeDurationMinNotice(bookingRule)); } @Test @@ -162,15 +165,17 @@ public void forbiddenPriorNoticeStartDayShouldGenerateNotice() { GtfsBookingRules bookingRule = new GtfsBookingRules.Builder() .setCsvRowNumber(1) + .setPriorNoticeDurationMin(1) .setBookingRuleId("rule-9") .setBookingType(GtfsBookingType.SAMEDAY) + .setPriorNoticeStartTime(GtfsTime.fromSecondsSinceMidnight(5000)) .setPriorNoticeDurationMax(30) // Duration max is set .setPriorNoticeStartDay(5) // Forbidden when duration max is set .build(); assertThat(generateNotices(bookingRule)) .containsExactly( - new BookingRulesEntityValidator.ForbiddenPriorNoticeStartDayNotice(bookingRule, 5, 30)); + new BookingRulesEntityValidator.ForbiddenPriorNoticeStartDayNotice(bookingRule)); } @Test @@ -185,4 +190,96 @@ public void priorNoticeLastDayAfterStartDayShouldGenerateNotice() { assertThat(generateNotices(bookingRule)) .contains(new PriorNoticeLastDayAfterStartDayNotice(bookingRule)); } + + @Test + public void missingPriorNoticeDurationMinShouldGenerateNotice() { + GtfsBookingRules bookingRule = + new GtfsBookingRules.Builder() + .setCsvRowNumber(1) + .setBookingRuleId("rule-10") + .setBookingType(GtfsBookingType.SAMEDAY) // SAMEDAY booking type + .build(); // No prior_notice_duration_min set + + assertThat(generateNotices(bookingRule)) + .containsExactly( + new BookingRulesEntityValidator.MissingPriorNoticeDurationMinNotice(bookingRule)); + } + + @Test + public void missingPriorDayBookingFieldValueShouldGenerateNotice() { + // Case 1: Missing both prior_notice_last_day and prior_notice_last_time + GtfsBookingRules bookingRule = + new GtfsBookingRules.Builder() + .setCsvRowNumber(1) + .setBookingRuleId("rule-11") + .setBookingType(GtfsBookingType.PRIORDAY) // PRIORDAY booking type + .build(); // No prior_notice_last_day or prior_notice_last_time set + + assertThat(generateNotices(bookingRule)) + .containsExactly( + new BookingRulesEntityValidator.MissingPriorDayBookingFieldValueNotice(bookingRule)); + + // Case 2: Missing prior_notice_last_time only + GtfsBookingRules bookingRuleMissingTime = + new GtfsBookingRules.Builder() + .setCsvRowNumber(2) + .setBookingRuleId("rule-12") + .setBookingType(GtfsBookingType.PRIORDAY) // PRIORDAY booking type + .setPriorNoticeLastDay(2) // Setting prior_notice_last_day + .build(); // No prior_notice_last_time set + + assertThat(generateNotices(bookingRuleMissingTime)) + .containsExactly( + new BookingRulesEntityValidator.MissingPriorDayBookingFieldValueNotice( + bookingRuleMissingTime)); + + // Case 3: Missing prior_notice_last_day only + GtfsBookingRules bookingRuleMissingDay = + new GtfsBookingRules.Builder() + .setCsvRowNumber(3) + .setBookingRuleId("rule-13") + .setBookingType(GtfsBookingType.PRIORDAY) // PRIORDAY booking type + .setPriorNoticeLastTime( + GtfsTime.fromSecondsSinceMidnight(5000)) // Setting prior_notice_last_time + .build(); // No prior_notice_last_day set + + assertThat(generateNotices(bookingRuleMissingDay)) + .containsExactly( + new BookingRulesEntityValidator.MissingPriorDayBookingFieldValueNotice( + bookingRuleMissingDay)); + } + + @Test + public void forbiddenPriorNoticeStartTimeShouldGenerateNotice() { + GtfsBookingRules bookingRule = + new GtfsBookingRules.Builder() + .setCsvRowNumber(2) + .setPriorNoticeLastTime(GtfsTime.fromSecondsSinceMidnight(5000)) + .setPriorNoticeLastDay(2) + .setBookingRuleId("rule-14") + .setBookingType(GtfsBookingType.PRIORDAY) + .setPriorNoticeStartTime(GtfsTime.fromSecondsSinceMidnight(5000)) // Set start time + .build(); // No prior_notice_start_day set + + assertThat(generateNotices(bookingRule)) + .containsExactly( + new BookingRulesEntityValidator.ForbiddenPriorNoticeStartTimeNotice(bookingRule)); + } + + @Test + public void missingPriorNoticeStartTimeShouldGenerateNotice() { + GtfsBookingRules bookingRule = + new GtfsBookingRules.Builder() + .setCsvRowNumber(2) + .setPriorNoticeLastTime(GtfsTime.fromSecondsSinceMidnight(5000)) + .setPriorNoticeLastDay(2) + .setBookingRuleId("rule-14") + .setBookingType(GtfsBookingType.PRIORDAY) + .setPriorNoticeStartDay(3) // Set start day + .build(); // No prior_notice_start_time set + + assertThat(generateNotices(bookingRule)) + .containsExactly( + new BookingRulesEntityValidator.MissingPriorNoticeStartTimeNotice(bookingRule)); + } } diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/DateTripsValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/DateTripsValidatorTest.java index 1a8e367351..3b42d555d6 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/DateTripsValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/DateTripsValidatorTest.java @@ -16,7 +16,7 @@ import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.notice.ValidationNotice; import org.mobilitydata.gtfsvalidator.table.*; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; +import org.mobilitydata.gtfsvalidator.table.TableStatus; import org.mobilitydata.gtfsvalidator.type.GtfsDate; import org.mobilitydata.gtfsvalidator.util.CalendarUtilTest; diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/ExpiredCalendarValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/ExpiredCalendarValidatorTest.java index eac18ce095..2ad7a1fe0c 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/ExpiredCalendarValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/ExpiredCalendarValidatorTest.java @@ -30,7 +30,7 @@ import org.mobilitydata.gtfsvalidator.input.DateForValidation; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.table.*; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; +import org.mobilitydata.gtfsvalidator.table.TableStatus; import org.mobilitydata.gtfsvalidator.type.GtfsDate; @RunWith(JUnit4.class) diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MatchingFeedAndAgencyLangValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MatchingFeedAndAgencyLangValidatorTest.java index bfeef3dc7c..6bdd7750c9 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MatchingFeedAndAgencyLangValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MatchingFeedAndAgencyLangValidatorTest.java @@ -31,7 +31,7 @@ import org.mobilitydata.gtfsvalidator.table.GtfsAgencyTableContainer; import org.mobilitydata.gtfsvalidator.table.GtfsFeedInfo; import org.mobilitydata.gtfsvalidator.table.GtfsFeedInfoTableContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; +import org.mobilitydata.gtfsvalidator.table.TableStatus; import org.mobilitydata.gtfsvalidator.validator.MatchingFeedAndAgencyLangValidator.FeedInfoLangAndAgencyLangMismatchNotice; @RunWith(JUnit4.class) diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingCalendarAndCalendarDateValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingCalendarAndCalendarDateValidatorTest.java index afb6e585c1..f9621bb995 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingCalendarAndCalendarDateValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingCalendarAndCalendarDateValidatorTest.java @@ -31,7 +31,7 @@ import org.mobilitydata.gtfsvalidator.table.GtfsCalendarDate; import org.mobilitydata.gtfsvalidator.table.GtfsCalendarDateTableContainer; import org.mobilitydata.gtfsvalidator.table.GtfsCalendarTableContainer; -import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus; +import org.mobilitydata.gtfsvalidator.table.TableStatus; import org.mobilitydata.gtfsvalidator.type.GtfsDate; import org.mobilitydata.gtfsvalidator.util.CalendarUtilTest; import org.mobilitydata.gtfsvalidator.validator.MissingCalendarAndCalendarDateValidator.MissingCalendarAndCalendarDateFilesNotice; diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingFeedInfoValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingFeedInfoValidatorTest.java index 0dd0553dff..43b109f93a 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingFeedInfoValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingFeedInfoValidatorTest.java @@ -25,9 +25,8 @@ private static List generateNotices( public void missingFeedInfoTranslationTableNotPresent() { assertThat( generateNotices( - GtfsFeedInfoTableContainer.forStatus(GtfsTableContainer.TableStatus.MISSING_FILE), - GtfsTranslationTableContainer.forStatus( - GtfsTableContainer.TableStatus.MISSING_FILE))) + GtfsFeedInfoTableContainer.forStatus(TableStatus.MISSING_FILE), + GtfsTranslationTableContainer.forStatus(TableStatus.MISSING_FILE))) .containsExactly(new MissingRecommendedFileNotice(GtfsFeedInfo.FILENAME)); } @@ -35,9 +34,8 @@ public void missingFeedInfoTranslationTableNotPresent() { public void missingFeedInfoWhenTranslationTableIsPresent() { assertThat( generateNotices( - GtfsFeedInfoTableContainer.forStatus(GtfsTableContainer.TableStatus.MISSING_FILE), - GtfsTranslationTableContainer.forStatus( - GtfsTableContainer.TableStatus.PARSABLE_HEADERS_AND_ROWS))) + GtfsFeedInfoTableContainer.forStatus(TableStatus.MISSING_FILE), + GtfsTranslationTableContainer.forStatus(TableStatus.PARSABLE_HEADERS_AND_ROWS))) .contains(new MissingRequiredFileNotice(GtfsFeedInfo.FILENAME)); } @@ -45,10 +43,8 @@ public void missingFeedInfoWhenTranslationTableIsPresent() { public void feedInfoPresentShouldGenerateNoNotice() { assertThat( generateNotices( - GtfsFeedInfoTableContainer.forStatus( - GtfsTableContainer.TableStatus.PARSABLE_HEADERS_AND_ROWS), - GtfsTranslationTableContainer.forStatus( - GtfsTableContainer.TableStatus.PARSABLE_HEADERS_AND_ROWS))) + GtfsFeedInfoTableContainer.forStatus(TableStatus.PARSABLE_HEADERS_AND_ROWS), + GtfsTranslationTableContainer.forStatus(TableStatus.PARSABLE_HEADERS_AND_ROWS))) .isEmpty(); } } diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NetworkIdConsistencyValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NetworkIdConsistencyValidatorTest.java index 0dfe65a6d2..baf0358156 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NetworkIdConsistencyValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NetworkIdConsistencyValidatorTest.java @@ -28,10 +28,9 @@ public void setup() { noticeContainer); routeNetworkTableContainer = new GtfsRouteNetworkTableContainer( - new GtfsRouteNetworkTableDescriptor(), GtfsTableContainer.TableStatus.MISSING_FILE); + new GtfsRouteNetworkTableDescriptor(), TableStatus.MISSING_FILE); networkTableContainer = - new GtfsNetworkTableContainer( - new GtfsNetworkTableDescriptor(), GtfsTableContainer.TableStatus.MISSING_FILE); + new GtfsNetworkTableContainer(new GtfsNetworkTableDescriptor(), TableStatus.MISSING_FILE); } @Test diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java index 7b8ee11c97..d46efe7589 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/NoticeFieldsTest.java @@ -62,6 +62,7 @@ public void testNoticeClassFieldNames() { "arrivalTime2", "attributionId", "blockId", + "bookingRuleId", "charIndex", "childFieldName", "childFilename", @@ -93,10 +94,13 @@ public void testNoticeClassFieldNames() { "fieldName", "fieldName1", "fieldName2", + "fieldNames", "fieldType", "fieldValue", "fieldValue1", "fieldValue2", + "fileNameA", + "fileNameB", "filename", "firstIndex", "geoDistanceToShape", @@ -105,9 +109,12 @@ public void testNoticeClassFieldNames() { "headerCount", "index", "intersection", + "isBidirectional", "latFieldName", "latFieldValue", "lineIndex", + "locationGroupId", + "locationId", "locationType", "locationTypeName", "locationTypeValue", @@ -117,6 +124,8 @@ public void testNoticeClassFieldNames() { "match1", "match2", "matchCount", + "maxShapeDistanceTraveled", + "maxTripDistanceTraveled", "message", "newCsvRowNumber", "oldCsvRowNumber", @@ -128,11 +137,17 @@ public void testNoticeClassFieldNames() { "parentStopName", "parsedContent", "pathwayId", + "pathwayMode", "prevCsvRowNumber", "prevEndTime", "prevShapeDistTraveled", "prevShapePtSequence", "prevStopSequence", + "priorNoticeDurationMax", + "priorNoticeDurationMin", + "priorNoticeLastDay", + "priorNoticeStartDay", + "priorNoticeStartTime", "recordId", "recordSubId", "routeColor", @@ -153,8 +168,8 @@ public void testNoticeClassFieldNames() { "serviceId", "serviceIdA", "serviceIdB", - "serviceWindowStartDate", "serviceWindowEndDate", + "serviceWindowStartDate", "shapeDistTraveled", "shapeId", "shapePtSequence", @@ -191,21 +206,7 @@ public void testNoticeClassFieldNames() { "tripIdB", "tripIdFieldName", "validator", - "value", - "maxShapeDistanceTraveled", - "maxTripDistanceTraveled", - "fileNameA", - "fileNameB", - "pathwayMode", - "isBidirectional", - "locationGroupId", - "locationId", - "bookingRuleId", - "fieldNames", - "priorNoticeDurationMin", - "priorNoticeDurationMax", - "priorNoticeStartDay", - "priorNoticeLastDay"); + "value"); } private static List discoverValidationNoticeFieldNames() { diff --git a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidatorTest.java b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidatorTest.java index d03d2be952..54d4b21249 100644 --- a/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidatorTest.java +++ b/main/src/test/java/org/mobilitydata/gtfsvalidator/validator/StopTimesGeographyIdPresenceValidatorTest.java @@ -51,6 +51,13 @@ public void OneGeographyIdShouldGenerateNothing() { } @Test + public void NoGeographyIdShouldGenerateNotice() { + assertThat(validationNoticesFor(new GtfsStopTime.Builder().setCsvRowNumber(2).build())) + .containsExactly(new MissingRequiredFieldNotice("stop_times.txt", 2, "stop_id")); + } + + // TODO: Put back when this notice is ready to be published. + // @Test public void MultipleGeographyIdShouldGenerateNotice() { assertThat( validationNoticesFor( diff --git a/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJson.java b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJson.java new file mode 100644 index 0000000000..d9ea7a108a --- /dev/null +++ b/model/src/main/java/org/mobilitydata/gtfsvalidator/annotation/GtfsJson.java @@ -0,0 +1,39 @@ +/* + * Copyright 2024 MobilityData + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.mobilitydata.gtfsvalidator.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Annotates an interface that defines schema for a single GTFS JSON, such as "locations.geojson". + * + *

Example. + * + *

+ *   {@literal @}GtfsJson("locations.geojson")
+ *   public interface GtfsLocationsSchema extends GtfsEntity {
+ *   }
+ * 
+ */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface GtfsJson { + String value(); +} diff --git a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableContainerGenerator.java b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableContainerGenerator.java index f215ecdf0a..2dc5735698 100644 --- a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableContainerGenerator.java +++ b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableContainerGenerator.java @@ -32,6 +32,7 @@ import org.mobilitydata.gtfsvalidator.parsing.CsvHeader; import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; +import org.mobilitydata.gtfsvalidator.table.TableStatus; /** * Generates code for a container for a loaded GTFS table. @@ -64,7 +65,12 @@ public TypeSpec generateGtfsContainerClass() { TypeSpec.Builder typeSpec = TypeSpec.classBuilder(classNames.tableContainerSimpleName()) .superclass( - ParameterizedTypeName.get(ClassName.get(GtfsTableContainer.class), gtfsEntityType)) + ParameterizedTypeName.get( + ClassName.get(GtfsTableContainer.class), + classNames.entityImplementationTypeName(), + ParameterizedTypeName.get( + ClassName.get(GtfsTableDescriptor.class), + classNames.entityImplementationTypeName()))) .addAnnotation(Generated.class) .addModifiers(Modifier.PUBLIC, Modifier.FINAL); @@ -126,7 +132,7 @@ private MethodSpec generateConstructorWithStatus() { return MethodSpec.constructorBuilder() .addModifiers(Modifier.PUBLIC) .addParameter(tableDescriptorType, "descriptor") - .addParameter(GtfsTableContainer.TableStatus.class, "tableStatus") + .addParameter(TableStatus.class, "tableStatus") .addStatement("super(descriptor, tableStatus, $T.EMPTY)", CsvHeader.class) .addStatement("this.entities = new $T<>()", ArrayList.class) .build(); @@ -182,7 +188,7 @@ private MethodSpec generateForStatusMethod() { "Creates a table with the given TableStatus. This method is intended to be" + " used in tests.") .addModifiers(Modifier.PUBLIC, Modifier.STATIC) - .addParameter(GtfsTableContainer.TableStatus.class, "tableStatus") + .addParameter(TableStatus.class, "tableStatus") .addStatement( "return new $T(new $T(), tableStatus)", tableContainerTypeName, diff --git a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java index 514bbe98c9..62b01ac216 100644 --- a/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java +++ b/processor/src/main/java/org/mobilitydata/gtfsvalidator/processor/TableDescriptorGenerator.java @@ -47,6 +47,7 @@ import org.mobilitydata.gtfsvalidator.table.GtfsFieldLoader; import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer; import org.mobilitydata.gtfsvalidator.table.GtfsTableDescriptor; +import org.mobilitydata.gtfsvalidator.table.TableStatus; /** * Generates code for a GtfsTableDescriptor subclass for a specific GTFS table. @@ -149,7 +150,7 @@ private MethodSpec generateCreateContainerForInvalidStatusMethod() { return MethodSpec.methodBuilder("createContainerForInvalidStatus") .addAnnotation(Override.class) .addModifiers(Modifier.PUBLIC) - .addParameter(GtfsTableContainer.TableStatus.class, "tableStatus") + .addParameter(TableStatus.class, "tableStatus") .returns(GtfsTableContainer.class) .addStatement("return new $T(this, tableStatus)", classNames.tableContainerTypeName()) .build(); diff --git a/web/client/.gitignore b/web/client/.gitignore index fef3daace4..d907181126 100644 --- a/web/client/.gitignore +++ b/web/client/.gitignore @@ -12,3 +12,4 @@ vite.config.ts.timestamp-* rules.json cypress/screenshots/ cypress/videos/ +/static/RULES.md