Merge branch 'master' into feat/make-staging-environ

.github/workflows/package_installers.yml

@@ -13,6 +13,19 @@ on:
     types: [ prereleased, released ]
   workflow_dispatch:
 
+env:
+  MACOS_APP_NAME: "GTFS Validator"
+  MACOS_NOTARIZATION_NAME: "notarization.zip"
+  MACOS_TARGET_PATH: "app/pkg/build/jpackage/GTFS Validator.app"
+  MACOS_TARGET_DEST_PATH: "app/pkg/build/jpackage"
+  MACOS_CERTIFICATE: ${{ secrets.MACOS_DEVELOPER_ID_APPLICATION_CERTIFICATE_P12_BASE64 }}
+  MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_DEVELOPER_ID_APPLICATION_CERTIFICATE_PASSWORD }}
+  MACOS_CERTIFICATE_NAME: ${{ secrets.MACOS_DEVELOPER_ID_APPLICATION_CERTIFICATE_NAME }}
+  MACOS_CI_KEYCHAIN_PWD: ${{ secrets.MACOS_KEYCHAIN_PASSWORD }}
+  MACOS_NOTARIZATION_APPLE_ID: ${{ secrets.MACOS_NOTARIZATION_USERNAME }}
+  MACOS_NOTARIZATION_TEAM_ID: ${{ secrets.MACOS_NOTARIZATION_TEAM_ID }}
+  MACOS_NOTARIZATION_PWD: ${{ secrets.MACOS_NOTARIZATION_PASSWORD }}
+
 jobs:
   validate_gradle_wrapper:
     runs-on: ubuntu-latest
@@ -25,7 +38,7 @@ jobs:
     name: Build and upload packaged app
     runs-on: ${{ matrix.os }}
     strategy:
-#     Adding fail-fast: false so at least some artifacts gets uploaded if others fail
+      # Adding fail-fast: false so at least some artifacts gets uploaded if others fail
       fail-fast: false
       matrix:
         os: [ macos-latest, windows-latest, ubuntu-latest ]
@@ -47,17 +60,18 @@ jobs:
       # We create a code-signing keychain on MacOS before building and packaging the app, as the
       # app will be signed as part of the jpackage build phase.
       - name: "MacOS - Import Certificate: Developer ID Application"
+        id: codesign
         if: matrix.os == 'macos-latest' && (github.event_name == 'push' || github.event_name == 'release')
-        uses: devbotsxyz/import-signing-certificate@v1
-        with:
-          # This should be a certificate + private key, exported as a p12 file and base64 encoded as
-          # a GitHub secret.  The certificate should be the:
-          # 'Developer ID Application: The International Data Organization For Transport (BF2U75HN4D)'
-          # certificate, locked with the specified passphrase.
-          certificate-data: ${{ secrets.MACOS_DEVELOPER_ID_APPLICATION_CERTIFICATE_P12_BASE64 }}
-          certificate-passphrase: ${{ secrets.MACOS_DEVELOPER_ID_APPLICATION_CERTIFICATE_PASSWORD }}
-          # The resulting keychain will be locked with this separate password.
-          keychain-password: ${{ secrets.MACOS_KEYCHAIN_PASSWORD }}
+        run: |
+          # Turn our base64-encoded certificate back to a regular .p12 file
+          echo $MACOS_CERTIFICATE | base64 --decode > certificate.p12
+
+          # We need to create a new keychain, otherwise using the certificate will prompt with a UI dialog asking for the certificate password, which we can't use in a headless CI environment
+          security create-keychain -p "${MACOS_CI_KEYCHAIN_PWD}" build.keychain
+          security default-keychain -s build.keychain
+          security unlock-keychain -p "${MACOS_CI_KEYCHAIN_PWD}" build.keychain
+          security import certificate.p12 -k build.keychain -P "${MACOS_CERTIFICATE_PWD}" -T /usr/bin/codesign
+          security set-key-partition-list -S apple-tool:,apple:,codesign: -s -k "${MACOS_CI_KEYCHAIN_PWD}" build.keychain
 
       - name: "Package GUI app installer with Gradle"
         uses: gradle/gradle-build-action@v2
@@ -69,24 +83,35 @@ jobs:
       # On MacOS, we now submit the app for "notarization", where Apple will scan the app for
       # malware and other issues.  This step can take a few minutes or more; the action will wait
       # until the report is available.
-      - name: "MacOS - Notarize Release Build"
+      - name: "MacOS - Notarize & Staple Release Build"
         if: matrix.os == 'macos-latest' && (github.event_name == 'push' || github.event_name == 'release')
-        uses: devbotsxyz/xcode-notarize@v1
-        with:
-          product-path: "app/pkg/build/jpackage/GTFS Validator.app"
-          # The Apple developer account used to run notarization.  This account will receive an
-          # email every time notarization is run.
-          appstore-connect-username: ${{ secrets.MACOS_NOTARIZATION_USERNAME }}
-          # The app-specific password configured for the Apple developer account that will be used
-          # for authentication (different from the main password for the dev account).
-          appstore-connect-password: ${{ secrets.MACOS_NOTARIZATION_PASSWORD }}
+        id: notarize_staple_macos_app
+        run: |
+          # Store the notarization credentials so that we can prevent a UI password dialog from blocking the CI
+          
+          echo "Create keychain profile"
+          xcrun notarytool store-credentials "notarytool-profile" --apple-id "${MACOS_NOTARIZATION_APPLE_ID}" --team-id "${MACOS_NOTARIZATION_TEAM_ID}" --password "${MACOS_NOTARIZATION_PWD}"
+          
+          # We can't notarize an app bundle directly, but we need to compress it as an archive.
+          # Therefore, we create a zip file containing our app bundle, so that we can send it to the
+          # notarization service
+          
+          echo "Creating temp notarization archive"
+          ditto -c -k --keepParent "${{ env.MACOS_TARGET_PATH }}" ${{ env.MACOS_NOTARIZATION_NAME }}
+          
+          # Here we send the notarization request to the Apple's Notarization service, waiting for the result.
+          # This typically takes a few seconds inside a CI environment, but it might take more depending on the App
+          # characteristics. Visit the Notarization docs for more information and strategies on how to optimize it if
+          # you're curious
+          
+          echo "Notarize app"
+          xcrun notarytool submit "${{ env.MACOS_NOTARIZATION_NAME }}" --keychain-profile "notarytool-profile" --wait
+          
+          # Finally, we need to "attach the staple" to our executable, which will allow our app to be
+          # validated by macOS even when an internet connection is not available.
+          echo "Attach staple"
+          xcrun stapler staple "${{ env.MACOS_TARGET_PATH }}"
 
-      # Now that we have a notarization report, we attach it to the app binary.
-      - name: "Mac OS - Staple Release Build"
-        if: matrix.os == 'macos-latest' && (github.event_name == 'push' || github.event_name == 'release')
-        uses: devbotsxyz/xcode-staple@v1
-        with:
-          product-path: "app/pkg/build/jpackage/GTFS Validator.app"
 
       # Now that we have a notarized app, we can package it.
       - name: "Mac OS - Package the app"
@@ -97,26 +122,26 @@ jobs:
           appVersion=${appVersion//-SNAPSHOT/}
           jpackage \
            --type dmg \
-           --name 'GTFS Validator' \
+           --name "${{ env.MACOS_APP_NAME }}" \
            --app-version ${appVersion} \
-           --app-image app/pkg/build/jpackage/GTFS\ Validator.app \
-           --dest app/pkg/build/jpackage
+           --app-image "${{ env.MACOS_TARGET_PATH }}" \
+           --dest ${{ env.MACOS_TARGET_DEST_PATH }}
           jpackage \
            --type pkg \
            --name 'GTFS Validator' \
            --app-version ${appVersion} \
-           --app-image app/pkg/build/jpackage/GTFS\ Validator.app \
-           --dest app/pkg/build/jpackage
+           --app-image "${{ env.MACOS_TARGET_PATH }}" \
+           --dest ${{ env.MACOS_TARGET_DEST_PATH }}
 
       - name: "Upload Installer"
         uses: actions/upload-artifact@v3
         with:
           name: Installer - ${{matrix.os}}
           path: |
-            app/pkg/build/jpackage/*.msi
-            app/pkg/build/jpackage/*.dmg
-            app/pkg/build/jpackage/*.pkg
-            app/pkg/build/jpackage/*.deb
+            ${{ env.MACOS_TARGET_DEST_PATH }}/*.msi
+            ${{ env.MACOS_TARGET_DEST_PATH }}/*.dmg
+            ${{ env.MACOS_TARGET_DEST_PATH }}/*.pkg
+            ${{ env.MACOS_TARGET_DEST_PATH }}/*.deb
 
       - name: "Upload assets to release"
         if: github.event_name == 'prerelease' || github.event_name == 'release'

core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsFeedContainer.java

@@ -80,7 +80,10 @@ public Collection<GtfsTableContainer<?>> getTables() {
   public String tableTotalsText() {
     List<String> totalList = new ArrayList<>();
     for (GtfsTableContainer<?> table : tables.values()) {
-      if (table.getTableStatus() == TableStatus.MISSING_FILE && !table.isRequired()) continue;
+      if (table.getTableStatus() == TableStatus.MISSING_FILE
+          && !table.getDescriptor().isRequired()) {
+        continue;
+      }
       totalList.add(
           table.gtfsFilename()
               + "\t"

core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableContainer.java

@@ -33,15 +33,23 @@
  */
 public abstract class GtfsTableContainer<T extends GtfsEntity> {
 
+  private final GtfsTableDescriptor<T> descriptor;
+
   private final TableStatus tableStatus;
 
   private final CsvHeader header;
 
-  public GtfsTableContainer(TableStatus tableStatus, CsvHeader header) {
+  public GtfsTableContainer(
+      GtfsTableDescriptor<T> descriptor, TableStatus tableStatus, CsvHeader header) {
+    this.descriptor = descriptor;
     this.tableStatus = tableStatus;
     this.header = header;
   }
 
+  public GtfsTableDescriptor<T> getDescriptor() {
+    return descriptor;
+  }
+
   public TableStatus getTableStatus() {
     return tableStatus;
   }
@@ -96,24 +104,6 @@ public boolean isMissingFile() {
     return tableStatus == TableStatus.MISSING_FILE;
   }
 
-  /**
-   * Tells if the file is recommended according to GTFS.
-   *
-   * <p>Note that a recommended file may be empty.
-   *
-   * @return true if the file is recommended, false otherwise
-   */
-  public abstract boolean isRecommended();
-
-  /**
-   * Tells if the file is required according to GTFS.
-   *
-   * <p>Note that a required file may be empty.
-   *
-   * @return true if the file is required, false otherwise
-   */
-  public abstract boolean isRequired();
-
   /**
    * Tells if the file was successfully parsed.
    *
@@ -135,7 +125,7 @@ public boolean isParsedSuccessfully() {
       case PARSABLE_HEADERS_AND_ROWS:
         return true;
       case MISSING_FILE:
-        return !isRequired();
+        return !descriptor.isRequired();
       default:
         return false;
     }

core/src/main/java/org/mobilitydata/gtfsvalidator/table/GtfsTableDescriptor.java

@@ -8,6 +8,10 @@
 import org.mobilitydata.gtfsvalidator.parsing.CsvHeader;
 
 public abstract class GtfsTableDescriptor<T extends GtfsEntity> {
+
+  // True if the specified file is required in a feed.
+  private boolean required;
+
   public abstract GtfsTableContainer createContainerForInvalidStatus(
       GtfsTableContainer.TableStatus tableStatus);
 
@@ -24,7 +28,13 @@ public abstract GtfsTableContainer createContainerForHeaderAndEntities(
 
   public abstract boolean isRecommended();
 
-  public abstract boolean isRequired();
+  public boolean isRequired() {
+    return this.required;
+  }
+
+  public void setRequired(boolean required) {
+    this.required = required;
+  }
 
   public abstract Optional<Integer> maxCharsPerColumn();
 

core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableContainer.java

@@ -30,13 +30,14 @@ public class GtfsTestTableContainer extends GtfsTableContainer<GtfsTestEntity> {
 
   private List<GtfsTestEntity> entities;
 
-  private GtfsTestTableContainer(CsvHeader header, List<GtfsTestEntity> entities) {
-    super(TableStatus.PARSABLE_HEADERS_AND_ROWS, header);
+  private GtfsTestTableContainer(
+      GtfsTestTableDescriptor descriptor, CsvHeader header, List<GtfsTestEntity> entities) {
+    super(descriptor, TableStatus.PARSABLE_HEADERS_AND_ROWS, header);
     this.entities = entities;
   }
 
   public GtfsTestTableContainer(TableStatus tableStatus) {
-    super(tableStatus, CsvHeader.EMPTY);
+    super(new GtfsTestTableDescriptor(), tableStatus, CsvHeader.EMPTY);
     this.entities = new ArrayList<>();
   }
 
@@ -50,37 +51,21 @@ public String gtfsFilename() {
     return GtfsTestEntity.FILENAME;
   }
 
-  @Override
-  public boolean isRecommended() {
-    return false;
-  }
-
-  @Override
-  public boolean isRequired() {
-    return true;
-  }
-
   @Override
   public List<GtfsTestEntity> getEntities() {
     return entities;
   }
 
   /** Creates a table with given header and entities */
   public static GtfsTestTableContainer forHeaderAndEntities(
-      CsvHeader header, List<GtfsTestEntity> entities, NoticeContainer noticeContainer) {
-    GtfsTestTableContainer table = new GtfsTestTableContainer(header, entities);
+      GtfsTestTableDescriptor descriptor,
+      CsvHeader header,
+      List<GtfsTestEntity> entities,
+      NoticeContainer noticeContainer) {
+    GtfsTestTableContainer table = new GtfsTestTableContainer(descriptor, header, entities);
     return table;
   }
 
-  /**
-   * Creates a table with given entities and empty header. This method is intended to be used in
-   * tests.
-   */
-  public static GtfsTestTableContainer forEntities(
-      List<GtfsTestEntity> entities, NoticeContainer noticeContainer) {
-    return forHeaderAndEntities(CsvHeader.EMPTY, entities, noticeContainer);
-  }
-
   @Override
   public ImmutableList<String> getKeyColumnNames() {
     return KEY_COLUMN_NAMES;

core/src/test/java/org/mobilitydata/gtfsvalidator/testgtfs/GtfsTestTableDescriptor.java

@@ -21,7 +21,7 @@ public GtfsTableContainer createContainerForInvalidStatus(
   @Override
   public GtfsTableContainer createContainerForHeaderAndEntities(
       CsvHeader header, List<GtfsTestEntity> entities, NoticeContainer noticeContainer) {
-    return GtfsTestTableContainer.forHeaderAndEntities(header, entities, noticeContainer);
+    return GtfsTestTableContainer.forHeaderAndEntities(this, header, entities, noticeContainer);
   }
 
   @Override

main/src/test/java/org/mobilitydata/gtfsvalidator/validator/DateTripsValidatorTest.java

@@ -18,6 +18,7 @@
 import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
 import org.mobilitydata.gtfsvalidator.notice.ValidationNotice;
 import org.mobilitydata.gtfsvalidator.table.*;
+import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus;
 import org.mobilitydata.gtfsvalidator.type.GtfsDate;
 import org.mobilitydata.gtfsvalidator.util.CalendarUtilTest;
 
@@ -85,8 +86,8 @@ private List<ValidationNotice> validateSimpleServiceWindow(
             ImmutableSet.copyOf(DayOfWeek.values()));
     var calendarTable =
         GtfsCalendarTableContainer.forEntities(ImmutableList.of(calendar), noticeContainer);
-    var dateTable = new GtfsCalendarDateTableContainer(GtfsTableContainer.TableStatus.EMPTY_FILE);
-    var frequencyTable = new GtfsFrequencyTableContainer(GtfsTableContainer.TableStatus.EMPTY_FILE);
+    var dateTable = GtfsCalendarDateTableContainer.forStatus(TableStatus.EMPTY_FILE);
+    var frequencyTable = GtfsFrequencyTableContainer.forStatus(TableStatus.EMPTY_FILE);
 
     var tripBlock = createTripBlock(serviceId, 6, "b1");
     var tripContainer = GtfsTripTableContainer.forEntities(tripBlock, noticeContainer);

main/src/test/java/org/mobilitydata/gtfsvalidator/validator/ExpiredCalendarValidatorTest.java

@@ -31,6 +31,7 @@
 import org.mobilitydata.gtfsvalidator.input.CurrentDateTime;
 import org.mobilitydata.gtfsvalidator.notice.NoticeContainer;
 import org.mobilitydata.gtfsvalidator.table.*;
+import org.mobilitydata.gtfsvalidator.table.GtfsTableContainer.TableStatus;
 import org.mobilitydata.gtfsvalidator.type.GtfsDate;
 
 @RunWith(JUnit4.class)
@@ -66,7 +67,7 @@ public void calendarEndDateOneDayAgoShouldGenerateNotice() {
     GtfsCalendarTableContainer calendarTable =
         GtfsCalendarTableContainer.forEntities(calendars, container);
 
-    var dateTable = new GtfsCalendarDateTableContainer(GtfsTableContainer.TableStatus.EMPTY_FILE);
+    var dateTable = GtfsCalendarDateTableContainer.forStatus(TableStatus.EMPTY_FILE);
     new ExpiredCalendarValidator(new CurrentDateTime(TEST_NOW), calendarTable, dateTable)
         .validate(container);
     assertThat(container.getValidationNotices())
@@ -93,7 +94,7 @@ public void calendarEndDateTodayShouldNotGenerateNotice() {
     GtfsCalendarTableContainer calendarTable =
         GtfsCalendarTableContainer.forEntities(calendars, container);
 
-    var dateTable = new GtfsCalendarDateTableContainer(GtfsTableContainer.TableStatus.EMPTY_FILE);
+    var dateTable = GtfsCalendarDateTableContainer.forStatus(TableStatus.EMPTY_FILE);
     new ExpiredCalendarValidator(new CurrentDateTime(TEST_NOW), calendarTable, dateTable)
         .validate(container);
     assertThat(container.getValidationNotices()).isEmpty();
@@ -119,7 +120,7 @@ public void calendarEndDateOneDayFromNowShouldNotGenerateNotice() {
     GtfsCalendarTableContainer calendarTable =
         GtfsCalendarTableContainer.forEntities(calendars, container);
 
-    var dateTable = new GtfsCalendarDateTableContainer(GtfsTableContainer.TableStatus.EMPTY_FILE);
+    var dateTable = GtfsCalendarDateTableContainer.forStatus(TableStatus.EMPTY_FILE);
     new ExpiredCalendarValidator(new CurrentDateTime(TEST_NOW), calendarTable, dateTable)
         .validate(container);
     assertThat(container.getValidationNotices()).isEmpty();
@@ -211,7 +212,7 @@ public void calendarWithNoDaysShouldNotGenerateNotice() {
 
     GtfsCalendarTableContainer calendarTable =
         GtfsCalendarTableContainer.forEntities(calendars, container);
-    var dateTable = new GtfsCalendarDateTableContainer(GtfsTableContainer.TableStatus.EMPTY_FILE);
+    var dateTable = GtfsCalendarDateTableContainer.forStatus(TableStatus.EMPTY_FILE);
     new ExpiredCalendarValidator(new CurrentDateTime(TEST_NOW), calendarTable, dateTable)
         .validate(container);
     assertThat(container.getValidationNotices()).isEmpty();

main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MatchingFeedAndAgencyLangValidatorTest.java

@@ -77,8 +77,8 @@ public static GtfsFeedInfo createFeedInfo(int csvRowNumber, @Nullable Locale fee
   public void noFeedInfoShouldNotGenerateNotice() {
     assertThat(
             generateNotices(
-                new GtfsAgencyTableContainer(TableStatus.EMPTY_FILE),
-                new GtfsFeedInfoTableContainer(TableStatus.EMPTY_FILE)))
+                GtfsAgencyTableContainer.forStatus(TableStatus.EMPTY_FILE),
+                GtfsFeedInfoTableContainer.forStatus(TableStatus.EMPTY_FILE)))
         .isEmpty();
   }
 

main/src/test/java/org/mobilitydata/gtfsvalidator/validator/MissingCalendarAndCalendarDateValidatorTest.java

@@ -118,8 +118,8 @@ public void calendarDateOnlyProvidedShouldNotGenerateNotice() {
   public void bothMissingFilesShouldGenerateNotice() {
     assertThat(
             generateNotices(
-                new GtfsCalendarTableContainer(TableStatus.MISSING_FILE),
-                new GtfsCalendarDateTableContainer(TableStatus.MISSING_FILE)))
+                GtfsCalendarTableContainer.forStatus(TableStatus.MISSING_FILE),
+                GtfsCalendarDateTableContainer.forStatus(TableStatus.MISSING_FILE)))
         .containsExactly(new MissingCalendarAndCalendarDateFilesNotice());
   }
 }