From 4085c77bc1b7f9e1978f5343f385af082eeacf56 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 12 Jan 2024 12:06:02 -0500 Subject: [PATCH 01/13] refactor storage helper to send the user-agent to producers --- .../gtfsvalidator/input/GtfsInput.java | 41 +----------- .../gtfsvalidator/util/HttpGetUtil.java | 67 +++++++++++++++++++ .../web/service/util/StorageHelper.java | 18 ++--- 3 files changed, 80 insertions(+), 46 deletions(-) create mode 100644 core/src/main/java/org/mobilitydata/gtfsvalidator/util/HttpGetUtil.java diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java index 5e65c0874c..c622c10217 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java @@ -33,6 +33,7 @@ import org.apache.http.impl.client.HttpClients; import org.mobilitydata.gtfsvalidator.notice.InvalidInputFilesInSubfolderNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; +import org.mobilitydata.gtfsvalidator.util.HttpGetUtil; /** * GtfsInput provides a common interface for reading GTFS data, either from a ZIP archive or from a @@ -42,8 +43,6 @@ public abstract class GtfsInput implements Closeable { public static final String invalidInputMessage = "At least 1 GTFS file is in a subfolder. All GTFS files must reside at the root level directly."; - public static final String USER_AGENT_PREFIX = "MobilityData GTFS-Validator"; - /** * Creates a specific GtfsInput to read data from the given path. * @@ -147,7 +146,7 @@ public static GtfsInput createFromUrl( Files.createDirectories(targetDirectory); } try (OutputStream outputStream = Files.newOutputStream(targetPath)) { - loadFromUrl(sourceUrl, outputStream, validatorVersion); + HttpGetUtil.loadFromUrl(sourceUrl, outputStream, validatorVersion); } return createFromPath(targetPath, noticeContainer); } @@ -166,7 +165,7 @@ public static GtfsInput createFromUrlInMemory( URL sourceUrl, NoticeContainer noticeContainer, String validatorVersion) throws IOException, URISyntaxException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { - loadFromUrl(sourceUrl, outputStream, validatorVersion); + HttpGetUtil.loadFromUrl(sourceUrl, outputStream, validatorVersion); File zipFile = new File(sourceUrl.toString()); String fileName = zipFile.getName().replace(".zip", ""); if (containsGtfsFileInSubfolder( @@ -179,40 +178,6 @@ public static GtfsInput createFromUrlInMemory( } } - /** - * Downloads data from network. - * - * @param sourceUrl the fully qualified URL - * @param outputStream the output stream - * @param validatorVersion - * @throws IOException if no file could not be found at the specified location - * @throws URISyntaxException if URL is malformed - */ - private static void loadFromUrl(URL sourceUrl, OutputStream outputStream, String validatorVersion) - throws IOException, URISyntaxException { - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - HttpGet httpGet = new HttpGet(sourceUrl.toURI()); - httpGet.setHeader("User-Agent", getUserAgent(validatorVersion)); - try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) { - httpResponse.getEntity().writeTo(outputStream); - } - } - } - - /** - * @param validatorVersion version of the validator - * @return the user agent string in the format: "MobilityData GTFS-Validator/{validatorVersion} - * (Java {java version})" - */ - private static String getUserAgent(String validatorVersion) { - return USER_AGENT_PREFIX - + "/" - + (validatorVersion != null ? validatorVersion : "") - + " (Java " - + System.getProperty("java.version") - + ")"; - } - /** * Lists all files inside the GTFS dataset, even if they are not CSV and do not have .txt * extension. diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/util/HttpGetUtil.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/util/HttpGetUtil.java new file mode 100644 index 0000000000..3fc8a8af05 --- /dev/null +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/util/HttpGetUtil.java @@ -0,0 +1,67 @@ +/* + * Copyright 2023 MobilityData + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.mobilitydata.gtfsvalidator.util; + +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URISyntaxException; +import java.net.URL; + +public class HttpGetUtil { + + public static final String USER_AGENT_PREFIX = "MobilityData GTFS-Validator"; + + /** + * @param validatorVersion version of the validator + * @return the user agent string in the format: "MobilityData GTFS-Validator/{validatorVersion} + * (Java {java version})" + */ + public static String getUserAgent(String validatorVersion) { + return USER_AGENT_PREFIX + + "/" + + (validatorVersion != null ? validatorVersion : "") + + " (Java " + + System.getProperty("java.version") + + ")"; + } + + /** + * Downloads data from network. + * + * @param sourceUrl the fully qualified URL + * @param outputStream the output stream + * @param validatorVersion the version of the validator + * @throws IOException if no file could not be found at the specified location + * @throws URISyntaxException if URL is malformed + */ + public static void loadFromUrl(URL sourceUrl, OutputStream outputStream, String validatorVersion) + throws IOException, URISyntaxException { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet httpGet = new HttpGet(sourceUrl.toURI()); + httpGet.setHeader("User-Agent", getUserAgent(validatorVersion)); + try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) { + httpResponse.getEntity().writeTo(outputStream); + } + } + } +} diff --git a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java index 9ae9015b12..f9aebaf833 100644 --- a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java +++ b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java @@ -11,6 +11,8 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; + +import org.mobilitydata.gtfsvalidator.util.HttpGetUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.context.ApplicationContext; @@ -105,14 +107,14 @@ public JobMetadata getJobMetadata(String jobId) { */ public void saveJobFileFromUrl(String jobId, String url) throws Exception { // Read file into memory - var urlInputStream = new BufferedInputStream(new URL(url).openStream()); - - // Upload to GCS - var blobId = BlobId.of(USER_UPLOAD_BUCKET_NAME, jobId + "/" + jobId + ".zip"); - var mimeType = "application/zip"; - var blobInfo = BlobInfo.newBuilder(blobId).setContentType(mimeType).build(); - var fileBytes = urlInputStream.readAllBytes(); - storage.create(blobInfo, fileBytes); + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + HttpGetUtil.loadFromUrl(new URL(url), outputStream, null); + var blobId = BlobId.of(USER_UPLOAD_BUCKET_NAME, jobId + "/" + FILE_NAME); + var mimeType = "application/zip"; + var blobInfo = BlobInfo.newBuilder(blobId).setContentType(mimeType).build(); + var fileBytes = outputStream.toByteArray(); + storage.create(blobInfo, fileBytes); + } } /** Generates a job-specific signed URL for uploading a file to GCS. */ From 9e4454d26362553fd68755c0f6207eca521a9711 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 12 Jan 2024 12:07:04 -0500 Subject: [PATCH 02/13] deploy to staging --- .github/workflows/stg_web_svc_merge.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/stg_web_svc_merge.yml b/.github/workflows/stg_web_svc_merge.yml index e8c1b7b9e7..a98bf1f8e2 100644 --- a/.github/workflows/stg_web_svc_merge.yml +++ b/.github/workflows/stg_web_svc_merge.yml @@ -4,6 +4,7 @@ on: push: branches: - master + - * jobs: From 0fece695be90f634ff859b4f7671c504ec482fd0 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 12 Jan 2024 12:08:21 -0500 Subject: [PATCH 03/13] deploy to staging --- .github/workflows/stg_web_svc_merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stg_web_svc_merge.yml b/.github/workflows/stg_web_svc_merge.yml index a98bf1f8e2..bf23b3084d 100644 --- a/.github/workflows/stg_web_svc_merge.yml +++ b/.github/workflows/stg_web_svc_merge.yml @@ -4,7 +4,7 @@ on: push: branches: - master - - * + - '*' jobs: From 27881f054a3c899133c4a4997fc590872560d331 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 12 Jan 2024 12:10:18 -0500 Subject: [PATCH 04/13] deploy to staging --- .github/workflows/stg_web_svc_merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stg_web_svc_merge.yml b/.github/workflows/stg_web_svc_merge.yml index bf23b3084d..b1d210142b 100644 --- a/.github/workflows/stg_web_svc_merge.yml +++ b/.github/workflows/stg_web_svc_merge.yml @@ -4,7 +4,7 @@ on: push: branches: - master - - '*' + - 'fix/1632-authorize-upload' jobs: From 34703e98d66ef4ea9c4f02b164e70748860aee57 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 12 Jan 2024 12:19:08 -0500 Subject: [PATCH 05/13] Revert "deploy to staging" This reverts commit 902bdbc044567bd75c3f936df59748b3b29a96db. --- .github/workflows/stg_web_svc_merge.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stg_web_svc_merge.yml b/.github/workflows/stg_web_svc_merge.yml index b1d210142b..bf23b3084d 100644 --- a/.github/workflows/stg_web_svc_merge.yml +++ b/.github/workflows/stg_web_svc_merge.yml @@ -4,7 +4,7 @@ on: push: branches: - master - - 'fix/1632-authorize-upload' + - '*' jobs: From d90400f05a53abbff94a350c84f96c457962d1bd Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 12 Jan 2024 12:20:21 -0500 Subject: [PATCH 06/13] revert deployment --- .github/workflows/stg_web_svc_merge.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/stg_web_svc_merge.yml b/.github/workflows/stg_web_svc_merge.yml index bf23b3084d..e8c1b7b9e7 100644 --- a/.github/workflows/stg_web_svc_merge.yml +++ b/.github/workflows/stg_web_svc_merge.yml @@ -4,7 +4,6 @@ on: push: branches: - master - - '*' jobs: From 84d10ea0ce6ce3ad637536a72282e4401c83d405 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 12 Jan 2024 13:32:15 -0500 Subject: [PATCH 07/13] upate unit tests --- .../controller/ValidationController.java | 3 ++- .../web/service/util/StorageHelper.java | 17 ++++++++-------- .../controller/CreateJobEndpointTest.java | 20 +++++++++++++------ 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/controller/ValidationController.java b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/controller/ValidationController.java index 61c8f1b3d1..765fa4d2be 100644 --- a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/controller/ValidationController.java +++ b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/controller/ValidationController.java @@ -62,7 +62,8 @@ public CreateJobResponse createJob(@RequestBody CreateJobRequest body) { storageHelper.saveJobMetadata(new JobMetadata(jobId, body.getCountryCode())); } if (!Strings.isNullOrEmpty(body.getUrl())) { - storageHelper.saveJobFileFromUrl(jobId, body.getUrl()); + var validatorVersion = versionResolver.resolveCurrentVersion(); + storageHelper.saveJobFileFromUrl(jobId, body.getUrl(), validatorVersion.orElse(null)); } else { uploadUrl = storageHelper.generateUniqueUploadUrl(jobId); } diff --git a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java index f9aebaf833..f19eb396b6 100644 --- a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java +++ b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java @@ -11,7 +11,6 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.TimeUnit; - import org.mobilitydata.gtfsvalidator.util.HttpGetUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -103,17 +102,19 @@ public JobMetadata getJobMetadata(String jobId) { * * @param jobId * @param url + * @param validatorVersion * @throws Exception */ - public void saveJobFileFromUrl(String jobId, String url) throws Exception { + public void saveJobFileFromUrl(String jobId, String url, String validatorVersion) + throws Exception { // Read file into memory try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { - HttpGetUtil.loadFromUrl(new URL(url), outputStream, null); - var blobId = BlobId.of(USER_UPLOAD_BUCKET_NAME, jobId + "/" + FILE_NAME); - var mimeType = "application/zip"; - var blobInfo = BlobInfo.newBuilder(blobId).setContentType(mimeType).build(); - var fileBytes = outputStream.toByteArray(); - storage.create(blobInfo, fileBytes); + HttpGetUtil.loadFromUrl(new URL(url), outputStream, validatorVersion); + var blobId = BlobId.of(USER_UPLOAD_BUCKET_NAME, jobId + "/" + FILE_NAME); + var mimeType = "application/zip"; + var blobInfo = BlobInfo.newBuilder(blobId).setContentType(mimeType).build(); + var fileBytes = outputStream.toByteArray(); + storage.create(blobInfo, fileBytes); } } diff --git a/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/controller/CreateJobEndpointTest.java b/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/controller/CreateJobEndpointTest.java index 556abf5fce..86e4bee783 100644 --- a/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/controller/CreateJobEndpointTest.java +++ b/web/service/src/test/java/org/mobilitydata/gtfsvalidator/web/service/controller/CreateJobEndpointTest.java @@ -4,9 +4,12 @@ import static org.mockito.Mockito.*; import com.fasterxml.jackson.databind.ObjectMapper; +import java.io.IOException; import java.net.URL; +import java.util.Optional; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mobilitydata.gtfsvalidator.util.VersionResolver; import org.mobilitydata.gtfsvalidator.web.service.util.JobMetadata; import org.mobilitydata.gtfsvalidator.web.service.util.StorageHelper; import org.mobilitydata.gtfsvalidator.web.service.util.ValidationHandler; @@ -32,10 +35,14 @@ public class CreateJobEndpointTest { @Captor ArgumentCaptor jobMetadataCaptor; + @MockBean private VersionResolver versionResolver; + private final ObjectMapper mapper = new ObjectMapper(); private String testJobId; private String testUploadUrl; + private final String VALIDATOR_TEST_VERSION = "1.0.0"; + private void makeCreateJobRequestAndCheckResult( CreateJobRequest request, String expectedJobId, String expectedUploadUrl) throws Exception { var json = mapper.writeValueAsString(request); @@ -51,10 +58,11 @@ private void makeCreateJobRequestAndCheckResult( } @BeforeEach - public void setUp() { + public void setUp() throws IOException { testJobId = "123"; testUploadUrl = "https://gcs.io/bucket/123"; doReturn(testJobId).when(storageHelper).createNewJobId(); + doReturn(Optional.of(VALIDATOR_TEST_VERSION)).when(versionResolver).resolveCurrentVersion(); } @Test @@ -67,7 +75,7 @@ public void createJobWithNoUrlNoCountryCode() throws Exception { // should not call saveJobMetadata verify(storageHelper, times(0)).saveJobMetadata(any(JobMetadata.class)); // should not call saveJobFileFromUrl - verify(storageHelper, times(0)).saveJobFileFromUrl(anyString(), anyString()); + verify(storageHelper, times(0)).saveJobFileFromUrl(anyString(), anyString(), anyString()); } @Test @@ -85,7 +93,7 @@ public void createJobWithCountryCodeButNoUrl() throws Exception { assert jobMetadata.getCountryCode().equals("US"); // should not call saveJobFileFromUrl - verify(storageHelper, times(0)).saveJobFileFromUrl(anyString(), anyString()); + verify(storageHelper, times(0)).saveJobFileFromUrl(anyString(), anyString(), anyString()); } @Test @@ -99,7 +107,7 @@ public void createJobWithUrlButNoCountryCode() throws Exception { // should not call saveJobMetadata verify(storageHelper, times(0)).saveJobMetadata(any(JobMetadata.class)); // should saveJobFileFromUrl - verify(storageHelper, times(1)).saveJobFileFromUrl(testJobId, url); + verify(storageHelper, times(1)).saveJobFileFromUrl(testJobId, url, VALIDATOR_TEST_VERSION); } @Test @@ -117,7 +125,7 @@ public void createJobWithUrlAndCountryCode() throws Exception { assert jobMetadata.getJobId().equals(testJobId); assert jobMetadata.getCountryCode().equals("US"); // should saveJobFileFromUrl - verify(storageHelper, times(1)).saveJobFileFromUrl(testJobId, url); + verify(storageHelper, times(1)).saveJobFileFromUrl(testJobId, url, VALIDATOR_TEST_VERSION); } @Test @@ -140,7 +148,7 @@ public void createJobShouldReturn500ErrorIfSaveJobFileFromUrlThrowsException() t doReturn(testJobId).when(storageHelper).createNewJobId(); doThrow(new RuntimeException("test exception")) .when(storageHelper) - .saveJobFileFromUrl(any(), any()); + .saveJobFileFromUrl(anyString(), anyString(), anyString()); String url = "http://myfilehost.com/myfile.zip"; var request = new CreateJobRequest("US", url); var json = mapper.writeValueAsString(request); From 3981f932400558e96ba2d1369be10216b207b3a2 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 12 Jan 2024 13:33:35 -0500 Subject: [PATCH 08/13] deploying to staging --- .github/workflows/stg_web_svc_merge.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/stg_web_svc_merge.yml b/.github/workflows/stg_web_svc_merge.yml index e8c1b7b9e7..b1d210142b 100644 --- a/.github/workflows/stg_web_svc_merge.yml +++ b/.github/workflows/stg_web_svc_merge.yml @@ -4,6 +4,7 @@ on: push: branches: - master + - 'fix/1632-authorize-upload' jobs: From 6345ab27f8cbebead18e2056f9d7797e748d2c47 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 12 Jan 2024 13:40:09 -0500 Subject: [PATCH 09/13] Revert "deploying to staging" This reverts commit 354cf4394b29334c705c21d86abae8a7897e73f2. --- .github/workflows/stg_web_svc_merge.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/stg_web_svc_merge.yml b/.github/workflows/stg_web_svc_merge.yml index b1d210142b..e8c1b7b9e7 100644 --- a/.github/workflows/stg_web_svc_merge.yml +++ b/.github/workflows/stg_web_svc_merge.yml @@ -4,7 +4,6 @@ on: push: branches: - master - - 'fix/1632-authorize-upload' jobs: From b497fc668e53588476cbb3eff15fd5b089395aea Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Fri, 12 Jan 2024 13:42:45 -0500 Subject: [PATCH 10/13] fix formatting --- .../gtfsvalidator/input/GtfsInput.java | 4 - .../gtfsvalidator/util/HttpGetUtil.java | 74 +++++++++---------- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java index c622c10217..05738d1f9c 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/input/GtfsInput.java @@ -27,10 +27,6 @@ import java.util.zip.ZipInputStream; import org.apache.commons.compress.archivers.zip.ZipFile; import org.apache.commons.compress.utils.SeekableInMemoryByteChannel; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; import org.mobilitydata.gtfsvalidator.notice.InvalidInputFilesInSubfolderNotice; import org.mobilitydata.gtfsvalidator.notice.NoticeContainer; import org.mobilitydata.gtfsvalidator.util.HttpGetUtil; diff --git a/core/src/main/java/org/mobilitydata/gtfsvalidator/util/HttpGetUtil.java b/core/src/main/java/org/mobilitydata/gtfsvalidator/util/HttpGetUtil.java index 3fc8a8af05..6290be4a5f 100644 --- a/core/src/main/java/org/mobilitydata/gtfsvalidator/util/HttpGetUtil.java +++ b/core/src/main/java/org/mobilitydata/gtfsvalidator/util/HttpGetUtil.java @@ -16,52 +16,50 @@ package org.mobilitydata.gtfsvalidator.util; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClients; - import java.io.IOException; -import java.io.InputStream; import java.io.OutputStream; import java.net.URISyntaxException; import java.net.URL; +import org.apache.http.client.methods.CloseableHttpResponse; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClients; public class HttpGetUtil { - public static final String USER_AGENT_PREFIX = "MobilityData GTFS-Validator"; + public static final String USER_AGENT_PREFIX = "MobilityData GTFS-Validator"; - /** - * @param validatorVersion version of the validator - * @return the user agent string in the format: "MobilityData GTFS-Validator/{validatorVersion} - * (Java {java version})" - */ - public static String getUserAgent(String validatorVersion) { - return USER_AGENT_PREFIX - + "/" - + (validatorVersion != null ? validatorVersion : "") - + " (Java " - + System.getProperty("java.version") - + ")"; - } + /** + * @param validatorVersion version of the validator + * @return the user agent string in the format: "MobilityData GTFS-Validator/{validatorVersion} + * (Java {java version})" + */ + public static String getUserAgent(String validatorVersion) { + return USER_AGENT_PREFIX + + "/" + + (validatorVersion != null ? validatorVersion : "") + + " (Java " + + System.getProperty("java.version") + + ")"; + } - /** - * Downloads data from network. - * - * @param sourceUrl the fully qualified URL - * @param outputStream the output stream - * @param validatorVersion the version of the validator - * @throws IOException if no file could not be found at the specified location - * @throws URISyntaxException if URL is malformed - */ - public static void loadFromUrl(URL sourceUrl, OutputStream outputStream, String validatorVersion) - throws IOException, URISyntaxException { - try (CloseableHttpClient httpClient = HttpClients.createDefault()) { - HttpGet httpGet = new HttpGet(sourceUrl.toURI()); - httpGet.setHeader("User-Agent", getUserAgent(validatorVersion)); - try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) { - httpResponse.getEntity().writeTo(outputStream); - } - } + /** + * Downloads data from network. + * + * @param sourceUrl the fully qualified URL + * @param outputStream the output stream + * @param validatorVersion the version of the validator + * @throws IOException if no file could not be found at the specified location + * @throws URISyntaxException if URL is malformed + */ + public static void loadFromUrl(URL sourceUrl, OutputStream outputStream, String validatorVersion) + throws IOException, URISyntaxException { + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet httpGet = new HttpGet(sourceUrl.toURI()); + httpGet.setHeader("User-Agent", getUserAgent(validatorVersion)); + try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) { + httpResponse.getEntity().writeTo(outputStream); + } } + } } From 0fa3eca948dadd32e67c737a908ded7158f2e405 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 15 Jan 2024 14:45:34 -0500 Subject: [PATCH 11/13] reduce memory footprint by sending file content via gcp write method --- .../web/service/util/StorageHelper.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java index f19eb396b6..4d69a8aee2 100644 --- a/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java +++ b/web/service/src/main/java/org/mobilitydata/gtfsvalidator/web/service/util/StorageHelper.java @@ -1,9 +1,11 @@ package org.mobilitydata.gtfsvalidator.web.service.util; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.cloud.WriteChannel; import com.google.cloud.storage.*; import java.io.*; import java.net.URL; +import java.nio.channels.Channels; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -107,14 +109,14 @@ public JobMetadata getJobMetadata(String jobId) { */ public void saveJobFileFromUrl(String jobId, String url, String validatorVersion) throws Exception { - // Read file into memory - try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + var blobId = BlobId.of(USER_UPLOAD_BUCKET_NAME, jobId + "/" + FILE_NAME); + var blobInfo = BlobInfo.newBuilder(blobId).setContentType("application/zip").build(); + URL signedURL = + storage.signUrl( + blobInfo, 1, TimeUnit.HOURS, Storage.SignUrlOption.httpMethod(HttpMethod.POST)); + try (WriteChannel writer = storage.writer(signedURL)) { + OutputStream outputStream = Channels.newOutputStream(writer); HttpGetUtil.loadFromUrl(new URL(url), outputStream, validatorVersion); - var blobId = BlobId.of(USER_UPLOAD_BUCKET_NAME, jobId + "/" + FILE_NAME); - var mimeType = "application/zip"; - var blobInfo = BlobInfo.newBuilder(blobId).setContentType(mimeType).build(); - var fileBytes = outputStream.toByteArray(); - storage.create(blobInfo, fileBytes); } } From 36712f4da8d20123430b21b0c814e30640b537d9 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 15 Jan 2024 15:48:00 -0500 Subject: [PATCH 12/13] deploy to staging --- .github/workflows/stg_web_svc_merge.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/stg_web_svc_merge.yml b/.github/workflows/stg_web_svc_merge.yml index e8c1b7b9e7..b1d210142b 100644 --- a/.github/workflows/stg_web_svc_merge.yml +++ b/.github/workflows/stg_web_svc_merge.yml @@ -4,6 +4,7 @@ on: push: branches: - master + - 'fix/1632-authorize-upload' jobs: From 05e81bd9d18dbd99ad69747a0d3ba4d033310954 Mon Sep 17 00:00:00 2001 From: David Gamez Diaz <1192523+davidgamez@users.noreply.github.com> Date: Mon, 15 Jan 2024 16:26:24 -0500 Subject: [PATCH 13/13] Revert "deploy to staging" This reverts commit 36712f4da8d20123430b21b0c814e30640b537d9. --- .github/workflows/stg_web_svc_merge.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/stg_web_svc_merge.yml b/.github/workflows/stg_web_svc_merge.yml index b1d210142b..e8c1b7b9e7 100644 --- a/.github/workflows/stg_web_svc_merge.yml +++ b/.github/workflows/stg_web_svc_merge.yml @@ -4,7 +4,6 @@ on: push: branches: - master - - 'fix/1632-authorize-upload' jobs: