From b5ea6c4c214955e5b3344d688803078b26bfdec4 Mon Sep 17 00:00:00 2001 From: Eric Martinez Date: Wed, 10 Feb 2021 11:16:20 -0700 Subject: [PATCH 1/2] Adding test to reproduce the observed behavior --- .../json/inline_content_example.json | 29 ++++++ .../io/DirectoryProductSourceTest.java | 88 +++++++++++++++++++ 2 files changed, 117 insertions(+) create mode 100644 etc/test_products/json/inline_content_example.json create mode 100644 src/test/java/gov/usgs/earthquake/product/io/DirectoryProductSourceTest.java diff --git a/etc/test_products/json/inline_content_example.json b/etc/test_products/json/inline_content_example.json new file mode 100644 index 00000000..1b4db4d4 --- /dev/null +++ b/etc/test_products/json/inline_content_example.json @@ -0,0 +1,29 @@ +{ + "contents": [ + { + "length": 157, + "modified": "2021-02-09T16:10:03.000Z", + "path": "", + "sha256": "hkomWf290QUmWGV7jOfxhqOu8kS45NglYAQkIUKOyGI=", + "type": "text/plain", + "url": "data:text/plain;base64,OyBFTU06IHNlbmRIZWFydGJlYXQuZXJiIHVwZGF0ZWQgMjAxOC0wOS0wNQpoZWFydGJlYXRfaGVhZF91cmw9J2h0dHBzOi8vZ2l0aHViLmNvbS91c2dzL3BkbC9ldGMvZXhhbXBsZXMvaGVhcnRiZWF0L3NlbmRIZWFydGJlYXQnOwpoZWFydGJlYXRfcGRsX3ZlcnNpb249Jyc7Cg==" + } + ], + "geometry": null, + "id": { + "code": "lenore", + "source": "prod01-pdl01.cr.usgs.gov", + "type": "heartbeat", + "updateTime": "2021-02-09T16:10:03.529Z" + }, + "links": [], + "properties": { + "original-signature": "MC4CFQCV4gz1rEKKwRfq9VqQJh+ghPFYpAIVAKfqy8jzF62OVMXP5UdrJC7pf632", + "pdl-client-version": "Version 2.5.1 2020-06-25", + "original-signature-version": "v1" + }, + "signature": "nVsadUpcmRYqICPtQ1oYfrReN0E0ILyYiZ4c/IBaG12W+Pr6nx+k3bu6jt2oUVuayOanHi0SLsbEqpvFdwLxicMt689mV5hdwGgHPWQ5Z163LXchSSFtELWdzR/b4vAlW0ehieZPNx6YOelPBNKMk0kvwcrlO6t9dcNKA26t1yl5OpRkjVqjHwZ3mw4hRYIcXvNkwP+PGYhCGiiqao4FAysuyvr94TGek3FJKj1pxiit9HRfAWQXi9aIhKt7hcSbmymovIufROaw/0XLJWErHMKI33EzRbIWKvwaoVpGsPGzI6z+grrCSdF1gIlf96+q+RgHDenQrR5D6a4b8DbX8HkneV7GqxxI9yS3NDf2w2cRBsFDGkKwG8yBhYVNHg3MT/8RoYJAX/QzB12P4gBwn+K1Zm0jEOtmtxMOjhEBrz/3ey06B3+++39PGq8oZpZoAlai08KftcgSjL4E387rFLMMnSP6J89UWBmTIDfj1xk6rf/FLHr3ET1NmgaF70HBexMGoi2+pTc4KxEc9qQQre45t083IH7PvaX32Yw3x8bnZqGFpiuRdVRW1pxYKdCieBOrfs/eRX9PyZpuvipiNOB7ieQNqMPoJ8UjkyadParPLDSfmJmgqU+0WlQ18C2NUf8PbSF3HiNOhltV7Xph7CrFTSnciLxiJO26gtLYKHI=", + "signatureVersion": "v2", + "status": "UPDATE", + "type": "Feature" +} diff --git a/src/test/java/gov/usgs/earthquake/product/io/DirectoryProductSourceTest.java b/src/test/java/gov/usgs/earthquake/product/io/DirectoryProductSourceTest.java new file mode 100644 index 00000000..4c05c108 --- /dev/null +++ b/src/test/java/gov/usgs/earthquake/product/io/DirectoryProductSourceTest.java @@ -0,0 +1,88 @@ +package gov.usgs.earthquake.product.io; + +import gov.usgs.earthquake.product.Content; +import gov.usgs.earthquake.product.FileContent; +import gov.usgs.earthquake.product.Product; +import gov.usgs.util.StreamUtils; + +import java.io.File; +import java.io.FileInputStream; +import java.io.InputStream; + +import javax.json.Json; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +public class DirectoryProductSourceTest { + private static String JSON_EXAMPLE_PRODUCT = "etc/test_products/json/inline_content_example.json"; + private static String STORAGE_BASE_PATH = "product-digest-test-storage"; + + /** + * DirectoryProductSource attempts to convert URLContent to FileContent as part + * of its `streamTo` method. This is typically fine, but if the URLContent is + * for the empty path, this causes problems because a file does not exist on + * disk and leads to FileNotFoundException downstream. + * + * This test ensures URLContent referencing the empty path is not converted to + * FileContent. In this case, the URLContent _should_ typically reference a data + * URL. + */ + @Test + public void directoryProductSource() { + try (InputStream fin = new FileInputStream(JSON_EXAMPLE_PRODUCT); + InputStream in = StreamUtils.getInputStream(fin);) { + File storageDirectory = new File(STORAGE_BASE_PATH); + + // Create product from JSON file and stream to directory [as XML] + Product jsonProduct = new JsonProduct().getProduct(Json.createReader(in).readObject()); + DirectoryProductHandler outputHandler = new DirectoryProductHandler(storageDirectory); + ObjectProductSource outputSource = new ObjectProductSource(jsonProduct); + outputSource.streamTo(outputHandler); + + // Create product from directory + ObjectProductHandler inputHandler = new ObjectProductHandler(); + DirectoryProductSource inputSource = new DirectoryProductSource(storageDirectory); + inputSource.streamTo(inputHandler); + Product directoryProduct = inputHandler.getProduct(); + + // Ensure the inline (empty path) content is not converted to file content + Content content = directoryProduct.getContents().get(""); + Assert.assertNotNull(content); + Assert.assertTrue("Empty path content not of type FileContent", !(content instanceof FileContent)); + } catch (Exception e) { + Assert.fail(e.getMessage()); + } + } + + /** + * Clean up the temporary storage directory after test(s). + * + */ + @After + public void cleanup() { + // Cleanup tmp storage directory + File storageDirectory = new File(STORAGE_BASE_PATH); + if (storageDirectory.isDirectory()) { + deleteDirectory(storageDirectory); + } + } + + /** + * Helper method to recursively delete a directory (or file). + * + * @param file The directory (or file) to delete. + */ + private boolean deleteDirectory(File directory) { + File[] files = directory.listFiles(); + + if (files != null) { + for (File file : files) { + deleteDirectory(file); + } + } + + return directory.delete(); + } +} \ No newline at end of file From 3760da102bf71f940b8f3929578e2da2aa46f290 Mon Sep 17 00:00:00 2001 From: Eric Martinez Date: Wed, 10 Feb 2021 11:17:10 -0700 Subject: [PATCH 2/2] Do not convert URLContent to FileContent if the path is empty (i.e., inline content) --- .../product/io/DirectoryProductSource.java | 54 +++++++++---------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/src/main/java/gov/usgs/earthquake/product/io/DirectoryProductSource.java b/src/main/java/gov/usgs/earthquake/product/io/DirectoryProductSource.java index cd49f26a..f9c45e47 100644 --- a/src/main/java/gov/usgs/earthquake/product/io/DirectoryProductSource.java +++ b/src/main/java/gov/usgs/earthquake/product/io/DirectoryProductSource.java @@ -18,7 +18,7 @@ /** * Load a product from a Directory. - * + * * Usually a directory is created using DirectoryProductOutput. It should * contain a product xml file named "product.xml". All other files are treated * as attachments. @@ -28,36 +28,30 @@ public class DirectoryProductSource implements ProductSource { /** The directory this product input references. */ private File directory; - private static final Logger LOGGER = Logger - .getLogger(DirectoryProductSource.class.getName()); + private static final Logger LOGGER = Logger.getLogger(DirectoryProductSource.class.getName()); /** * Construct a new DirectoryProductSource object. - * - * @param directory - * the directory containing a product. + * + * @param directory the directory containing a product. */ public DirectoryProductSource(final File directory) { this.directory = directory; } /** - * Load Product from a directory, then send product events to the - * ProductOutput. - * - * @param out - * the ProductOutput that will receive the product. + * Load Product from a directory, then send product events to the ProductOutput. + * + * @param out the ProductOutput that will receive the product. */ public void streamTo(ProductHandler out) throws Exception { InputStream in = null; try { - in = StreamUtils.getInputStream(new File(directory, - DirectoryProductHandler.PRODUCT_XML_FILENAME)); + in = StreamUtils.getInputStream(new File(directory, DirectoryProductHandler.PRODUCT_XML_FILENAME)); // load product from xml - Product product = ObjectProductHandler - .getProduct(new XmlProductSource(in)); + Product product = ObjectProductHandler.getProduct(new XmlProductSource(in)); // Convert URLContent to FileContent Map contents = product.getContents(); @@ -66,26 +60,27 @@ public void streamTo(ProductHandler out) throws Exception { for (String key : contents.keySet()) { urlContent = contents.get(key); if (urlContent instanceof URLContent) { - File filePath = new File(directory, key); - if (filePath.exists()) { - FileContent fileContent = new FileContent(filePath); - fileContent.setContentType(urlContent.getContentType()); - fileContent.setLastModified(urlContent.getLastModified()); - fileContent.setLength(urlContent.getLength()); - // go direct to file based on key - contents.put(key, fileContent); - } else { - // old way - contents.put(key, new FileContent((URLContent) urlContent)); - } foundURLContent = true; + if (!"".equals(key)) { + File filePath = new File(directory, key); + if (filePath.exists()) { + FileContent fileContent = new FileContent(filePath); + fileContent.setContentType(urlContent.getContentType()); + fileContent.setLastModified(urlContent.getLastModified()); + fileContent.setLength(urlContent.getLength()); + // go direct to file based on key + contents.put(key, fileContent); + } else { + // old way + contents.put(key, new FileContent((URLContent) urlContent)); + } + } } } if (!foundURLContent) { LOGGER.log(Level.FINER, - "[DirectoryProductSource] Product does not have any " - + " URLContent. Scraping directory for files."); + "[DirectoryProductSource] Product does not have any " + " URLContent. Scraping directory for files."); // load contents from directory contents.putAll(FileContent.getDirectoryContents(directory)); @@ -101,7 +96,6 @@ public void streamTo(ProductHandler out) throws Exception { } } - /** * Free any resources associated with this source. */