Handle non-ascii characters in single and bulk link scraper URLs (#2495)

handle non-ascii characters in urls
Mintplex-Labs · Oct 18, 2024 · 41522cd · 41522cd
1 parent 93d7ce6
commit 41522cd
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 2 deletions.
diff --git a/collector/processLink/convert/generic.js b/collector/processLink/convert/generic.js
@@ -27,7 +27,8 @@ async function scrapeGenericUrl(link, textOnly = false) {
   }
 
   const url = new URL(link);
-  const filename = (url.host + "-" + url.pathname).replace(".", "_");
+  const decodedPathname = decodeURIComponent(url.pathname);
+  const filename = `${url.hostname}${decodedPathname.replace(/\//g, '_')}`;
 
   const data = {
     id: v4(),

diff --git a/collector/utils/extensions/WebsiteDepth/index.js b/collector/utils/extensions/WebsiteDepth/index.js
@@ -108,7 +108,8 @@ async function bulkScrapePages(links, outFolderPath) {
       }
 
       const url = new URL(link);
-      const filename = (url.host + "-" + url.pathname).replace(".", "_");
+      const decodedPathname = decodeURIComponent(url.pathname);
+      const filename = `${url.hostname}${decodedPathname.replace(/\//g, '_')}`;
 
       const data = {
         id: v4(),