Small fixes to fix memory usage (#58)

internetarchive · Jan 13, 2024 · 5af2058 · 5af2058
1 parent a8bd42a
commit 5af2058
Show file tree

Hide file tree

Showing 5 changed files with 10 additions and 19 deletions.
diff --git a/internal/pkg/crawl/capture.go b/internal/pkg/crawl/capture.go
@@ -232,18 +232,18 @@ func (c *Crawl) Capture(item *frontier.Item) {
 
 	// Execute site-specific code on the request, before sending it
 	if tiktok.IsTikTokURL(utils.URLToString(item.URL)) {
-		req = tiktok.AddHeaders(req)
+		tiktok.AddHeaders(req)
 	} else if telegram.IsTelegramURL(utils.URLToString(item.URL)) && !telegram.IsTelegramEmbedURL(utils.URLToString(item.URL)) {
 		// If the URL is a Telegram URL, we make an embed URL out of it
-		embedURL := telegram.CreateEmbedURL(item.URL)
+		telegram.TransformURL(item.URL)
 
 		// Then we create an item
-		embedItem := frontier.NewItem(embedURL, item, item.Type, item.Hop, item.ID)
+		embedItem := frontier.NewItem(item.URL, item, item.Type, item.Hop, item.ID)
 
 		// And capture it
 		c.Capture(embedItem)
 	} else if vk.IsVKURL(utils.URLToString(item.URL)) {
-		req = vk.AddHeaders(req)
+		vk.AddHeaders(req)
 	}
 
 	// Execute request

diff --git a/internal/pkg/crawl/sitespecific/telegram/telegram.go b/internal/pkg/crawl/sitespecific/telegram/telegram.go
@@ -13,15 +13,11 @@ func IsTelegramURL(url string) bool {
 	return strings.Contains(url, "/t.me/")
 }
 
-func CreateEmbedURL(URL *url.URL) *url.URL {
+func TransformURL(URL *url.URL) {
 	// Add embed=1 to the URL, without changing the original URL
-	embedURL := *URL
-
-	if len(embedURL.RawQuery) > 0 {
-		embedURL.RawQuery += "&embed=1&mode=tme"
+	if len(URL.RawQuery) > 0 {
+		URL.RawQuery += "&embed=1&mode=tme"
 	} else {
-		embedURL.RawQuery = "embed=1&mode=tme"
+		URL.RawQuery = "embed=1&mode=tme"
 	}
-
-	return &embedURL
 }
diff --git a/internal/pkg/crawl/sitespecific/tiktok/tiktok.go b/internal/pkg/crawl/sitespecific/tiktok/tiktok.go
@@ -9,7 +9,7 @@ func IsTikTokURL(URL string) bool {
 	return strings.Contains(URL, "/tiktok.com")
 }
 
-func AddHeaders(req *http.Request) *http.Request {
+func AddHeaders(req *http.Request) {
 	req.Header.Set("Authority", "www.tiktok.com")
 	req.Header.Set("Sec-Ch-Ua", "\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"99\", \"Microsoft Edge\";v=\"99\"")
 	req.Header.Set("Sec-Ch-Ua-Mobile", "?0")
@@ -23,6 +23,4 @@ func AddHeaders(req *http.Request) *http.Request {
 	req.Header.Set("Sec-Fetch-User", "?1")
 	req.Header.Set("Sec-Fetch-Dest", "document")
 	req.Header.Set("Accept-Language", "en-US,en;q=0.9,fr;q=0.8")
-
-	return req
 }
diff --git a/internal/pkg/crawl/sitespecific/vk/vk.go b/internal/pkg/crawl/sitespecific/vk/vk.go
@@ -9,7 +9,7 @@ func IsVKURL(URL string) bool {
 	return strings.Contains(URL, "/vk.com")
 }
 
-func AddHeaders(req *http.Request) *http.Request {
+func AddHeaders(req *http.Request) {
 	req.Header.Set("Authority", "vk.com")
 	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7")
 	req.Header.Set("Accept-Language", "fr-FR,fr;q=0.9,en-US;q=0.8,en;q=0.7")
@@ -26,6 +26,4 @@ func AddHeaders(req *http.Request) *http.Request {
 
 	// Gives "Your browser is out of date" error when using default UA.
 	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/118.0.0.0 Safari/537.36")
-
-	return req
 }
diff --git a/internal/pkg/frontier/save.go b/internal/pkg/frontier/save.go
@@ -60,7 +60,6 @@ func (f *Frontier) Save() {
 	defer encodeFile.Close()
 
 	// Write to the file
-
 	if err := SyncMapEncode(f.HostPool, encodeFile); err != nil {
 		f.LoggingChan <- &FrontierLogMessage{
 			Fields: logrus.Fields{