Merge pull request #129 from internetarchive/fix/url-parsing

Add additional tests and validate current URL behavior
internetarchive · Aug 13, 2024 · c220114 · c220114
2 parents 9dbc9ae + 2b105d5
commit c220114
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 17 deletions.
diff --git a/internal/pkg/utils/url.go b/internal/pkg/utils/url.go
@@ -35,23 +35,6 @@ func URLToString(u *url.URL) string {
 		}
 	}
 
-	tempHost, err := idna.ToASCII(u.Hostname())
-	if err != nil {
-		slog.Warn("cannot encode punycode hostname to ASCII", "error", err)
-		tempHost = u.Hostname()
-	}
-
-	if strings.Contains(tempHost, ":") && !(strings.HasPrefix(tempHost, "[") && strings.HasSuffix(tempHost, "]")) {
-		tempHost = "[" + tempHost + "]"
-	}
-
-	port := u.Port()
-	if len(port) > 0 {
-		u.Host = tempHost + ":" + port
-	} else {
-		u.Host = tempHost
-	}
-
 	return u.String()
 }
 

diff --git a/internal/pkg/utils/url_test.go b/internal/pkg/utils/url_test.go
@@ -30,3 +30,68 @@ func TestURLToStringPunycodeWithPort(t *testing.T) {
 		t.Fatalf("Expected %s, got %s", expected, actual)
 	}
 }
+
+func TestURLToStringUnicodetoIDNA(t *testing.T) {
+	u, err := url.Parse("https://о-змладйвеклблнозеж.xn--p1ia:8080/pic/file/map_of_sarlat.pdf")
+	if err != nil {
+		t.Fatalf("Error parsing URL: %v", err)
+	}
+
+	expected := "https://xn----8sbddjhbicfsohgbg1aeo.xn--p1ia:8080/pic/file/map_of_sarlat.pdf"
+	actual := URLToString(u)
+	if actual != expected {
+		t.Fatalf("Expected %s, got %s", expected, actual)
+	}
+}
+
+func TestURLToStringWithPath(t *testing.T) {
+	u, err := url.Parse("http://παράδειγμα.δοκιμή/Αρχική_σελίδα")
+	if err != nil {
+		t.Fatalf("Error parsing URL: %v", err)
+	}
+
+	expected := "http://xn--hxajbheg2az3al.xn--jxalpdlp/%CE%91%CF%81%CF%87%CE%B9%CE%BA%CE%AE_%CF%83%CE%B5%CE%BB%CE%AF%CE%B4%CE%B1"
+	actual := URLToString(u)
+	if actual != expected {
+		t.Fatalf("Expected %s, got %s", expected, actual)
+	}
+}
+
+func TestURLToStringUnicodetoIDNAWithPort(t *testing.T) {
+	u, err := url.Parse("https://о-змладйвеклблнозеж.xn--p1ia:8080/pic/file/map_of_sarlat.pdf")
+	if err != nil {
+		t.Fatalf("Error parsing URL: %v", err)
+	}
+
+	expected := "https://xn----8sbddjhbicfsohgbg1aeo.xn--p1ia:8080/pic/file/map_of_sarlat.pdf"
+	actual := URLToString(u)
+	if actual != expected {
+		t.Fatalf("Expected %s, got %s", expected, actual)
+	}
+}
+
+func TestURLwithIPv6(t *testing.T) {
+	u, err := url.Parse("https://[2600:4040:23c7:a620:3642:ebaa:ab23:735e]/test")
+	if err != nil {
+		t.Fatalf("Error parsing URL: %v", err)
+	}
+
+	expected := "https://[2600:4040:23c7:a620:3642:ebaa:ab23:735e]/test"
+	actual := URLToString(u)
+	if actual != expected {
+		t.Fatalf("Expected %s, got %s", expected, actual)
+	}
+}
+
+func TestURLwithIPv6WithPort(t *testing.T) {
+	u, err := url.Parse("https://[2600:4040:23c7:a620:3642:ebaa:ab23:735e]:8080/test")
+	if err != nil {
+		t.Fatalf("Error parsing URL: %v", err)
+	}
+
+	expected := "https://[2600:4040:23c7:a620:3642:ebaa:ab23:735e]:8080/test"
+	actual := URLToString(u)
+	if actual != expected {
+		t.Fatalf("Expected %s, got %s", expected, actual)
+	}
+}