diff --git a/src/docrawler/url.go b/src/docrawler/url.go index 055886f..2d088c4 100644 --- a/src/docrawler/url.go +++ b/src/docrawler/url.go @@ -1,3 +1,22 @@ package main -// scheme://[userinfo@]host/path[?query][#fragment] +import ( + "net/url" + "strings" +) + +// cleanURL takes a URL, normalizes it by downcasing the host part, and reassembles it. +// returns the updated host and reassembled URL +func cleanURL(dirtyURL string) (string, string) { + // parse the URL into its parts + u, err := url.Parse(dirtyURL) + if err != nil { + return "", dirtyURL // TODO what should we really return here? + } + + // convert host to all lower case + u.Host = strings.ToLower(u.Host) + + // return the updated host and reassembled URL + return u.Host, u.String() +} diff --git a/src/docrawler/url_test.go b/src/docrawler/url_test.go index ef08652..61b7fd9 100644 --- a/src/docrawler/url_test.go +++ b/src/docrawler/url_test.go @@ -1,13 +1,16 @@ package main import ( + "fmt" "testing" ) // TestURLCleaner tests cleaning up (regularization) of URLs func TestURLCleaner(t *testing.T) { host, url := cleanURL("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo") - if host != "somehost.com" { + if host != "somehost.com:8765" { + fmt.Printf(" Got: '%v'\n", host) + fmt.Printf("Wanted: '%v'\n", "somehost.com:8765") t.Error("host extraction failed") } if url != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {