basic url cleaning test passes

src/docrawler/url.go

@@ -1,3 +1,22 @@
 package main
 
-// scheme://[userinfo@]host/path[?query][#fragment]
+import (
+	"net/url"
+	"strings"
+)
+
+// cleanURL takes a URL, normalizes it by downcasing the host part, and reassembles it.
+// returns the updated host and reassembled URL
+func cleanURL(dirtyURL string) (string, string) {
+	// parse the URL into its parts
+	u, err := url.Parse(dirtyURL)
+	if err != nil {
+		return "", dirtyURL // TODO what should we really return here?
+	}
+
+	// convert host to all lower case
+	u.Host = strings.ToLower(u.Host) 
+
+	// return the updated host and reassembled URL
+	return u.Host, u.String()
+}

src/docrawler/url_test.go

@@ -1,13 +1,16 @@
 package main
 
 import (
+	"fmt"
 	"testing"
 )
 
 // TestURLCleaner tests cleaning up (regularization) of URLs
 func TestURLCleaner(t *testing.T) {
 	host, url := cleanURL("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo")
-	if host != "somehost.com" {
+	if host != "somehost.com:8765" {
+		fmt.Printf("   Got: '%v'\n", host)
+		fmt.Printf("Wanted: '%v'\n", "somehost.com:8765")
 		t.Error("host extraction failed")
 	}
 	if url != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {