add initial failing test for cleanURL

src/docrawler/url.go

@@ -0,0 +1,3 @@
+package main
+
+// scheme://[userinfo@]host/path[?query][#fragment]

src/docrawler/url_test.go

@@ -0,0 +1,16 @@
+package main
+
+import (
+	"testing"
+)
+
+// TestURLCleaner tests cleaning up (regularization) of URLs
+func TestURLCleaner(t *testing.T) {
+	host, url := cleanURL("http://swaits:pass@someHOST.com:8765/blah/blah.html?x=y#foo")
+	if host != "somehost.com" {
+		t.Error("host extraction failed")
+	}
+	if url != "http://swaits:pass@somehost.com:8765/blah/blah.html?x=y#foo" {
+		t.Error("url cleanup failed")
+	}
+}